"""Cloud account lifecycle service. The domain-facing API for everything in the scope → grant model: * **Scope management** — IT admin assigns / transfers owner, sets budget, closes a scope. * **Grant lifecycle** — employee requests, scope owner approves (or owner-direct adds bypass approval), self-renewal, revoke. The service calls into the cloud driver's ``grant_access`` / ``revoke_access``; driver errors surface as ``GrantDispatchError`` (lifecycle grant row stays ``pending``, UI shows the error). * **Scheduled sweeps** — expiry (T-14/T-7/T-1 reminder + T+0 revoke) and inactivity (T+0 notify owner → T+14 notify IT → T+30 disable). * **Daily billing** — persisted per (integration, scope, date). The billing sweep checks monthly rollups against budget + day-over-day anomaly thresholds. All router-facing mutations audit through ``AuditService``. The service NEVER raises from its public methods on driver failures — it returns a structured result so the caller can decide whether to 500 or surface a 502 to the user. """ from __future__ import annotations import hashlib import json as _json import logging import uuid from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from decimal import Decimal from typing import Any from sqlalchemy import case, delete, func, null, or_, select, update from sqlalchemy import true as sa_true from sqlalchemy.dialects.mysql import insert as mysql_insert from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm.attributes import flag_modified from app.audit.service import AuditService from app.auth.models import AuthRole, AuthUserRole from app.core.database import async_session_factory from app.core.distributed_lock import LockBusy, distributed_lock from app.core.exceptions import ( AppException, ConflictError, ForbiddenError, NotFoundError, format_exception_for_log, is_transaction_aborting_db_error, ) from app.core.locale import pick_localized, pick_variant, resolve_lang from app.core.rbac import CLOUD_ADMIN_ROLE, GLOBAL_ADMIN_ROLE from app.core.safe_logging import get_safe_logger from app.core.txn import best_effort, savepoint from app.domains.cmdb.cost_view import ( audit_cost_basis_detail, can_view_invoice_measure, ) from app.domains.cmdb.credential_crypto import ( CredentialDecryptionError, decrypt_fields, ) from app.domains.cmdb.models import ( CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY, CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY_SPIKE, CLOUD_SCOPE_ALERT_RULE_TYPE_MONTHLY, GRANT_NOTIFY_INACTIVE_IT, GRANT_NOTIFY_INACTIVE_OWNER, GRANT_NOTIFY_T1, GRANT_NOTIFY_T7, GRANT_NOTIFY_T14, GRANT_REVOKE_REASON_DRIFT_REMOVED, GRANT_REVOKE_REASON_EXPIRED, GRANT_REVOKE_REASON_INACTIVE, GRANT_REVOKE_REASON_OFFBOARDING, GRANT_REVOKE_REASON_OWNER, GRANT_REVOKE_REASON_SCOPE_CLOSED, GRANT_STATUS_ACTIVE, GRANT_STATUS_DISABLED, GRANT_STATUS_EXPIRED, GRANT_STATUS_PENDING, GRANT_STATUS_REVOKED, GRANT_STATUSES_RESURRECTABLE, GRANT_TTL_HARD_CAP_DAYS, OWNER_CONFIRMATION_KIND_IDLE_NOTIFIED, OWNER_CONFIRMATION_STATUS_RESPONDED, PROD_ENVIRONMENTS, RENEWAL_REVIVAL_MAX_AGE_DAYS, SCOPE_STATUS_ACTIVE, SCOPE_STATUS_CLOSED, SCOPE_TYPE_GCP_BILLING_ACCOUNT, TENANT_LEVEL_SCOPE_TYPES, CmdbCloudAccessGrant, CmdbCloudBillingDaily, CmdbCloudRole, CmdbCloudScope, CmdbCloudScopeAlertFired, CmdbCloudScopeOwnerConfirmation, CmdbIamUser, classify_scope_environment, ) from app.domains.cmdb.role_classifier import is_billing_role from app.domains.cmdb.sweep_settings import ( drop_exceeds_tolerance, get_sweep_drop_tolerance_percent, minimum_observed_within_tolerance, ) from app.identity.person_models import IdentityPerson from app.integrations.cloud_role_governance import ( TIER_NORMAL, TIER_UNGRANTABLE, RoleGovernance, most_severe, ) from app.integrations.driver_base import ( ClearBlockersResult, CloseBlocker, ClosePreflight, CloseScopeResult, CloudAccessContext, CloudDriver, CloudGrantResult, CloudPrincipalNotFoundError, CloudScopeRef, DiscoveredGrant, ) from app.integrations.models import Integration from app.integrations.registry import get_driver_class from app.integrations.service import get_platform_setting from app.notifications.service import ( CardAction, LocalizedText, NotificationEnvelope, NotificationService, ) from app.notifications.templates.cloud import TEMPLATES as _CLOUD_TEMPLATES logger = get_safe_logger(__name__) # Copy for the Tencent CAM first-login credential mail. Resolved at import so a # missing / renamed key fails the smoke-import in CI rather than at the moment # a new hire's cloud account is provisioned. That send bypasses # ``NotificationService.dispatch`` (it must go out even though the notification # pipeline would treat the password as ordinary body text), so this dict is the # only place its wording exists — see ``_grant_and_notify``. # Typed ``dict[str, object]`` purely so it satisfies ``pick_localized``'s # parameter — ``dict`` is invariant in its value type. _CAM_MAIL_TEMPLATE: dict[str, object] = dict( _CLOUD_TEMPLATES["tencent_cam_initial_password"] ) # ── Idle-sweep tunables ─────────────────────────────────────────── # Dedup window for ``cmdb.scope_idle_sweep`` — once a scope has been # notified, the next sweep that lands inside this window skips it. # 14 days = two weekly cron cycles, picked over 7d (= no dedup) and 30d # (= idle accounts stay forgotten too long). Owners who actively want # a longer suppression use the snooze endpoint instead, which sets # ``idle_snoozed_until`` to a concrete date (default +30d). # # Defined here at module scope (before ``CloudLifecycleService``) so # the class's method default ``days=IDLE_SCOPE_DEFAULT_SNOOZE_DAYS`` # resolves at class-body evaluation time. Moving these below the class # triggers ``NameError`` at import. IDLE_SWEEP_DEDUP_DAYS = 14 # Default snooze duration applied by ``snooze_idle_scope`` when the # caller doesn't pass an explicit value. The owner-facing UI passes 30 # explicitly; the constant exists so MCP and ad-hoc callers have a # stable default without re-hardcoding 30 everywhere. IDLE_SCOPE_DEFAULT_SNOOZE_DAYS = 30 # Consecutive failed sweep retries of a cloud-side revoke before the # grant is ESCALATED (one-shot page to cloud_admin + global_admin, then # the hourly sweep stops picking it up). 12 hourly retries ≈ half a day # — long enough to ride out transient cloud throttles / propagation # delays, short enough that a structurally-broken revoke (wrong param, # IAM lock, vanished principal shape the driver doesn't recognise) # pages a human the same day instead of after the 7-day aging # compliance incident. The May-2026 driver-bug storm burned 463 silent # attempts over ~3 weeks; this constant is the cap that makes that # class impossible. Admin "重试云侧回收" resets the counter. _MANUAL_REVOKE_ESCALATE_AFTER = 12 # ── Display-label helpers ────────────────────────────────────────── # Provider strings stored on ``cmdb_cloud_scopes.provider`` are short # canonical codes. Map them to a human label per language for # notification / alert UX. Falls back to a Title-cased version of the # raw code for unknown providers so admins still see something # meaningful (e.g. "Tencent" for ``tencent``) instead of a blank. _PROVIDER_LABELS: dict[str, tuple[str, str]] = { "aliyun": ("阿里云", "Aliyun"), "aws": ("AWS", "AWS"), "azure": ("Azure", "Azure"), "gcp": ("GCP", "GCP"), "tencent": ("腾讯云", "Tencent Cloud"), "huawei": ("华为云", "Huawei Cloud"), "volc": ("火山引擎", "Volcano Engine"), } def _format_cloud_provider_label(provider: str | None) -> tuple[str, str]: """Return ``(zh_label, en_label)`` for a cloud provider code. ``provider`` is the value stored on ``cmdb_cloud_scopes.provider`` (lowercase canonical, e.g. ``aliyun``). Empty / unknown codes fall back to a Title-cased rendering rather than a hard-coded "Unknown" so a fresh provider added in CMDB still shows usefully in notifications without a notifications-side code change. """ code = (provider or "").strip().lower() if not code: return ("未知云厂商", "Unknown cloud") if code in _PROVIDER_LABELS: return _PROVIDER_LABELS[code] # Unknown provider: title-case the raw code into both slots so # the zh and en notification surfaces don't crash on a brand-new # CMDB provider that hasn't been added to ``_PROVIDER_LABELS``. # Yes, this means a Chinese subject line will read "[云账单·Newcloud]" # rather than a localised name — that's the deliberate trade-off # vs failing closed. Add the new code to the map when you see it. fallback = code.title() return (fallback, fallback) # ── Config resolution helpers ────────────────────────────────────── async def _setting_json( session: AsyncSession, key: str, default: Any, ) -> Any: v = await get_platform_setting(key, default=default, session=session) if isinstance(v, str): try: return _json.loads(v) except (ValueError, TypeError): return default return v async def _setting_int( session: AsyncSession, key: str, default: int, ) -> int: v = await get_platform_setting(key, default=default, session=session) try: # setting value comes back JSON-decoded; a quoted number lands as str. return int(v if not isinstance(v, str) else v.strip('"')) except (ValueError, TypeError): return default async def _setting_float( session: AsyncSession, key: str, default: float, ) -> float: v = await get_platform_setting(key, default=default, session=session) try: return float(v if not isinstance(v, str) else v.strip('"')) except (ValueError, TypeError): return default def _extract_scope_email(raw: dict | None) -> str | None: """Pull the provider's per-account contact email out of a scope's ``raw`` blob for promotion to the ``cloud_account_email`` column. AWS Organizations reports it at ``raw["email"]``. Returns a stripped string, or ``None`` when the key is absent / blank / non-string (the single-account "org-not-in-use" AWS branch and every non-AWS driver carry no email). Deliberately does NOT lower-case: the local part of an email is technically case-sensitive and this is a display / contact value, not a join key. """ if not raw: return None value = raw.get("email") if not isinstance(value, str): return None value = value.strip() return value or None # ── Default alert rules for newly-synced scopes ──────────────────── _DEFAULT_ALERT_TEMPLATE: list[dict[str, Any]] = [ # Daily-spike threshold: a single day's cost crossing 25% of the # monthly budget. Catches one-off mistakes (a forgotten GPU # instance, a runaway batch job) without depending on the # rolling-7-day spike multiplier (which only fires once a baseline # exists, ie not on a brand-new scope). {"type": "daily", "threshold_pct": 25, "active": True}, # Monthly thresholds: classic 50/80/100% of the planned spend. # ``threshold_pct`` is interpreted at evaluation time against the # scope's then-current ``budget_usd_monthly`` so admin tweaks to # the budget don't require rewriting the rules. {"type": "monthly", "threshold_pct": 50, "active": True}, {"type": "monthly", "threshold_pct": 80, "active": True}, {"type": "monthly", "threshold_pct": 100, "active": True}, ] async def _build_default_alert_rules( session: AsyncSession, monthly_budget_usd: float, ) -> list[dict[str, Any]]: """Materialise the default alert templates into concrete rules keyed off ``monthly_budget_usd``. The templates carry ``threshold_pct``; this helper converts each into the absolute ``threshold_usd`` shape that ``evaluate_scope_alert_rules`` already consumes — keeps the rule evaluator's hot path simple (no percentage math at billing-sweep time) while still letting an admin retune the platform default template via ``platform_settings`` without touching every scope. Override the template via ``cloud.default_alert_rules`` (JSON array). Falls back to the built-in list above on missing/invalid. """ template = await _setting_json( session, "cloud.default_alert_rules", _DEFAULT_ALERT_TEMPLATE, ) if not isinstance(template, list) or not template: template = _DEFAULT_ALERT_TEMPLATE out: list[dict[str, Any]] = [] for entry in template: if not isinstance(entry, dict): continue rule_type = str(entry.get("type") or "").strip() if rule_type not in ("daily", "monthly"): continue # Either ``threshold_pct`` (preferred — survives budget edits) # or ``threshold_usd`` (absolute) is allowed in the template. if "threshold_pct" in entry: try: pct = float(entry["threshold_pct"]) except (TypeError, ValueError): continue threshold = round(monthly_budget_usd * pct / 100.0, 2) elif "threshold_usd" in entry: try: threshold = float(entry["threshold_usd"]) except (TypeError, ValueError): continue else: continue if threshold <= 0: continue out.append({ "type": rule_type, "threshold_usd": threshold, "threshold_pct": entry.get("threshold_pct"), "cc_person_ids": list(entry.get("cc_person_ids") or []), "active": bool(entry.get("active", True)), }) return out # ── Domain exceptions ─────────────────────────────────────────────── class GrantDispatchError(AppException): """Driver returned ok=False or raised — surface to the caller.""" def __init__(self, message: str) -> None: super().__init__( code="CLOUD_GRANT_DISPATCH_FAILED", message=message, status_code=502, ) class GrantPrincipalNotFoundError(AppException): """The driver deterministically reported the grant principal missing at the provider IdP (``CloudPrincipalNotFoundError`` — e.g. a deleted Google account, an email with no AWS Identity Center user). Distinct from :class:`GrantDispatchError` (transient driver failure, 502, retry may succeed): retrying with the same principal is guaranteed to fail until someone provisions / restores the account, so callers must NOT blind-retry — the backfill cron stamps the ``auto_owner_grant_unresolvable_at`` backoff marker, HTTP callers surface the 409 to the user. This implements the contract promised by the ``CloudPrincipalNotFoundError`` docstring in driver_base (previously aspirational — ``_dispatch_grant``'s blanket ``except Exception`` flattened it into a generic PENDING-forever failure; prod 2026-07: 31+ setIamPolicy retries at 10-min cadence on one deleted principal).""" def __init__(self, message: str) -> None: super().__init__( code="CLOUD_PRINCIPAL_NOT_FOUND", message=message, status_code=409, ) class ScopeUnownedError(AppException): """Scope has no owner — grants are blocked until IT assigns one.""" def __init__(self, scope_id: str) -> None: super().__init__( code="CLOUD_SCOPE_UNOWNED", message=( f"Scope {scope_id} has no owner — IT admin must assign an " "owner at /admin/cloud/scopes before members can be added." ), status_code=409, ) # ── Thin result types ────────────────────────────────────────────── @dataclass(slots=True) class LifecycleGrantOutcome: grant_id: str status: str driver_result: CloudGrantResult | None = None warning: str | None = None # ── Driver construction + credential loading ─────────────────────── async def _load_cloud_driver( session: AsyncSession, integration: Integration, ) -> CloudDriver: driver_cls = get_driver_class("cloud", integration.provider) if driver_cls is None: raise AppException( code="CLOUD_DRIVER_UNKNOWN", message=f"no driver registered for cloud/{integration.provider}", status_code=500, ) merged: dict[str, Any] = {} if integration.config_public: merged.update(integration.config_public) if integration.secrets_encrypted: try: merged.update( decrypt_fields( integration.secrets_encrypted, integration.secrets_nonce, integration_id=str(integration.id), ) ) except CredentialDecryptionError as exc: raise AppException( code="CLOUD_INTEGRATION_CREDENTIAL_INVALID", message=( f"integration {integration.slug or integration.id} has invalid " f"credentials: {exc}" ), status_code=500, ) from exc merged["_alias"] = integration.name driver = driver_cls(integration_id=str(integration.id), config=merged) return driver # type: ignore[return-value] # Driver ``CloseScopeResult.error_code`` values that mean "this close is # mid-flight and will continue on the next attempt", as opposed to "the # provider said no". They surface as 409 ``SCOPE_CLOSE_PRECHECK_RUNNING`` # rather than 502 so the UI reports progress. Every code listed here MUST # come from a driver path that changed nothing it cannot re-derive on the # next call — the whole point is that the admin simply clicks again. _RESUMABLE_CLOSE_ERROR_CODES = frozenset({ "deletion_precheck_running", "cloud_account_switch_pending", }) # Driver ``CloseScopeResult.error_code`` values that mean "the provider # refused because the scope still contains something Helix can delete". # These are the codes that used to become a red modal with a CLI command # in it — i.e. Helix handing the decommission back to the person who # asked for it. With a confirmed blocker list they instead start the # cleanup path (``_begin_blocker_cleanup`` → ``cmdb.decommission_ # cleanup_resume``). # # Only codes whose blockers a driver can BOTH enumerate # (``preflight_close_scope``) and delete (``clear_close_blockers``) # belong here. ``pending_charges`` / ``missing_owner`` / # ``billing_attached`` deliberately do not: those need a human with # authority Helix does not have, and pretending otherwise would leave a # decommission pending forever while nobody was told. _CLEANABLE_CLOSE_ERROR_CODES = frozenset({ "resources_existing", # Azure ResourcesExistingOnSubscription "liens_exist", # GCP project liens }) # ``decommission_cleanup_state`` vocabulary. Only ``cleaning`` is a # state the resume job keeps working on; the other three are terminal # until a human acts, and each has been notified exactly once. CLEANUP_STATE_CLEANING = "cleaning" CLEANUP_STATE_NEEDS_RECONFIRM = "needs_reconfirm" CLEANUP_STATE_BLOCKED = "blocked" CLEANUP_STATE_STALLED = "stalled" # 6 hours at one attempt every 5 minutes. Sized against the slow end of # a real Azure resource-group deletion (minutes for storage/network, # tens of minutes for an AKS cluster or a large managed disk set), with # room to spare — the budget exists to stop an ENDLESS pending state, # not to second-guess Azure's pace. Spending it escalates to the human # who clicked, once. MAX_CLEANUP_ATTEMPTS = 72 @dataclass(slots=True) class BlockerCleanupPlan: """Split live blockers against what the admin actually confirmed. The whole safety property of automatic cleanup lives in this function: ``deletable`` is the intersection of "still there" and "the admin was shown it and confirmed", so nothing that appeared after the confirmation dialog can be deleted by the retry loop. * ``deletable`` — keys to hand to ``clear_close_blockers``. * ``unseen`` — live blockers nobody confirmed. Presence of ANY of these stops the cleanup and asks for a fresh confirmation; it is the difference between "finish the job I approved" and "delete whatever you find in there". * ``unremovable`` — live, confirmed, but the driver says a human must clear it. Escalates once instead of retrying 72 times. """ deletable: list[str] unseen: list[CloseBlocker] unremovable: list[CloseBlocker] def plan_blocker_cleanup( blockers: list[CloseBlocker], confirmed_keys: list[str] | None, ) -> BlockerCleanupPlan: """Pure: decide what may be deleted, given a confirmed key list. Keys are compared case-insensitively — ARM ids come back from different endpoints with different casing on the ``resourceGroups``/``resourcegroups`` segment, and a case-sensitive comparison would classify a confirmed resource group as ``unseen`` and stall every Azure decommission on a spurious re-confirmation. """ confirmed = {str(k or "").strip().casefold() for k in (confirmed_keys or [])} confirmed.discard("") deletable: list[str] = [] unseen: list[CloseBlocker] = [] unremovable: list[CloseBlocker] = [] for blocker in blockers: key = str(blocker.key or "").strip() if not key: continue if key.casefold() not in confirmed: unseen.append(blocker) continue if not blocker.removable: unremovable.append(blocker) continue deletable.append(key) return BlockerCleanupPlan( deletable=deletable, unseen=unseen, unremovable=unremovable, ) def _clear_pending_decommission(scope: CmdbCloudScope) -> None: """Drop every in-flight-decommission column, together. They are one fact ("a confirmed decommission is mid-cleanup") spread over eight columns; clearing a subset would leave a half-state that the resume-job query (``decommission_pending_since IS NOT NULL``) and the UI pill read differently. """ scope.decommission_pending_since = None scope.decommission_pending_by = None scope.decommission_pending_reason = None scope.decommission_cleanup_keys = None scope.decommission_cleanup_state = None scope.decommission_cleanup_attempts = 0 scope.decommission_cleanup_error = None scope.decommission_cleanup_trace_id = None def describe_blocker(blocker: CloseBlocker) -> str: """One human line for a blocker — used in messages and notifications.""" bits = [blocker.name or blocker.key] if blocker.detail: bits.append(f"({blocker.detail})") if blocker.child_count: inner = ", ".join(blocker.children[:3]) more = ( f" and {blocker.child_count - len(blocker.children[:3])} more" if blocker.child_count > len(blocker.children[:3]) else "" ) bits.append( f"containing {inner}{more}" if inner else f"containing {blocker.child_count} resource(s)" ) return " ".join(bits) async def _load_driver_for_scope( session: AsyncSession, scope: CmdbCloudScope, ) -> tuple[Integration, CloudDriver]: integration = await session.get(Integration, uuid.UUID(str(scope.integration_id))) if integration is None: raise NotFoundError("Integration", str(scope.integration_id)) driver = await _load_cloud_driver(session, integration) return integration, driver def _require_verified_gcp_scope_boundary( integration: Integration | None, scope: CmdbCloudScope, ) -> None: """Fence every GCP cloud mutation to a strictly verified scope row. Older GCP discovery stamped the configured organization onto every credential-visible project without proving ancestry. Those rows can remain active until the operator completes the #2446 dry-run/migration sync. They must not be grant/revoke/decommission targets during that window. """ if str(getattr(integration, "provider", "") or "").strip().lower() != "gcp": return raw = scope.raw_data or {} config_public = getattr(integration, "config_public", None) or {} configured_org = str( config_public.get("organization_id") or "", ).strip() verified_org = str( raw.get("organization_id") if isinstance(raw, dict) else "", ).strip() is_verified = ( isinstance(raw, dict) and raw.get("tenant_boundary_verified") is True and bool(configured_org) and verified_org == configured_org ) if not is_verified: raise AppException( code="GCP_SCOPE_TENANT_BOUNDARY_UNVERIFIED", message=( "GCP scope tenant boundary is not verified for the configured " "organization; run the integration connection dry-run, complete " "the #2446 configuration/IAM migration, and sync scopes before " "attempting IAM or decommission mutations" ), status_code=409, ) async def _classify_scope_role( session: AsyncSession, scope: CmdbCloudScope, role_key: str, role_display_name: str | None, ) -> RoleGovernance: """Governance tier for a role on ``scope``. Live driver classification is the base (no secrets, sync-independent), with one twist: an admin tier override (set via the role-catalog UI, `governance_overridden_by_admin`) is honored so the gate AGREES with what the catalog/dropdown shows — otherwise an admin who locks a role down to `restricted` would see the UI block it but the gate would still allow it (and vice-versa). The one thing an override CANNOT loosen is `ungrantable`: that's a provider fact (the driver would reject the grant anyway), so it's a hard floor. Unknown provider → `normal` (fail-open on classification only). """ integration = await session.get( Integration, uuid.UUID(str(scope.integration_id)), ) provider = getattr(integration, "provider", None) cls = get_driver_class("cloud", provider) if provider else None live = ( cls.classify_role(role_key, role_display_name, scope.scope_type) if cls is not None else RoleGovernance(TIER_NORMAL, "unknown provider") ) if live.tier == TIER_UNGRANTABLE: return live # Honor admin tier override(s) for this (integration, scope-or-tenant- # wide, role_key). Multiple rows (scope-specific + tenant-wide) → take # the most severe so a lock-down on any matching row wins. override_tiers = (await session.execute( select(CmdbCloudRole.governance_tier).where( CmdbCloudRole.integration_id == str(scope.integration_id), CmdbCloudRole.role_key == role_key, CmdbCloudRole.governance_overridden_by_admin.is_(True), CmdbCloudRole.deleted_at.is_(None), or_( CmdbCloudRole.scope_id == str(scope.id), CmdbCloudRole.scope_id.is_(None), ), ) )).scalars().all() if override_tiers: return RoleGovernance(most_severe(*override_tiers), "admin override") return live async def _preflight_request_role_shapes( session: AsyncSession, scope: CmdbCloudScope, role_keys: list[str], ) -> None: """Reject un-bindable role shapes at request/add-member time. Calls ``driver.validate_request_role`` for each key so a stale catalog row (e.g. a RAM-shape ``System:ReadOnlyAccess`` on a CloudSSO-managed AliCloud tenant) is refused BEFORE the request enters the approval queue — instead of passing submit, routing through the owner, and only failing at grant time with a toast the approver (not the requester) sees. Raises ``AppException(422)`` on the first un-grantable key. Fails OPEN: any driver-load / credential / transport error is swallowed (the hook itself is contracted to fail open, but a driver that can't even be constructed must not block a legit request — the grant path stays the authoritative gate). The driver is always closed. """ integration = await session.get( Integration, uuid.UUID(str(scope.integration_id)), ) if integration is None: return try: driver = await _load_cloud_driver(session, integration) except Exception: # noqa: BLE001 — fail open on driver-load failure return scope_ref = CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ) try: for r_key in role_keys: try: problem = await driver.validate_request_role(scope_ref, r_key) except Exception: # noqa: BLE001 — hook must fail open problem = None if problem: raise AppException( code="CLOUD_ROLE_NOT_GRANTABLE", message=problem, status_code=422, ) finally: try: await driver.close() except Exception: # noqa: BLE001 pass async def _resolve_aws_bq_billing( session: AsyncSession, integration: Integration, ) -> dict[str, Any] | None: """Build the ``bq_billing`` kwarg the AWS driver expects when ``config_public.billing_source == 'bigquery'``. Threaded into ``list_billing_daily`` / ``list_cost_facts`` / ``list_billing_by_dimension`` by their respective callers. Returns None for non-AWS or AWS-on-CE integrations — the driver short- circuits past the BQ branch in those cases. Raises ``RuntimeError`` loudly on any half-configured BQ source: silent fall-through to CE would surface 60+ AccessDenied errors that look identical to the original prod failure this migration was supposed to fix. Reads the SA JSON from the GCP cloud integration referenced by ``billing_bq_credentials_integration_id``. That's a single source of truth (same SA the GCP CMDB sync uses) — rotation happens once on the GCP integration, not duplicated across AWS configs. """ if integration.provider != "aws": return None cfg = integration.config_public or {} billing_source = str( cfg.get("billing_source") or "cost_explorer", ).strip().lower() if billing_source != "bigquery": return None integration_label = integration.slug or str(integration.id) creds_int_id_raw = str( cfg.get("billing_bq_credentials_integration_id") or "", ).strip() if not creds_int_id_raw: raise RuntimeError( f"AWS integration {integration_label}: billing_source=bigquery " "but billing_bq_credentials_integration_id is unset — point it " "at the GCP integration whose SA has bigquery.dataViewer on " "the CUR dataset", ) dataset = str(cfg.get("billing_bq_dataset") or "").strip() if not dataset: raise RuntimeError( f"AWS integration {integration_label}: billing_source=bigquery " "but billing_bq_dataset is unset (expected " "``project.dataset``, e.g. " "``gcp-digital-bi.aws_billing_no_payer``)", ) # Layout switch — determines table-resolution / partition-pruning / # scope-filter strategy. ``per_account`` (CUR 1.0) keeps the legacy # one-table-per-account behaviour for prod's older # ``aws_nonprod_billing`` dataset; ``per_month`` (CUR 2.0 / Data # Exports) reads from one native-partitioned table per month with # all linked accounts merged (e.g. ``aws_billing_no_payer``). layout = str( cfg.get("billing_bq_table_layout") or "per_account", ).strip().lower() if layout not in ("per_account", "per_month"): raise RuntimeError( f"AWS integration {integration_label}: " f"billing_bq_table_layout {layout!r} is invalid — " "expected 'per_account' or 'per_month'", ) # Tag schema switch — CUR 1.0 flattens user-defined tags into # ``resource_tags_user_`` columns; CUR 2.0 nests them in # ``resource_tags.key_value`` ARRAY>. tag_schema = str( cfg.get("billing_bq_tag_schema") or "flat", ).strip().lower() if tag_schema not in ("flat", "nested"): raise RuntimeError( f"AWS integration {integration_label}: " f"billing_bq_tag_schema {tag_schema!r} is invalid — " "expected 'flat' or 'nested'", ) # Fact-grain switch — selects aggregation grain for cost_facts. # ``per_product`` (default) keeps cost_facts row volume bounded # to ~hundreds per scope per day; ``per_resource`` blows up to # ~thousands when CUR 2.0 ``line_item_resource_id`` is present # for every row. Default preserves the CUR 1.0 absent-column # behaviour Helix shipped originally. fact_grain = str( cfg.get("billing_bq_fact_grain") or "per_product", ).strip().lower() if fact_grain not in ("per_product", "per_resource"): raise RuntimeError( f"AWS integration {integration_label}: " f"billing_bq_fact_grain {fact_grain!r} is invalid — " "expected 'per_product' or 'per_resource'", ) # Pattern default depends on layout — the placeholder shape # differs (``{account_id}`` for per_account, ``{yyyymmdd}`` for # per_month). Picking a sensible default per layout keeps existing # ``aws-primary`` integrations (which never had to set the pattern # explicitly) working through the layout flip. pattern_default = ( "{account_id}_aws_billing_data" if layout == "per_account" else "aws_billing_{yyyymmdd}" ) pattern = str( cfg.get("billing_bq_table_pattern") or pattern_default, ).strip() try: gcp_int_id = uuid.UUID(creds_int_id_raw) except ValueError as exc: raise RuntimeError( f"AWS integration {integration_label}: " f"billing_bq_credentials_integration_id {creds_int_id_raw!r} " "is not a valid UUID", ) from exc gcp_int = await session.get(Integration, gcp_int_id) if ( gcp_int is None or gcp_int.provider != "gcp" or gcp_int.kind != "cloud" ): raise RuntimeError( f"AWS integration {integration_label}: " f"billing_bq_credentials_integration_id {creds_int_id_raw} " "does not point to a GCP cloud integration", ) if not gcp_int.enabled: raise RuntimeError( f"AWS integration {integration_label}: referenced GCP " f"integration {gcp_int.slug or gcp_int.id} is disabled", ) secrets: dict[str, Any] = {} if gcp_int.secrets_encrypted: try: secrets = decrypt_fields( gcp_int.secrets_encrypted, gcp_int.secrets_nonce, integration_id=str(gcp_int.id), ) except CredentialDecryptionError as exc: raise RuntimeError( f"AWS integration {integration_label}: failed to decrypt " f"GCP integration {gcp_int.slug or gcp_int.id}'s " f"credentials: {exc}", ) from exc sa_raw = secrets.get("service_account_json") if not sa_raw: raise RuntimeError( f"AWS integration {integration_label}: GCP integration " f"{gcp_int.slug or gcp_int.id} has no service_account_json " "secret to use for BQ billing", ) if isinstance(sa_raw, str): import json as _json try: sa_json = _json.loads(sa_raw) except ValueError as exc: raise RuntimeError( f"AWS integration {integration_label}: GCP integration " f"{gcp_int.slug or gcp_int.id}'s service_account_json is " f"not valid JSON: {exc}", ) from exc elif isinstance(sa_raw, dict): sa_json = sa_raw else: raise RuntimeError( f"AWS integration {integration_label}: GCP integration " f"{gcp_int.slug or gcp_int.id}'s service_account_json is of " f"unexpected type {type(sa_raw).__name__}", ) return { "credentials_json": sa_json, "dataset": dataset, "table_pattern": pattern, "table_layout": layout, "tag_schema": tag_schema, "fact_grain": fact_grain, } # Widest jitter applied to a discovered grant's TTL, in days. 14 is not # arbitrary: it matches the T-14 first-reminder stage, so spreading # expiry across this window spreads the REMINDER wave by the same # amount. Anything narrower still leaves a visible spike. DISCOVERY_TTL_JITTER_DAYS = 14 # Cap on how many unmapped cloud-side principal emails one sync run # records in its summary. These identify people who hold live access # that Helix has NO row for — invisible in every members table and # untouchable by every sweep — so naming them is the whole point; the # cap only stops a badly-mapped tenant from bloating the summary JSON. _UNRESOLVED_SAMPLE_CAP = 50 def _discovery_ttl_expires_at( *, now: datetime, ttl_days: int, scope_id: str, person_id: str, role_key: str, ) -> datetime: """``expires_at`` for a newly discovered grant, jittered so a freshly onboarded cloud account does not expire every principal at once. The problem this solves: ``_apply_discovered_grants`` used to write a flat ``now + ttl_days``, and every row inserted by one sync run shares the same ``now``. Onboarding an account therefore scheduled its entire membership to expire in the SAME SECOND — 164 grants on one instant for aws-account 385595570414 — which then produced a single-day reminder storm at T-14 (861 cards to 198 people on 2026-07-15) and a single-day revoke cliff at T+0. Self-service renewal re-anchors to "click time + 90d", so the cohort re-clumps and the storm reproduces every quarter forever. The jitter is: * **Deterministic** — derived from a BLAKE2b digest of ``(scope_id, person_id, role_key)``, never ``random`` and never the process-salted builtin ``hash()``. Re-running a sync, replaying a migration, or asserting in a test all produce the same instant, so the value is reproducible and reviewable. * **Backwards only** — it SUBTRACTS from ``now + ttl_days`` and never adds. Jittering forward would hand out more access than the configured TTL allows, which is a policy weakening dressed up as an ergonomics fix. Worst case a discovered grant gets ``ttl_days - 14``. * **Floored at half the TTL** so a short-TTL scope stays sane: a 10-day scope jitters within 0–5 days, not 0–14 (which would land in the past and be revoked by the very next sweep). """ span_days = max(0, min(DISCOVERY_TTL_JITTER_DAYS, ttl_days // 2)) base = now + timedelta(days=int(ttl_days)) if span_days <= 0: return base digest = hashlib.blake2b( f"{scope_id}|{person_id}|{role_key}".encode(), digest_size=8, ).digest() seed = int.from_bytes(digest, "big") # Two independent draws off one digest: whole days from the low half, # minutes-within-the-day from the high half. Without the minute term # every row in a cohort would still share the sync run's exact # clock time, so a 14-day spread would just become 14 spikes. offset_days = seed % span_days offset_minutes = (seed // span_days) % (24 * 60) return base - timedelta(days=offset_days, minutes=offset_minutes) async def load_perpetual_person_ids(session: AsyncSession) -> set[str]: """Person IDs whose cloud grants are PERPETUAL (``expires_at IS NULL``). Single source of truth for the perpetual-grant predicate, shared by every writer AND every sweeper. Membership is ``cloud_admin`` OR ``global_admin`` (including the legacy ``it_admin`` alias) — the same set ``_is_perpetual_grant_holder`` answers for one person and the same set the frontend's ``isPerpetualCloudGrantRoleSet`` renders as 「永久授权」. Why this exists (2026-07-29 incident, cn68 / 317766989874): "perpetual" used to be decided independently at each site, and one site did not decide it at all — * ``_dispatch_grant`` (approve / owner-add) checked the former single-person cloud-admin predicate → wrote ``expires_at = NULL``. Correct for the policy at that time. * ``_renew_grant_locked`` checked the same predicate → renewed to ``NULL``. Correct for the policy at that time. * ``_upsert_discovered_grants`` (grants_sync) checked NOTHING and stamped ``now + scope.default_grant_ttl_days`` on every discovered row — including a cloud admin's. Wrong. * ``sweep_expiring_grants`` had owner + prod exemptions but NO cloud-admin exemption, while its sibling ``sweep_inactive_grants`` did have one. Wrong, and the asymmetry is what made the bug invisible in review. Net effect: a cloud admin whose access was DISCOVERED cloud-side (rather than granted through Helix) got a 90-day TTL and was auto-revoked at T+0, while the UI displayed 「永久授权」 the whole time. 162 grants across 3 cloud admins were revoked between 2026-07-24 and 2026-07-29 before this was found; 150 more were queued up behind it. Every perpetual decision now routes through this helper (or ``_is_perpetual_grant_holder`` for the single-person case). Adding a role to the perpetual set is a one-line change here that both the writers and the sweepers pick up — the two can no longer drift. 2026-08-02 policy: ``global_admin`` is also perpetual. A global admin must stay reachable for platform/cloud incidents and must never receive a renewal nag or lose cloud access to the ordinary expiry sweep. """ return { str(pid) for pid in (await session.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthRole.name.in_({ CLOUD_ADMIN_ROLE, GLOBAL_ADMIN_ROLE, "it_admin", })) )).scalars().all() } def _is_prod_environment(environment: str | None) -> bool: """True when a scope ``environment`` value is production per ``PROD_ENVIRONMENTS``. Single source for the lifecycle sweeps' prod exclusion: production cloud access is NOT managed by Helix's expiry/inactivity automation (no reminder cards, no escalation, no auto-revoke/disable) — the R&D security team owns prod end-to-end. Deriving ``"prod"`` inline at any one call site risks the passes drifting apart when ``PROD_ENVIRONMENTS`` grows (e.g. adding ``"production"`` would have produced no-reminder + auto-revoke-at-T+0 while the revoke pass still hardcoded ``env.lower() == "prod"``). ``environment`` is declared NOT NULL with Python default ``"dev"`` but historic rows inserted before the column gained a server-side default may carry NULL — coerce so the check never trips on ``None.lower()`` and NULL stays on the managed (non-prod) path. """ return (environment or "dev").lower() in { e.lower() for e in PROD_ENVIRONMENTS } def _refuse_prod_self_service(scope: CmdbCloudScope) -> None: """Refuse a self-service grant flow on a production scope. Production cloud access is owned by the R&D security team (Kidd.li) outside Helix self-service. The block applies to every caller — admins use the security team's intake, not a Helix endpoint. Centralised so ``submit_access_request`` and ``owner_add_member`` raise an identical error (same code + message) and a future addition can't drift. The HTTP-facing message is intentionally short and English-only. The frontend translates ``CLOUD_SCOPE_PROD_BLOCKED`` into the bilingual ``cloudLifecycle.myGrants.prodBlockTitle`` user-facing copy (see ``MyCloudGrantsPage`` + the i18n bundles); embedding Chinese in the backend message would violate the project i18n rule and make the en-US locale render Chinese-on-failure. """ if scope.environment in PROD_ENVIRONMENTS: raise AppException( code="CLOUD_SCOPE_PROD_BLOCKED", message=( "Production cloud access is granted by the R&D " "security team (Kidd.li); not in Helix self-service." ), status_code=403, ) def _refuse_ungrantable_scope_type(scope: CmdbCloudScope) -> None: """Refuse a grant flow on a scope that is not a unit of cloud access. Only ``TENANT_LEVEL_SCOPE_TYPES`` are grantable. Sub-tenant rows (today only ``azure_resource_group``, one minted per approved AI API key request by the Azure OpenAI auto-grant flow) exist purely as a cost-binding / ownership anchor: their ``external_id`` is an RG name, and every cloud IAM API this platform drives is tenant-addressed (the Azure driver builds ``/subscriptions/{external_id}/...``). A grant on one could only fail at dispatch time with ARM ``400 InvalidSubscriptionId`` — AFTER the requester waited and the owner approved. These rows are ALSO active + owned + ``environment='dev'``, so before this guard they matched every predicate of the requestable picker and showed up in the employee "申请云账号权限" list as ``helix-aoai-… · AI API · REQ-…`` — a scope with an empty role dropdown (roles_sync can't enumerate roles for it either) and no reachable outcome. The picker now filters on scope_type; this is the service-layer second line of defence for a direct POST, and mirrors ``_refuse_prod_self_service`` so the two flat invariants are enforced identically at every entry point. """ if scope.scope_type not in TENANT_LEVEL_SCOPE_TYPES: raise AppException( code="CLOUD_SCOPE_NOT_GRANTABLE", message=( f"Scope type {scope.scope_type!r} is not a unit of " "cloud access (it exists for cost attribution); " "request access on its parent account instead." ), status_code=400, ) def scope_self_request_block_reason(scope: CmdbCloudScope) -> str | None: """Why ``POST /grants/request`` would refuse this scope — or ``None``. Presentation helper for the scope serializer: the detail pages render a 「申请权限」 button and need to know up-front whether self-service access is reachable on THIS scope, so a dead button (or a button that only fails after the user filled the whole form) never ships. Deliberately implemented by *invoking the real guards* rather than re-stating their predicates: ``_refuse_prod_self_service`` and ``_refuse_ungrantable_scope_type`` are the enforcement chokepoints :meth:`CloudLifecycleService.submit_access_request` runs, so a future edit to either rule moves this flag with it. Reason order mirrors the order those checks run in ``submit_access_request`` (status → prod → scope type → unowned) so the surfaced reason is the one the caller would actually have hit. Reasons: ``inactive`` | ``prod`` | ``not_grantable`` | ``unowned``. """ if scope.status != SCOPE_STATUS_ACTIVE: return "inactive" try: _refuse_prod_self_service(scope) _refuse_ungrantable_scope_type(scope) except AppException as exc: return { "CLOUD_SCOPE_PROD_BLOCKED": "prod", "CLOUD_SCOPE_NOT_GRANTABLE": "not_grantable", }.get(exc.code, "not_grantable") # Billing-account scopes are ownerless by design — their approval # authority is the cloud-admin team (see submit_access_request). if ( not scope.owner_person_id and scope.scope_type != SCOPE_TYPE_GCP_BILLING_ACCOUNT ): return "unowned" return None async def _audit_decommission_failure( *, trace_id: str, actor_user_id: uuid.UUID, scope_id: str, detail: dict[str, Any], ) -> None: """Write a ``cloud.scope.decommission/failed`` audit row in a fresh session that commits independently of the caller. Used by :meth:`CloudLifecycleService.decommission_scope` on the ``NotImplementedError`` and provider-error paths: the caller's transaction is about to be rolled back by the ``AppException`` raise, so an in-transaction audit write would also be rolled back — losing the forensic trail of a destructive admin action that failed on the cloud side. We open a fresh session here, write the audit row, and commit before the exception unwinds. Failures of this audit write are logged but do NOT mask the real exception. """ try: async with async_session_factory() as audit_session: audit = AuditService(audit_session) await audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.decommission", status="failed", resource_type="cmdb_cloud_scope", resource_id=scope_id, detail=detail, ) await audit_session.commit() except Exception as exc: # noqa: BLE001 — best-effort logging logger.warning( "Out-of-tx audit write for decommission failure on scope %s " "failed: %s", scope_id, exc, ) # ── Lifecycle service (per-request) ──────────────────────────────── class CloudLifecycleService: """Domain-facing API bundled to one AsyncSession.""" def __init__(self, db: AsyncSession) -> None: self.db = db self.audit = AuditService(db) self.notify = NotificationService(db) # ── Scope: owner / budget / close ─────────────────────── async def _disarm_owner_handover_trap( self, scope: Any, *, prev_owner: Any, now: datetime, ) -> int: """Re-stamp the OUTGOING owner's grants with a fresh TTL. The expiry sweep's owner exemption is evaluated against the scope's CURRENT owner, and an exempt grant's ``expires_at`` is never acted on — so it rots (140 production rows were months past their stamped expiry on 2026-07-29). The instant ownership moves, the outgoing owner's grants stop being exempt while already long overdue, and the next nightly sweep revokes them on the spot: no T-14, no T-7, no T-1, because every reminder window is in the past too. They lose access with zero warning, mid-handover, which is exactly when they still need it. Shared by BOTH owner-change paths (#2134). It lived inline in ``assign_scope_owner`` when it was added on 2026-07-29, and the external-API path ``apply_open_update`` — which an integrator can drive in BULK over ``PATCH /open/cloud/subscriptions/{key}`` — just wrote the column, so a reconciliation run cut every outgoing owner off the same night. Returns the number of grants re-stamped. """ if not prev_owner: return 0 # A grant can hold ``expires_at IS NULL`` for two unrelated reasons, # and the first cut of this helper skipped BOTH: # # * the holder is a perpetual-ROLE holder (``cloud_admin`` / # ``global_admin``) — perpetual because of who they are. Ownership # moving changes nothing; re-stamping a TTL here would newly # subject a cloud admin to expiry. Correctly skipped. # * the grant is an OWNER grant the auto-owner backfill wrote with # ``force_perpetual`` — perpetual because they were the owner. # The moment ownership moves that reason evaporates, and because # ``sweep_expiring_grants`` keys on ``expires_at IS NOT NULL``, # the row is never revisited by anything again: a permanent grant # for someone who is neither the owner nor a perpetual-role # holder. Production 2026-08-18: 7 such rows, all from the # 2026-06-02 backfill batch, every one of them on a scope whose # ownership had since transferred (the audit rows read # ``prev_owner_grants_restamped: 0``, and the old comment here # called that "normal"). # # So the exemption is decided by the ROLE, not by the column being # NULL — via the same ``load_perpetual_person_ids`` every writer and # sweeper shares. if str(prev_owner) in await load_perpetual_person_ids(self.db): return 0 ttl_days = int( scope.default_grant_ttl_days or await _setting_int( self.db, "cloud.grant_default_ttl_days", 90, ) ) restamp = await self.db.execute( update(CmdbCloudAccessGrant) .where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.person_id == str(prev_owner), CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, ) .values( expires_at=( now.replace(tzinfo=None) + timedelta(days=ttl_days) ), last_notified_stage=None, ) ) return int(restamp.rowcount or 0) async def assign_scope_owner( self, scope_id: str, new_owner_person_id: str, *, actor_user_id: uuid.UUID, trace_id: str, revoke_previous_owner_grant: bool = False, actor_role_names: list[str] | None = None, ) -> CmdbCloudScope: # SELECT ... FOR UPDATE so concurrent rebinds (e.g. owner + # admin both PATCHing at once) serialize on the row instead of # last-writer-wins-with-two-audit-rows. The router gate # (``_require_scope_owner_or_admin``) was evaluated against a # snapshot before this call, so without the lock a slow caller # could overwrite a faster caller's transfer using stale "I'm # the owner" authority. Pin the row for the rest of the txn. scope = (await self.db.execute( select(CmdbCloudScope) .where(CmdbCloudScope.id == uuid.UUID(scope_id)) .with_for_update() )).scalar_one_or_none() if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) # Closed scopes are decommissioned — the cloud account is # closed-or-closing on the provider side. Rebinding ownership # of a dead account adds an audit row for nothing real and # confuses the leaver-blocked sweep (which trusts ``status == # active`` as the definition of "live resource"). Reject so # closed-scope state stays terminal. if scope.status == SCOPE_STATUS_CLOSED: raise AppException( code="SCOPE_OWNER_REBIND_CLOSED_FORBIDDEN", message="Cannot reassign owner on a closed scope.", status_code=409, ) owner = await self.db.get(IdentityPerson, uuid.UUID(new_owner_person_id)) if owner is None or owner.deleted_at is not None: raise NotFoundError("IdentityPerson", new_owner_person_id) # Refuse to seat a leaver as scope owner. The IAM-cleanup # transfer path and the manual-provision path already enforce # this; mirror it here so the offboarding invariant ("no # active leaver owns active resources") holds across every # owner-rebind path. Without this, a global_admin can # accidentally hand a live cloud account to someone whose # accounts are about to be wiped on T+0. # Round-20 audit (A1F10): direct attribute access, NOT # ``getattr(..., default="active")``. A future column rename # silently fail-OPEN to "active" would treat leavers as active # and let an admin hand a live cloud account to them. The # column is non-nullable in the schema; if a future migration # removes it the import / first read fails loudly instead. if owner.employment_status in ("left", "leaver_pending"): raise AppException( code="SCOPE_OWNER_LEAVER_FORBIDDEN", message=( "Cannot assign scope ownership to a person whose " "employment_status is 'left' or 'leaver_pending'." ), status_code=409, ) prev_owner = scope.owner_person_id # Self-rebind is a no-op — the row already says scope.owner = # owner. Skip writing an audit row for what is effectively a # double-click; otherwise an attacker who somehow reaches this # path could spam the audit table with self-rebinds and bury a # real attack underneath. Returning the scope unchanged matches # the HTTP-layer expectation (200 + serialized scope). if prev_owner == str(owner.id): return scope scope.owner_person_id = str(owner.id) # Transfer implicitly confirms current ownership: the 180-day # confirmation clock restarts from now for the new owner so the # cron doesn't immediately re-prompt them. Pair with the # ``cloud.scope.assign_owner`` audit row below — the # ``last_owner_confirmed_at`` bump is intentional, not silent. now = datetime.now(UTC) scope.updated_at = now scope.last_owner_confirmed_at = now # Manual operator intervention — clear the auto-Owner-grant # backoff marker so the backfill cron re-attempts the new owner's # grant on its next tick instead of waiting out the staleness # window. (The unresolvable state is about the permission-set # name, not the owner identity, but an admin actively re-seating # ownership is a clear "re-evaluate this scope now" signal.) scope.auto_owner_grant_unresolvable_at = None # Disarm the ownership-handover trap. # # The expiry sweep's owner exemption is evaluated against the # scope's CURRENT owner, and an exempt grant's ``expires_at`` is # never acted on — so it rots (140 rows across production were # months past their stamped expiry on 2026-07-29). The instant # ownership moves, the outgoing owner's grants stop being exempt # while already long overdue, and the next nightly sweep revokes # them on the spot: no T-14, no T-7, no T-1, because every # reminder window is in the past too. The outgoing owner loses # access with zero warning, mid-handover, which is exactly when # they still need it. # # Re-stamp their grants with a fresh TTL here, at the one moment # the exemption changes hands, so they re-enter the normal cycle # with a full reminder runway. Done at the event rather than by # rolling every owner grant forward nightly — that alternative # invents a meaningful-looking expiry for rows that are exempt and # empties the sweep's ``owner_skipped`` counter. restamped = await self._disarm_owner_handover_trap( scope, prev_owner=prev_owner, now=now, ) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.assign_owner", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "previous_owner_person_id": prev_owner, "new_owner_person_id": str(owner.id), # Outgoing owner's grants given a fresh reminder runway # instead of being revoked with no warning on the next # sweep. 0 is normal (they may hold no grants, or only # perpetual ones). "prev_owner_grants_restamped": restamped, }, ) # Mark any outstanding semiannual_pending row as transferred so # the timeline reflects "owner chose transfer" and the cron's # at-most-one-outstanding-prompt invariant resets to a clean # state for the new owner. If there's no pending row (admin # initiated the rebind, or owner self-transferred without an # outstanding semiannual prompt), write a standalone # ``owner_transferred`` leaf row so the governance timeline # always records the transfer — the audit_log row alone is # invisible to the detail-page tab. from app.domains.cmdb.models import ( OWNER_CONFIRMATION_KIND_OWNER_TRANSFERRED, OWNER_CONFIRMATION_KIND_SEMIANNUAL_PENDING, OWNER_CONFIRMATION_STATUS_PENDING, OWNER_CONFIRMATION_STATUS_RESPONDED, CmdbCloudScopeOwnerConfirmation, ) pending_res = await self.db.execute( select(CmdbCloudScopeOwnerConfirmation) .where( CmdbCloudScopeOwnerConfirmation.scope_id == str(scope.id), CmdbCloudScopeOwnerConfirmation.kind == OWNER_CONFIRMATION_KIND_SEMIANNUAL_PENDING, CmdbCloudScopeOwnerConfirmation.status == OWNER_CONFIRMATION_STATUS_PENDING, ) .with_for_update() ) pending_row = pending_res.scalar_one_or_none() if pending_row is not None: from app.domains.cmdb.owner_confirmation_service import ( OwnerConfirmationService, ) # ``responder_person_id`` is the ACTOR who initiated the # transfer (self-owner or admin), NOT the new owner. The # detail-page UI distinguishes self vs admin transfers by # comparing this against the prev owner — stamping the # new owner here would make every admin-initiated rebind # render as self-transfer. AuthUser aliases IdentityPerson # so ``actor_user_id`` is a person UUID, safe to pass. # Fixed 2026-05-16 alongside the new owner_transferred leaf # path below, which already stamps actor correctly. await OwnerConfirmationService(self.db).mark_transferred( pending_row.id if isinstance(pending_row.id, uuid.UUID) else uuid.UUID(str(pending_row.id)), new_owner_person_id=str(owner.id), responder_person_id=str(actor_user_id), actor_user_id=actor_user_id, trace_id=trace_id, # The owner this transfer actually replaced — NOT the # pending card's addressee, which may be stale. previous_owner_person_id=prev_owner, ) elif prev_owner is not None: # No outstanding semiannual prompt and there WAS a prior # owner — record the transfer as a standalone leaf row. # ``actor_user_id`` IS the actor's person id (AuthUser # table aliases IdentityPerson), so it doubles as # ``responded_by_person_id``. The UI distinguishes # admin-initiated vs self-transfer by comparing # ``responded_by_person_id`` against ``owner_person_id`` # (prev owner): equal → self; different → admin / IAM # cleanup / other admin path. # # Skip the row when ``prev_owner is None`` (initial owner # assignment on a freshly-discovered scope) — there is no # "transfer from no one" event worth showing in the # governance timeline, and a NULL ``owner_person_id`` # would render as "transferred from —" which is # misleading. self.db.add( CmdbCloudScopeOwnerConfirmation( scope_id=str(scope.id), owner_person_id=prev_owner, kind=OWNER_CONFIRMATION_KIND_OWNER_TRANSFERRED, status=OWNER_CONFIRMATION_STATUS_RESPONDED, response_kind=OWNER_CONFIRMATION_KIND_OWNER_TRANSFERRED, response_at=now, responded_by_person_id=str(actor_user_id), transferred_to_person_id=str(owner.id), created_at=now, updated_at=now, ) ) # Optionally revoke the OUTGOING owner's owner-equivalent grant. # Symmetric with the new-owner auto-grant the router triggers after # commit: when the operator opts in via the transfer dialog, the # person who is no longer owner also loses the access they held BY # VIRTUE OF being owner. Scoped to owner-equivalent grants only — # their unrelated single-request / discovered member grants survive # (the dialog offers exactly this, per the product decision). Runs # inside this txn so the rebind + revoke commit atomically; the # ``prev_owner == owner`` self-rebind case already returned above, # so ``prev_owner`` here is always a real, different prior owner. if revoke_previous_owner_grant and prev_owner: await self._revoke_owner_equivalent_grants( scope=scope, person_id=prev_owner, actor_user_id=actor_user_id, trace_id=trace_id, actor_role_names=actor_role_names, ) # Notify the new owner. NotificationService is fire-and-forget # over the outbox so a transient Feishu API hiccup leaves a # retryable outbox row instead of failing the whole transfer. # Wrapped in try/except to keep the legal-side-effect (owner # transfer + audit row) hard-committed even if dispatch raises # — the outbox + audit log together cover discoverability. prev_owner_name: str | None = None if prev_owner: try: prev_owner_person = await self.db.get( IdentityPerson, uuid.UUID(prev_owner), ) if prev_owner_person is not None: prev_owner_name = ( prev_owner_person.display_name or prev_owner_person.work_email ) except (ValueError, TypeError): prev_owner_name = None try: from app.domains.cmdb.owner_confirmation_cards import ( build_new_owner_card_fields, ) from app.notifications.service import ( NotificationEnvelope, NotificationService, ) # Best-effort snapshot for the welcome card. ``next_confirm_due`` # is exactly 180 days from now (matches the cron's eligibility # threshold). The two helpers used here are protected names # on OwnerConfirmationService (intra-package use is fine); # the import is deferred since OwnerConfirmationService imports # back into this module's models. try: from app.domains.cmdb.owner_confirmation_service import ( OwnerConfirmationService as OwnerConfirmationSvc, ) snapshot_svc = OwnerConfirmationSvc(self.db) spend_30 = await snapshot_svc._sum_spend( scope.id, days_back=30, ) member_count = await snapshot_svc._count_members(scope.id) except Exception: spend_30 = Decimal("0") member_count = 0 fields = build_new_owner_card_fields( scope_display_name=scope.display_name or scope.external_id, scope_environment=scope.environment, previous_owner_name=prev_owner_name, spend_recent_30d=spend_30, member_count=member_count, next_confirm_due_at=now + timedelta(days=180), detail_url=f"/cloud/scopes/{scope.id}", lang=owner.preferred_language or "zh-CN", ) envelope = NotificationEnvelope( recipient=owner, related_resource_type="cmdb_cloud_scope", related_resource_id=scope.id, template_key="cloud.owner_transferred_to_you", **fields, ) # ``enqueue``, not ``dispatch`` (#2156). This runs while the # scope row is held ``FOR UPDATE`` — and, when the operator # opted into "同时清除前 Owner 的权限", after a series of # cloud IAM revokes on the same transaction. ``dispatch`` # adds a synchronous Feishu round-trip inside that window, # so a slow Identity Center or a slow Feishu stretches the # row lock and the nightly ``cmdb.scopes_sync`` upsert on the # same row waits it out into 1205. The durable outbox row is # what the caller actually needs; the # ``notification.outbox_retry`` worker delivers it moments # later, off the lock. Same move ``decommission_scope`` # already made for the identical hazard. await NotificationService(self.db).enqueue(envelope) except Exception: logger.exception( "new-owner notification dispatch failed scope=%s new_owner=%s", scope.id, owner.id, ) return scope async def _revoke_owner_equivalent_grants( self, *, scope: CmdbCloudScope, person_id: str, actor_user_id: uuid.UUID, trace_id: str, actor_role_names: list[str] | None, ) -> None: """Revoke ``person_id``'s **owner-equivalent** grants on ``scope``. Invoked from :meth:`assign_scope_owner` when the operator opts into "同时清除前 Owner 的权限" in the transfer dialog. "Owner-equivalent" means the grant that ownership itself provisions (``detail.auto_owner_grant``) OR a grant whose role matches the provider's default owner role — NOT the person's unrelated single-request / discovered member grants, which survive the transfer. This matches the product decision to scope the cleanup to owner-equivalent access only. Owner-role identity is resolved CHEAPLY (no driver call): static providers by ``role_key`` from :data:`~app.domains.cmdb.lifecycle_router._DEFAULT_OWNER_ROLE_BY_PROVIDER`; AWS by the configured permission-set NAME in platform_settings. This keeps the synchronous PATCH fast — an Identity Center round-trip on the transfer path would add seconds and a failure mode for no real gain. Production scopes are skipped: their IAM is governed by the prod write lock and is out of every discretionary Helix flow (the new-owner auto-grant is likewise blocked on prod). Keeping both sides symmetric avoids a Helix-says-revoked / cloud-still-granted split-brain on prod. Best-effort per grant: revoke passes ``suppress_driver_error=True`` so a provider hiccup can never roll back the owner transfer this cleanup rides on — the row flips revoked and the reconciliation pass re-attempts the cloud unbind via ``detail.requires_manual_cloud_revoke``. """ if scope.environment in PROD_ENVIRONMENTS: logger.info( "skip previous-owner grant revoke on prod scope=%s " "(prod IAM is out of discretionary Helix flows)", scope.id, ) return # Deferred import: the owner-role constants live in the router, # which imports this service module — a top-level import would be # circular. Same pattern the backfill cron uses. from app.domains.cmdb.lifecycle_router import ( _AWS_DEFAULT_OWNER_ROLE_NAME_FALLBACK, _AWS_DEFAULT_OWNER_ROLE_NAME_SETTING, _DEFAULT_OWNER_ROLE_BY_PROVIDER, ) integration = await self.db.get( Integration, uuid.UUID(str(scope.integration_id)), ) provider = str(integration.provider) if integration else "" static_owner_role_key: str | None = None aws_owner_name: str | None = None if provider == "aws": aws_owner_name = ( str( await get_platform_setting( _AWS_DEFAULT_OWNER_ROLE_NAME_SETTING, default=_AWS_DEFAULT_OWNER_ROLE_NAME_FALLBACK, session=self.db, ) or "", ).strip() or _AWS_DEFAULT_OWNER_ROLE_NAME_FALLBACK ) else: entry = _DEFAULT_OWNER_ROLE_BY_PROVIDER.get(provider) if entry and entry[0] != "__resolve_at_runtime__": static_owner_role_key = entry[0] rows = (await self.db.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.person_id == str(person_id), CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ) )).scalars().all() for grant in rows: detail = grant.detail or {} is_owner_equiv = ( detail.get("auto_owner_grant") is True or ( static_owner_role_key is not None and grant.role_key == static_owner_role_key ) or ( aws_owner_name is not None and (grant.role_display_name or "").strip() == aws_owner_name ) ) if not is_owner_equiv: continue try: await self.revoke_grant( str(grant.id), reason=GRANT_REVOKE_REASON_OWNER, actor_user_id=actor_user_id, trace_id=trace_id, # The owner transfer is the primary side-effect; a # driver failure must not 500 it. Row flips revoked, # reconciliation retries the cloud unbind. suppress_driver_error=True, actor_role_names=actor_role_names, ) except Exception: # noqa: BLE001 — cleanup never fails the transfer logger.exception( "revoke previous-owner grant %s failed (scope=%s " "person=%s)", grant.id, scope.id, person_id, ) async def update_scope_config( self, scope_id: str, *, budget_usd_monthly: Decimal | None = None, budget_measure: str | None = None, auto_suspend_after_days: int | None = None, default_grant_ttl_days: int | None = None, notes: str | None = None, status: str | None = None, environment: str | None = None, account_email_override: str | None = None, actor_user_id: uuid.UUID, actor_role_names: list[str] | None = None, trace_id: str, ) -> CmdbCloudScope: """PATCH the scope's config fields. The route gate is owner-OR-cloud-admin, and a scope owner can be a plain employee. ``budget_measure`` therefore carries a SECOND, narrower authority check: moving a scope's cost basis re-denominates every percentage alert threshold derived from the budget, which is exactly the mutation ``set_scope_alert_rules`` refuses to accept from an owner. See :func:`apply_scope_budget`. ``actor_role_names`` is the caller's roles when the caller knows them (the HTTP route does — ``user.role_names``, which honours ``X-View-Role`` narrowing). Omitted (MCP / internal / tests), the actor's STORED roles are read from ``auth_user_roles``; both ``cloud_admin`` and ``global_admin`` are stored roles, so the DB is authoritative for this gate. """ # ``FOR UPDATE`` (#2189-5). The closed-scope refusal a few lines # down is a check-then-write: read the status, then write other # columns. Without the lock a decommission committing inside # that window leaves this call writing to a row it already # validated as active — reviving a closed scope's config with no # error. The sibling write paths (``assign_scope_owner``, # ``_decommission_scope_locked``) all pin the row; these two did # not. scope = (await self.db.execute( select(CmdbCloudScope) .where(CmdbCloudScope.id == uuid.UUID(scope_id)) .with_for_update() )).scalar_one_or_none() if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) changes: dict[str, Any] = {} # Resolved ONLY when a basis write is actually requested — an ordinary # notes/TTL/budget-number PATCH must not pay for a roles query. can_change_measure = False if budget_measure is not None: from app.core.rbac import can_manage_cloud as _can_manage_cloud names = ( actor_role_names if actor_role_names is not None else await _stored_role_names(self.db, actor_user_id) ) can_change_measure = _can_manage_cloud(names) # Budget + its cost basis + the percentage rules derived from them # move together — see ``apply_scope_budget``. changes.update(apply_scope_budget( scope, budget_usd_monthly=budget_usd_monthly, budget_measure=budget_measure, can_change_measure=can_change_measure, )) if auto_suspend_after_days is not None: scope.auto_suspend_after_days = auto_suspend_after_days changes["auto_suspend_after_days"] = auto_suspend_after_days if default_grant_ttl_days is not None: scope.default_grant_ttl_days = default_grant_ttl_days changes["default_grant_ttl_days"] = default_grant_ttl_days if notes is not None: scope.notes = notes changes["notes"] = notes if account_email_override is not None: # "" clears the override (revert to the cloud value); any # other value sets it. Stamp who/when only when the stored # value actually changes so a no-op re-save doesn't rewrite # the modifier or emit a misleading audit entry. Empty string # is normalised to NULL so ``account_email_override or # cloud_account_email`` resolves the effective value cleanly. new_override = account_email_override.strip() or None if new_override != scope.account_email_override: scope.account_email_override = new_override scope.account_email_modified_by_person_id = str(actor_user_id) scope.account_email_modified_at = datetime.now(UTC) changes["account_email_override"] = new_override if status is not None: # Service-layer transition guard. The schema Literal blocks # an explicit PATCH to ``"closed"``, but it cannot enforce # transitions — if the caller bypasses the schema (MCP / # internal call / future router) we still must refuse to # reactivate a decommissioned scope. Decommission is # intentionally irreversible (the cloud-side resource is # gone) and the only legal write to ``status`` here is # ``active <-> suspended``. if status not in {"active", "suspended"}: raise AppException( code="INVALID_SCOPE_STATUS", message=( f"Scope status update rejected: status={status!r} " "is not allowed via update_scope_config — only " "'active' / 'suspended' transitions are permitted. " "Use decommission_scope to close a scope." ), status_code=400, ) if scope.status == SCOPE_STATUS_CLOSED and status != SCOPE_STATUS_CLOSED: raise AppException( code="SCOPE_ALREADY_DECOMMISSIONED", message=( "Scope is decommissioned; reactivation is not " "supported. The cloud-side resource has been " "closed (or is in pending-closure) and cannot be " "re-opened from Helix. Create a new scope if a " "fresh tenant is required." ), status_code=409, ) scope.status = status changes["status"] = status if environment is not None: scope.environment = environment changes["environment"] = environment # ``updated_at`` only ticks when something actually changed — # otherwise a no-op PATCH (e.g. the all-None call the # ``update_scope`` route makes when the request body had only # ``owner_person_id`` set, or a self-rebind that already short- # circuited in ``assign_scope_owner``) would silently bump the # row timestamp without an audit row to explain it. Drift # tooling that watches ``MAX(updated_at) > MAX(audit.created_at)`` # treats that gap as a bug; keeping the bump conditional on # real changes makes the two timestamps line up by construction. if changes: scope.updated_at = datetime.now(UTC) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.update", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail=changes, ) return scope # ── Scope: external-consumer metadata write (open API) ─────── async def apply_open_update( self, scope_id: str, *, owner_person_id: str | None = None, environment: str | None = None, display_name: str | None = None, notes: str | None = None, budget_usd_monthly: Decimal | None = None, budget_measure: str | None = None, account_email_override: str | None = None, client_id: uuid.UUID, client_slug: str, client_ip: str | None = None, trace_id: str, ) -> CmdbCloudScope: """Apply a metadata update from an External API client holding ``cloud.subscriptions:write`` (the ``/api/v1/open/cloud`` surface). Deliberately distinct from the human-facing ``update_scope_config`` / ``assign_scope_owner``: * the audit row is attributed to ``actor_type="service"`` with the API client named in ``detail`` and **no** ``actor_user_id`` (there is no human actor) — the same service-principal audit convention the directory pull API uses for its reads; * it **skips** the human owner-confirmation timeline side-effects (semiannual-clock reset, ``owner_transferred`` leaf rows, the new-owner notification). An external system writing the owner field is a data write, not a human confirming ownership; firing the confirmation machinery would pollute the governance timeline and mis-render transfers in the detail-page UI. Guards mirror the human paths so behaviour can't drift: a closed (decommissioned) scope rejects all writes, and an owner can only be set to an **active, non-leaver** person. These are Helix-metadata fields with no cloud-side effect, so the prod-write lock (which guards cloud *resource* mutations) does not apply. Per-field semantics: a field left at ``None`` (omitted by the caller) is unchanged. ``owner_person_id`` / ``budget_usd_monthly`` can only be set, never nulled out via this API (clearing them stays an admin action in the Helix console). ``display_name`` / ``notes`` accept an empty string to clear the field to NULL. Idempotent: a call that changes nothing writes no audit row and returns the row unchanged. """ # ``FOR UPDATE`` — see ``update_scope_config`` (#2189-5). The # closed-scope refusal below is a check-then-write, and this is # the EXTERNAL-API path, so a bulk reconciliation run makes the # window easy to hit. scope = (await self.db.execute( select(CmdbCloudScope) .where(CmdbCloudScope.id == uuid.UUID(scope_id)) .with_for_update() )).scalar_one_or_none() if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) if scope.status == SCOPE_STATUS_CLOSED: raise AppException( code="SCOPE_CLOSED_READONLY", message=( "Cannot modify a decommissioned (closed) scope — the " "cloud-side resource has been closed." ), status_code=409, ) changes: dict[str, Any] = {} if owner_person_id is not None: owner = await self.db.get(IdentityPerson, uuid.UUID(owner_person_id)) if owner is None or owner.deleted_at is not None: raise NotFoundError("IdentityPerson", owner_person_id) # Mirror assign_scope_owner: never seat a leaver as owner of # a live cloud account (offboarding invariant). if owner.employment_status in ("left", "leaver_pending"): raise AppException( code="SCOPE_OWNER_LEAVER_FORBIDDEN", message=( "Cannot assign scope ownership to a person whose " "employment_status is 'left' or 'leaver_pending'." ), status_code=409, ) if scope.owner_person_id != str(owner.id): prev_owner = scope.owner_person_id changes["owner_person_id"] = { "from": prev_owner, "to": str(owner.id), } scope.owner_person_id = str(owner.id) # Same two follow-ups ``assign_scope_owner`` does — this # path used to write the column and nothing else (#2134). # An integrator reconciling ownership in bulk over # ``cloud.subscriptions:write`` therefore cut every # outgoing owner's exempt-but-long-expired grants that # same night, with no T-14/T-7/T-1 warning. changes["owner_grants_restamped"] = ( await self._disarm_owner_handover_trap( scope, prev_owner=prev_owner, now=datetime.now(UTC), ) ) # An active re-seat is a "re-evaluate this scope now" # signal for the auto-owner-grant backfill too. scope.auto_owner_grant_unresolvable_at = None if environment is not None and environment != scope.environment: changes["environment"] = { "from": scope.environment, "to": environment, } scope.environment = environment if display_name is not None: new_name = display_name or None if new_name != scope.display_name: changes["display_name"] = { "from": scope.display_name, "to": new_name, } scope.display_name = new_name if notes is not None: new_notes = notes or None if new_notes != scope.notes: changes["notes"] = {"from": scope.notes, "to": new_notes} scope.notes = new_notes if account_email_override is not None: # "" clears the override; any other value sets it. Only the # override column is writable — ``cloud_account_email`` stays # provider-owned. There is no human person behind an External # API write, so the modifier person id is left NULL (the audit # row below, attributed to the API client, IS the provenance); # the timestamp still records WHEN it changed. new_override = account_email_override.strip() or None if new_override != scope.account_email_override: changes["account_email_override"] = { "from": scope.account_email_override, "to": new_override, } scope.account_email_override = new_override scope.account_email_modified_by_person_id = None scope.account_email_modified_at = datetime.now(UTC) # Budget + its cost basis + the percentage rules derived from them # move together — see ``apply_scope_budget``. An external client # writing a number onto a list-denominated budget without naming the # basis gets 400 ``CLOUD_BUDGET_MEASURE_REQUIRED``, same as a human. # # ``can_change_measure=True``: the authority this path proves is the # ``cloud.subscriptions:write`` consumer scope, which IT issues to an # integrating SYSTEM — it is not a scope owner and holds no personal # relationship to the scope, so it sits on the admin side of the # owner/admin line ``update_scope_config`` draws. It already writes # owner / environment / budget on any scope. changes.update(apply_scope_budget( scope, budget_usd_monthly=budget_usd_monthly, budget_measure=budget_measure, can_change_measure=True, )) if changes: scope.updated_at = datetime.now(UTC) await self.audit.log( trace_id=trace_id, actor_type="service", actor_user_id=None, action="cloud.scope.open_update", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "client_slug": client_slug, "client_id": str(client_id), "changes": changes, }, ip_address=client_ip, ) return scope # ── Scope: idle-sweep snooze (owner self-service) ──────── async def snooze_idle_scope( self, scope_id: str, *, days: int = IDLE_SCOPE_DEFAULT_SNOOZE_DAYS, actor_user_id: uuid.UUID, actor_role_names: list[str], trace_id: str, ) -> CmdbCloudScope: """Suppress the weekly idle-sweep notification for this scope. Owner-driven escape hatch for the ``cmdb.scope_idle_sweep`` cron: sets ``idle_snoozed_until = now + days`` so the sweep skips the scope until the timestamp passes. Distinct from ``last_idle_notified_at`` (which is the cron's own auto-backoff after a send) — the snooze is an explicit human "I know it's idle, I'm keeping it" signal. RBAC: the current scope owner OR a caller with ``can_manage_cloud`` (cloud_admin / it_admin). An admin acting on behalf of the owner is fine; a non-owner non-admin gets a 403. Idempotent w.r.t. the schedule: clicking 「忽略 30 天」 a second time inside the existing snooze window just pushes the deadline further into the future. Audit records every set with the previous + new ``snoozed_until`` so the timeline is reconstructible. ``days`` is bounded 1..365 to refuse pathological inputs (a 10- year snooze isn't an audit-friendly form of "decommission"). """ if not isinstance(days, int) or days < 1 or days > 365: raise AppException( code="IDLE_SNOOZE_DAYS_OUT_OF_RANGE", message="days must be between 1 and 365", status_code=400, ) scope = (await self.db.execute( select(CmdbCloudScope) .where( CmdbCloudScope.id == uuid.UUID(scope_id), CmdbCloudScope.deleted_at.is_(None), ) .with_for_update() )).scalar_one_or_none() if scope is None: raise NotFoundError("CmdbCloudScope", scope_id) # Closed scopes never appear in the sweep anyway (filter is # ``status == active``) — refusing here keeps the audit trail # honest about why a snooze was set, instead of recording # snoozes against already-decommissioned rows. if scope.status != SCOPE_STATUS_ACTIVE: raise AppException( code="IDLE_SNOOZE_SCOPE_NOT_ACTIVE", message=( "Snooze is only meaningful for active scopes; this " f"scope is {scope.status!r}." ), status_code=409, ) from app.core.rbac import can_manage_cloud as _can_manage_cloud # Two paths to authority: # 1. caller IS the current scope owner (owner_person_id match), or # 2. caller is a cloud/IT admin (covers cloud_admin + it_admin # via ``can_manage_cloud``). # Unowned scopes (``owner_person_id IS NULL``) can ONLY take # path 2 — there's no owner to match. The error message below # still reads "owner or admin"; for an unowned scope that # literally means "an admin", which is the only correct actor. is_owner = ( scope.owner_person_id is not None and scope.owner_person_id == str(actor_user_id) ) if not is_owner and not _can_manage_cloud(actor_role_names or []): raise ForbiddenError( "Only the scope owner or a cloud/IT admin can snooze " "the idle-sweep reminder for this scope." ) now = datetime.now(UTC) previous = scope.idle_snoozed_until scope.idle_snoozed_until = now + timedelta(days=days) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.idle_snooze", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "days": days, "previous_snoozed_until": ( previous.isoformat() if previous else None ), "new_snoozed_until": scope.idle_snoozed_until.isoformat(), "is_owner": is_owner, }, ) return scope # ── Scope: manual decommission (irreversible) ──────────── async def preflight_decommission_scope( self, scope_id: str, ) -> tuple[CmdbCloudScope, Integration, ClosePreflight]: """Read-only: what will a decommission of this scope destroy? Runs when the admin opens the 下线 dialog. Two jobs: 1. **Consent.** "1 个资源组，内含存储账号 billingexport123， 会被一并删除" is information needed to agree to a decommission at all. Before this existed the dialog said only "不可逆", and the resources inside surfaced as a 502 afterwards. 2. **Correcting the premise.** The production case that motivated this (it-team-us, 2026-09-07) was decommissioned with the reason 从未使用过 — true of its spend ($0.00) and false of its contents. A $0 subscription is not an empty one; preflight is where that gets said, while the decision is still open. Never raises on a provider problem: an unreachable provider comes back as ``checked=False`` and the dialog says so. The decommission itself is not gated on this read — a preflight that fails must not be able to block a takedown. """ scope = await self.db.scalar( select(CmdbCloudScope).where( CmdbCloudScope.id == uuid.UUID(scope_id), CmdbCloudScope.deleted_at.is_(None), ) ) if scope is None: raise NotFoundError("CmdbCloudScope", scope_id) integration, driver = await _load_driver_for_scope(self.db, scope) scope_ref = CloudScopeRef( scope_type=(scope.scope_type or "").strip(), external_id=(scope.external_id or "").strip(), display_name=scope.display_name, ) try: preflight = await driver.preflight_close_scope(scope_ref) except NotImplementedError: preflight = ClosePreflight(supported=False) except Exception as exc: # noqa: BLE001 # A driver bug or an unmapped SDK exception must degrade to # "we could not look", never to "nothing to clean" — the # latter would let the dialog promise an empty subscription. logger.warning( "decommission preflight failed for scope %s: %s", scope_id, format_exception_for_log(exc), ) preflight = ClosePreflight( supported=True, checked=False, error_code="preflight_failed", error_message=str(exc)[:300], ) return scope, integration, preflight async def decommission_scope( self, scope_id: str, *, reason: str, actor_user_id: uuid.UUID, trace_id: str, cleanup_blockers: bool = False, confirmed_blocker_keys: list[str] | None = None, ) -> tuple[CmdbCloudScope, int, CloseScopeResult]: """Admin marks a scope as closed. The flow is **cloud-first, Helix-second**: we ask the driver to actually close the account on the provider (AWS Organizations ``close_account``, Azure subscription cancel, GCP ``projects.delete``, Aliyun Resource Directory ``UpdateAccount`` → ``CheckAccountDelete`` → ``DeleteAccount``, IAM-mode evacuation of every Helix-tracked IAM user). Only when the provider acknowledges the closure do we flip Helix's row to ``closed`` and disable outstanding grants. Failures are loud: * Driver returned ``ok=False`` → raise ``AppException`` with the provider's error message; Helix row stays unchanged. * Driver raised ``NotImplementedError`` → 501 to the admin so the action is never silently a no-op. Idempotent on already-closed scopes: the driver returns ``already_closed=True`` (e.g. AWS reports ``SUSPENDED``) and we re-sync ``decommission_provider_state`` without overwriting the original ``decommissioned_at`` / ``_by`` / ``_reason`` (which record the first close's actor + timestamp + reason — the authoritative audit of record). Failure-mode reasoning for production: if the cloud-side close succeeds but the Helix transaction commit (in the router) fails — DB connection drop, deadlock, conflict — the cloud account is already in ``PENDING_CLOSURE`` while Helix's row still says ``active``. The next admin retry hits ``describe_account`` first, sees ``SUSPENDED``, returns ``already_closed=True``, and re-converges Helix on the next successful commit. The system is **eventually consistent**, not corrupt; the trade-off is the first attempt's audit log was rolled back with the failed transaction. Acceptable for an admin-triggered, irreversible, rare action — escalate to an out-of-tx audit write if this becomes a real incident. Returns ``(scope, disabled_grant_count, close_result)``. """ reason = (reason or "").strip() if not reason: raise AppException( code="DECOMMISSION_REASON_REQUIRED", message="reason must not be empty", status_code=400, ) # Concurrency: serialize decommission clicks via a Redis advisory # lock keyed by scope_id, NOT by holding a MySQL row lock across # the cloud-side close call. AWS Organizations.CloseAccount can # take 30+s; an InnoDB FOR UPDATE held that long would block # every concurrent cmdb-sync upsert touching this row and trip # ``innodb_lock_wait_timeout`` (50s default). The Redis lock # gives us the same single-flight semantics without touching DB # locks at all. Cloud APIs are idempotent, so even on the rare # case of a Redis failure the second click would observe # ``already_closed=True`` and re-converge. # NB: ``LockBusy`` is raised inside ``__aenter__`` of the # ``@asynccontextmanager`` — i.e. it surfaces from the # ``async with`` line, NOT from the bare call that constructs # the context-manager object. Wrap the ``async with`` itself. try: async with distributed_lock( f"decommission_scope:{scope_id}", ttl=600, ): return await self._decommission_scope_locked( scope_id, reason=reason, actor_user_id=actor_user_id, trace_id=trace_id, cleanup_blockers=cleanup_blockers, confirmed_blocker_keys=confirmed_blocker_keys, ) except LockBusy as exc: raise ConflictError( f"Scope {scope_id} is already being decommissioned" ) from exc async def _decommission_scope_locked( self, scope_id: str, *, reason: str, actor_user_id: uuid.UUID, trace_id: str, cleanup_blockers: bool = False, confirmed_blocker_keys: list[str] | None = None, ) -> tuple[CmdbCloudScope, int, CloseScopeResult]: scope = await self.db.scalar( select(CmdbCloudScope).where( CmdbCloudScope.id == uuid.UUID(scope_id), CmdbCloudScope.deleted_at.is_(None), ) ) if scope is None: raise NotFoundError("CmdbCloudScope", scope_id) previous_status = scope.status # Resolve the driver + collect Helix-tracked IAM principals on # this scope. The driver evacuates them for IAM-mode scopes; # multi-account drivers (AWS Organizations / Azure / GCP / # Aliyun RD) ignore the list and call the account-close API. integration, driver = await _load_driver_for_scope(self.db, scope) iam_account_ids = await self._iam_account_ids_for_scope(scope) # Normalize whitespace + casing on the dispatch keys before # handing them to the driver. ``external_id`` is sent verbatim # to AWS Organizations / Azure ARM / GCP / Aliyun RD APIs — # those reject ``" 111111111111 "`` with opaque # ``ValidationException`` / ``InvalidParameter`` codes that # surface to the admin as "driver bug" instead of "data # hygiene". Stripping here covers any historical row that # picked up trailing whitespace from manual SQL fix-ups or # legacy imports. ``scope_type`` matters for the per-driver # ``==`` dispatch (see e.g. aws_cloud.py); a trailing space # would silently go to NotImplementedError + 501. scope_ref = CloudScopeRef( scope_type=(scope.scope_type or "").strip(), external_id=(scope.external_id or "").strip(), display_name=scope.display_name, ) try: _require_verified_gcp_scope_boundary(integration, scope) close_result = await driver.close_scope_on_provider( scope_ref, iam_account_ids=iam_account_ids, ) except NotImplementedError as exc: # Out-of-tx audit write: the main transaction is about to # be rolled back by the AppException raise below, so an # in-tx audit.log() would also be rolled back — losing the # forensic trail of a destructive admin action. Use a # fresh session/commit so the failure record survives. await _audit_decommission_failure( trace_id=trace_id, actor_user_id=actor_user_id, scope_id=str(scope.id), detail={ "reason": reason, "error": "not_implemented", "error_message": str(exc), "external_id": scope.external_id, "integration_id": str(integration.id), "provider": integration.provider, }, ) raise AppException( code="SCOPE_CLOSE_NOT_SUPPORTED", message=( f"The {integration.provider!r} driver cannot close " f"scope_type={scope.scope_type!r} on the provider. " "Open an issue — every scope_type must support real " "cloud-side closure." ), status_code=501, ) from exc if ( not close_result.ok and cleanup_blockers and close_result.error_code in _CLEANABLE_CLOSE_ERROR_CODES ): # The provider refused because the scope still holds things # it wants gone first — and the admin already confirmed a # list of exactly those things. Take the work rather than # handing it back: delete what was confirmed, record the # decommission as in-flight, and let # ``cmdb.decommission_cleanup_resume`` carry it to the end. # # This branch returns NORMALLY (no raise) even though # ``close_result.ok`` is False, because the pending-state # writes it makes must survive: an ``AppException`` here # would roll back the very columns the resume job reads. # The caller distinguishes it by # ``error_code == 'cleanup_in_progress'``. accepted = await self._begin_blocker_cleanup( scope, integration, driver, scope_ref, reason=reason, actor_user_id=actor_user_id, trace_id=trace_id, confirmed_keys=confirmed_blocker_keys, close_result=close_result, ) if accepted is not None: return scope, 0, accepted if not close_result.ok: # Out-of-tx audit write — see the NotImplementedError branch # above for why. Provider failures (AWS ConstraintViolation, # Azure 403, etc.) are exactly the cases an admin needs a # forensic trail of, so losing the audit row to a rollback # is the worst possible outcome. await _audit_decommission_failure( trace_id=trace_id, actor_user_id=actor_user_id, scope_id=str(scope.id), detail={ "reason": reason, "error_code": close_result.error_code, "error_message": close_result.error_message, "provider_state": close_result.provider_state, "revoked_users": close_result.revoked_users, "external_id": scope.external_id, "integration_id": str(integration.id), "provider": integration.provider, "close_detail": close_result.detail, }, ) # Some driver outcomes are PROGRESS, not provider rejections: # Aliyun's member-deletion pre-check runs for 3–5 minutes and # its member type switch settles asynchronously, and the # driver is resumable across both. Give them their own code # and a 409 so the UI can say "still running, click again # shortly" instead of painting a red failure — while the # failure audit row above still records the partial work # (notably the CloudAccount→ResourceAccount switch, which # disables the member's root user). if close_result.error_code in _RESUMABLE_CLOSE_ERROR_CODES: raise AppException( code="SCOPE_CLOSE_PRECHECK_RUNNING", message=( close_result.error_message or "Provider pre-check still running" ), status_code=409, ) raise AppException( code="SCOPE_CLOSE_PROVIDER_ERROR", message=( close_result.error_message or "Cloud provider rejected the scope closure" ), status_code=502, ) # Cloud provider accepted (or scope was already closed). Mirror # the decision into Helix. now = datetime.now(UTC) disabled_grant_count = 0 # ``FOR UPDATE`` is load-bearing here, not defensive noise # (#2157). This SELECT runs AFTER the provider call — AWS # ``CloseAccount`` alone can take 30s+ — but a plain read under # REPEATABLE READ is served from the snapshot taken at the start # of this transaction, i.e. from BEFORE the cloud call. A grant # approved by another reviewer during that window is invisible # to it and therefore never cascaded, leaving an ``active`` # grant attached to a closed scope — permanently, because every # drift sweep excludes closed scopes. (The decommission Redis # lock only serialises decommissions; it does not block # approvals.) A locking read bypasses the snapshot and returns # the latest committed rows, which is exactly what the cascade # needs. Same shape as the 2026-05-28 UCloud zombie grants, # where only the sweep-close path was fixed. grant_rows = (await self.db.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ).with_for_update() )).scalars().all() for grant in grant_rows: grant.status = GRANT_STATUS_DISABLED # Stamp the revoke trail so compliance audits / the "已撤销" # tab can distinguish scope-decommission cascade from other # close paths (sweep-close uses the same reason; the audit # rows attribute them apart). Also clear # ``last_notified_stage`` so a future inactivity sweep # cannot re-fire a stale notification on this row. grant.revoke_reason = GRANT_REVOKE_REASON_SCOPE_CLOSED grant.revoked_at = now grant.last_notified_stage = None grant.updated_at = now disabled_grant_count += 1 # If the driver evacuated IAM users, mark them revoked too so # the IAM-users page reflects reality. ``cmdb_iam_users`` has no # scope_id — it is keyed # ``(integration_id, user_name, sub_account_id)`` — so the # sub-account predicate is what scopes this UPDATE to THIS # scope's users. Matching on integration + user_name alone # marked a same-named user belonging to a DIFFERENT sub-account # under the same integration as revoked (#2158): Helix would # record "revoked" for an identity that is still alive in the # cloud, which is the one direction an audit trail must never # get wrong. The SELECT that built ``iam_account_ids`` already # carries exactly this predicate, and its comment says it is # there to prevent that drift — the UPDATE just never mirrored # it. (Under today's 1:1 invariant ``sub_account_id`` is always # ``""``, so this cannot fire yet; a half-built defence is worth # one predicate to finish.) if close_result.revoked_users > 0 and iam_account_ids: await self.db.execute( update(CmdbIamUser) .where( CmdbIamUser.integration_id == str(scope.integration_id), CmdbIamUser.user_name.in_(iam_account_ids), or_( CmdbIamUser.sub_account_id == scope.external_id, CmdbIamUser.sub_account_id == "", ), CmdbIamUser.revoked_at.is_(None), CmdbIamUser.deleted_at.is_(None), ) .values( revoked_at=now, revoke_reason="scope_decommissioned", ), ) scope.status = SCOPE_STATUS_CLOSED scope.decommission_provider_state = close_result.provider_state scope.updated_at = now # The in-flight-decommission bookkeeping has served its purpose # the moment the provider accepts: the pending columns exist to # answer "what is the resume job still working on", and this # scope is no longer one of them. Clearing them here — rather # than only in the resume job — also covers the paths that # bypass cleanup entirely (an admin who deleted the resources # by hand and clicked again, or a provider that accepted on the # first try after a partial cleanup). _clear_pending_decommission(scope) # Preserve the original decommission audit trail. When the scope # was already closed (re-running the action — admin retried for # idempotency / cloud-state sync), keep the first close's # actor + timestamp + reason. Only stamp these on the # transition. The provider_state above always refreshes so a # later AWS PENDING_CLOSURE → SUSPENDED state change is captured. if previous_status != SCOPE_STATUS_CLOSED: scope.decommissioned_at = now scope.decommissioned_by = str(actor_user_id) scope.decommission_reason = reason # Distinguish three distinct admin actions in the audit log: # * ``cloud.scope.decommission`` — admin clicked decommission # AND the driver actually cancelled at the provider. # * ``cloud.scope.decommission_observed_closed`` — admin # clicked decommission but the provider reported the scope # was already closed at the start of the call (e.g. Azure # ``SubscriptionNotInRegisteredState`` after an out-of-band # portal cancel; AWS ``SUSPENDED`` from a prior org # close-account). The admin OBSERVED the closure and # Helix-side bookkeeping converged — they didn't actually # perform the cancel. Calling this ``decommission`` would # misattribute the action to a passer-by. (Round-52 logic # review caught the misattribution.) # * ``cloud.scope.decommission_idempotent_resync`` — admin # retried decommission on a Helix scope that was already # in ``status='closed'``. Pure resync, no transition. if previous_status == SCOPE_STATUS_CLOSED: action_name = "cloud.scope.decommission_idempotent_resync" elif close_result.already_closed: action_name = "cloud.scope.decommission_observed_closed" else: action_name = "cloud.scope.decommission" await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action=action_name, status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "previous_status": previous_status, "reason": reason, "disabled_grants": disabled_grant_count, "external_id": scope.external_id, "display_name": scope.display_name, "integration_id": str(scope.integration_id), "provider": integration.provider, "provider_state": close_result.provider_state, "already_closed": close_result.already_closed, "revoked_users": close_result.revoked_users, "close_detail": close_result.detail, }, ) return scope, disabled_grant_count, close_result async def _begin_blocker_cleanup( self, scope: CmdbCloudScope, integration: Integration, driver: CloudDriver, scope_ref: CloudScopeRef, *, reason: str, actor_user_id: uuid.UUID, trace_id: str, confirmed_keys: list[str] | None, close_result: CloseScopeResult, ) -> CloseScopeResult | None: """Delete the confirmed blockers and put the close in flight. Returns a ``CloseScopeResult`` with ``error_code='cleanup_in_progress'`` when the cleanup was accepted (the caller returns it as a 200 with a progress payload), or ``None`` to fall through to the normal provider-error path. Raises — rather than pending — for the two cases where the admin is still at the keyboard and a decision is theirs: * **The blocker set changed.** Something in the scope was not on the list they confirmed. Deleting it anyway is exactly the behaviour automatic cleanup must never have, so this is a 409 and the dialog re-reads the preflight. (Note this is not "reporting work back": the admin has not yet spent their confirmation on the new list, so there is no approved job to finish.) * **Helix cannot delete it.** A management lock the service principal has no permission on, or a blocker the driver marks non-removable. Pending would mean going quiet on work that will never progress. Nothing has been written to the transaction before those raises, so the rollback they trigger costs nothing. """ try: preflight = await driver.preflight_close_scope(scope_ref) except NotImplementedError: return None except Exception as exc: # noqa: BLE001 logger.warning( "cleanup preflight failed for scope %s: %s", scope.id, format_exception_for_log(exc), ) return None if not (preflight.supported and preflight.cleanup_supported): return None if not preflight.blockers: # The provider says "resources exist" and the enumeration # says otherwise. Do NOT start a cleanup on that: either the # read failed (``checked=False``) or the two APIs disagree, # and both are cases where the honest move is to surface the # provider's refusal rather than to act on a list we do not # believe. Falls through to the 502. return None plan = plan_blocker_cleanup(preflight.blockers, confirmed_keys) if plan.unseen: # English, like every sibling message on this path: the # frontend prefers its own localized copy for this code and # only falls back to the raw text, so a Chinese string here # would reach en/ja admins verbatim. raise AppException( code="SCOPE_CLOSE_BLOCKERS_CHANGED", message=( f"{len(plan.unseen)} item(s) in this scope are not on " "the list you confirmed (they appeared after the " "dialog was rendered). Helix never deletes what " "nobody confirmed — re-open the decommission dialog " "to confirm the current list: " + "; ".join(describe_blocker(b) for b in plan.unseen[:5]) ), status_code=409, ) if plan.unremovable and not plan.deletable: raise AppException( code="SCOPE_CLOSE_BLOCKER_NOT_REMOVABLE", message=( "The decommission is blocked by items Helix has no " "permission to delete (or must not delete on its " "own). Clear them in the cloud console, then " "decommission again: " + "; ".join( f"{describe_blocker(b)}" + (f" —— {b.blocked_reason}" if b.blocked_reason else "") for b in plan.unremovable[:5] ) ), status_code=502, ) if not plan.deletable: return None clear: ClearBlockersResult = await driver.clear_close_blockers( scope_ref, keys=plan.deletable, ) if not (clear.started or clear.completed): # Could not act at all. The admin is present, so the # specific provider reason goes to them now instead of # becoming a notification hours later. first_error = ( clear.failed[0].get("error") if clear.failed else None ) raise AppException( code="SCOPE_CLOSE_CLEANUP_FAILED", message=( "Helix tried to clean up this scope's resources but " "could not start: " + ( clear.error_message or first_error or "the provider refused" ) + ". The resources are still there and the scope is " "NOT decommissioned." ), status_code=502, ) now = datetime.now(UTC) scope.decommission_pending_since = now scope.decommission_pending_by = str(actor_user_id) scope.decommission_pending_reason = reason # Persist the CONSENT, not the work list: the resume job # re-reads live blockers every tick and intersects them with # this list again. Storing only what we deleted this round would # make a resource group that appears in ``Deleting`` late look # unconfirmed on the next pass. scope.decommission_cleanup_keys = [ str(k) for k in (confirmed_keys or plan.deletable) ] scope.decommission_cleanup_state = CLEANUP_STATE_CLEANING scope.decommission_cleanup_attempts = 1 scope.decommission_cleanup_error = ( (clear.error_message or "")[:500] or None ) scope.decommission_cleanup_trace_id = trace_id scope.updated_at = now await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.decommission_cleanup_started", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "reason": reason, "provider": integration.provider, "external_id": scope.external_id, "close_error_code": close_result.error_code, "confirmed_keys": scope.decommission_cleanup_keys, "deleted_keys": plan.deletable, "started": clear.started, "completed": clear.completed, "failed": clear.failed, "blockers": [ { "key": b.key, "kind": b.kind, "name": b.name, "child_count": b.child_count, } for b in preflight.blockers ], }, ) return CloseScopeResult( ok=False, error_code="cleanup_in_progress", provider_state="cleanup_in_progress", # Shown to the admin as a progress toast. The frontend has # a localized equivalent (``decommissionCleanupStarted``) # and uses this only when present, so keep it factual and # in the same language as its siblings. error_message=( f"Started cleaning up {len(plan.deletable)} item(s). " "Cloud-side deletion is asynchronous; Helix retries the " "close automatically and notifies you when it lands." ), detail={ "started": clear.started, "completed": clear.completed, "failed": clear.failed, "in_progress": clear.in_progress, }, ) async def _iam_account_ids_for_scope( self, scope: CmdbCloudScope, ) -> list[str]: """Helix-tracked IAM user names on this scope, deduplicated. IAM evacuation only applies to ``standalone_iam`` scopes — ``cmdb_iam_users`` is indexed per integration and standalone_iam scopes are 1:1 with their integration, so we filter by ``integration_id``. SSO scopes (aws_account / azure_subscription / gcp_project / aliyun_account RD) get an empty list; their drivers ignore it and call the multi-account close API instead. """ if scope.scope_type != "standalone_iam": return [] # Defense-in-depth scope binding. The 1:1 invariant # (standalone_iam scopes ↔ integration) is documented but NOT # enforced by a DB constraint. A future schema drift / partial # migration that creates a second standalone_iam scope on the # same integration would otherwise let decommissioning ONE # wipe every Helix-tracked IAM user on the OTHER scope. # # ``cmdb_iam_users.sub_account_id`` is ``String(64) NOT NULL # DEFAULT ''`` (see models.py:487 + iam_upsert.normalize_iam_ # sub_account_id which coerces None → ""). Single-scope drivers # (AWS / Tencent / UCloud / plain Aliyun) store the empty # string; multi-account drivers (AlicloudIntl, Aliyun RD) # stamp the actual sub-account id. The match logic: # * scope.external_id == sub_account_id → same sub-account # (multi-account case, scope-bounded) # * sub_account_id == "" → single-scope-per-integration # case (no sub-account stamping); integration filter is # sufficient because the 1:1 invariant holds # Both branches are scope-safe. scope_external = (scope.external_id or "").strip() rows = (await self.db.execute( select(CmdbIamUser.user_name).where( CmdbIamUser.integration_id == str(scope.integration_id), CmdbIamUser.revoked_at.is_(None), CmdbIamUser.deleted_at.is_(None), CmdbIamUser.user_name.isnot(None), or_( CmdbIamUser.sub_account_id == scope_external, CmdbIamUser.sub_account_id == "", ), ) )).scalars().all() # Preserve order while deduping (some clouds list a user twice # when they're members of multiple groups; deleting twice is # idempotent but noisy in audit). seen: set[str] = set() out: list[str] = [] for r in rows: v = (r or "").strip() if not v or v in seen: continue seen.add(v) out.append(v) return out # ── Grant: request / approve / reject / direct-add ────── async def _reuse_or_create_grant_row( self, *, scope: CmdbCloudScope, person: IdentityPerson, role_key: str, role_display_name: str | None, new_detail: dict[str, Any], request_group_id: str | None, granted_by_person_id: str | None, now_dt: datetime, conflict_message_builder: Callable[[str, str], str], ) -> tuple[CmdbCloudAccessGrant, bool]: """Reuse a terminal-state row for ``(scope, person, role_key)`` or insert a new ``pending`` row when none exists. Returns ``(grant, was_resurrection)``. Raises ``ConflictError`` when an ``active`` / ``pending`` row already covers this triple — caller supplies a ``conflict_message_builder(status, role) -> str`` callable that the helper invokes with the live row's status + role_key. Using a callable (instead of a ``.format()`` template) means the caller's f-string interpolation of dynamic content like ``person.display_name`` cannot leak ``{`` / ``}`` into a downstream ``.format()`` and crash with KeyError / IndexError on names that legitimately contain those characters. Why this exists. The unique key ``uq_cmdb_cloud_grants_scope_person_role`` does not partition by status, so a re-submit on a row whose status is now ``revoked`` / ``expired`` / ``disabled`` MUST update the row in-place; an ``INSERT`` would trip the constraint and 500. Both ``submit_access_request`` (employee-initiated) and ``owner_add_member`` (admin direct-add) hit the same case, so the resurrection logic lives here once. Concurrency. Acquires the same Redis advisory lock (``grant_revoke:``) that ``revoke_grant`` / ``renew_grant`` use, so a concurrent admin revoke / renew on the row we're resurrecting serialises against this writer instead of producing a lost-update / two-cards split-brain. Lock is held for the duration of the row mutation only; the caller's outer notification dispatch is outside the critical section by design (cloud-HTTP latency must not block the revoke path). Forensic preservation. On resurrection we MERGE ``existing.detail`` into the new detail rather than overwriting — the disable path stashes ``disable_result`` (driver outcome + reason + ISO timestamp) in ``detail`` and the grants_sync upsert stashes the cloud-side raw row, both of which are useful when reconstructing "what happened to this grant between then and now". Resurrection markers (``_resurrected_from_status``, ``_resurrected_from_revoke_reason``, ``_resurrected_from_revoked_at``) are stamped so a security auditor can answer "why is this pending now when last week it was revoked?" without correlating two audit rows by grant_id. Source normalization. A ``source='discovered'`` row that gets resurrected becomes a Helix-issued grant by definition (the upcoming approve / dispatch flow is Helix-driven). We flip ``source = 'helix'`` and clear ``last_seen_grants_sync_*`` so the next ``cmdb.grants_sync`` drift sweep doesn't treat this row as discovered-but-missing and re-revoke it. Stale ``external_ref`` / ``external_principal_*`` (the prior cloud- side assignment id, recovered email) is also cleared so dispatch re-creates the cloud-side binding cleanly. """ # Defense-in-depth: refuse production scopes here too even # though both call sites already gate at the top of their # flow. A future third caller forgetting the gate would # otherwise silently produce a prod grant. Idempotent for # current callers. _refuse_prod_self_service(scope) # Same reasoning for sub-tenant (cost-only) scope rows — a # grant there can only fail at dispatch with a provider 400. _refuse_ungrantable_scope_type(scope) existing = await self.db.scalar( select(CmdbCloudAccessGrant) .where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.person_id == str(person.id), CmdbCloudAccessGrant.role_key == role_key, ) ) if existing is not None and existing.status in ( GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING, ): raise ConflictError( conflict_message_builder(existing.status, role_key), ) if existing is not None: # Resurrect. ``ttl=120`` matches ``revoke_grant`` / # ``renew_grant`` so the three writers serialise on the # same lock with a TTL that comfortably covers a slow DB # tick (TiDB election failover / autovacuum pause). The # mutation itself is sub-second pure DB work, so the long # TTL is conservative — never the gating factor — but the # parity prevents a rare "lock expired mid-mutation while # peer writer acquired same key" silent overwrite class. try: async with distributed_lock( f"grant_revoke:{existing.id}", ttl=120, ): return await self._mutate_inactive_to_pending( existing=existing, role_display_name=role_display_name, new_detail=new_detail, request_group_id=request_group_id, granted_by_person_id=granted_by_person_id, now_dt=now_dt, ) except LockBusy as e: # Concurrent revoke/renew/resurrect on the same row — # surface as 409 so the caller retries rather than # silently losing the writer race. raise ConflictError( f"Grant for role {role_key!r} is being modified " "concurrently; retry in a moment.", ) from e grant = CmdbCloudAccessGrant( scope_id=str(scope.id), person_id=str(person.id), role_key=role_key, role_display_name=role_display_name, status=GRANT_STATUS_PENDING, granted_at=now_dt, granted_by_person_id=granted_by_person_id, expires_at=None, # filled on approve / dispatch request_group_id=request_group_id, detail=dict(new_detail), # caller's dict isn't shared source="helix", ) self.db.add(grant) return grant, False async def _mutate_inactive_to_pending( self, *, existing: CmdbCloudAccessGrant, role_display_name: str | None, new_detail: dict[str, Any], request_group_id: str | None, granted_by_person_id: str | None, now_dt: datetime, ) -> tuple[CmdbCloudAccessGrant, bool]: """Inner half of resurrection — runs under the row lock. Defends against the row having flipped to active/pending between the outer probe and the lock acquisition (e.g. a concurrent owner-add for the same triple landed first). In that case raise ``ConflictError`` so the caller's transaction rolls back cleanly instead of silently overwriting another writer's freshly-active row. Stale-cache pitfall (round-2 audit). The outer probe at ``_reuse_or_create_grant_row`` ran BEFORE we held the lock; SQLAlchemy parks the loaded row in the session identity map and a naive ``existing.status`` re-read returns the cached value, not the current DB row. Without an explicit refresh, a competing writer that landed during the outer-probe ↔ lock-acquire window goes undetected and we silently overwrite their write. ``await self.db.refresh(existing, …)`` forces a re-SELECT of the columns we care about so the guard sees ground truth. Refresh is column-scoped to keep the round-trip small; we only re-read the fields the resurrection clobbers plus ``status`` itself. """ await self.db.refresh( existing, attribute_names=[ "status", "revoked_at", "revoked_by_person_id", "revoke_reason", "expires_at", "last_renewed_at", "last_notified_stage", "last_login_at", "request_group_id", "detail", "source", "last_seen_grants_sync_job_id", "last_seen_grants_sync_at", "external_ref", "external_principal_id", "external_principal_email", "granted_by_person_id", "employee_request_id", ], ) if existing.status not in GRANT_STATUSES_RESURRECTABLE: raise ConflictError( f"Grant for role {existing.role_key!r} flipped to " f"{existing.status} during the request; not resurrecting.", ) prior_status = existing.status prior_revoke_reason = existing.revoke_reason prior_revoked_at = existing.revoked_at prior_granted_by_person_id = existing.granted_by_person_id merged_detail: dict[str, Any] = {} if existing.detail: merged_detail.update(existing.detail) merged_detail.update(new_detail) merged_detail["_resurrected_from_status"] = prior_status if prior_revoke_reason is not None: merged_detail["_resurrected_from_revoke_reason"] = ( prior_revoke_reason ) if prior_revoked_at is not None: merged_detail["_resurrected_from_revoked_at"] = ( prior_revoked_at.isoformat() ) if prior_granted_by_person_id is not None: merged_detail["_resurrected_from_granted_by_person_id"] = ( str(prior_granted_by_person_id) ) # Resurrection HISTORY — a row that has been resurrected # multiple times keeps every prior cycle, not just the most # recent. The flat ``_resurrected_from_*`` markers above are # the latest entry; the history list grows append-only so a # security auditor can replay the full cycle of "active → # revoked → resurrected → revoked → resurrected …". history_entry: dict[str, Any] = { "at": now_dt.isoformat(), "from_status": prior_status, } if prior_revoke_reason is not None: history_entry["from_revoke_reason"] = prior_revoke_reason if prior_revoked_at is not None: history_entry["from_revoked_at"] = prior_revoked_at.isoformat() if prior_granted_by_person_id is not None: history_entry["from_granted_by_person_id"] = str( prior_granted_by_person_id, ) history = list(merged_detail.get("_resurrection_history") or []) history.append(history_entry) merged_detail["_resurrection_history"] = history existing.status = GRANT_STATUS_PENDING existing.role_display_name = role_display_name existing.granted_at = now_dt existing.granted_by_person_id = granted_by_person_id existing.expires_at = None # filled on approve / dispatch existing.request_group_id = request_group_id existing.detail = merged_detail existing.revoked_at = None existing.revoked_by_person_id = None existing.revoke_reason = None existing.last_renewed_at = None existing.last_notified_stage = None existing.last_login_at = None existing.employee_request_id = None # ``source='discovered'`` rows that get resurrected become # Helix-issued grants going forward; clearing the sync # bookkeeping prevents the next drift sweep from re-revoking # them. ``external_ref`` / ``external_principal_*`` are # cleared so dispatch re-creates the cloud-side binding # cleanly rather than trying to undo a stale assignment id. existing.source = "helix" existing.last_seen_grants_sync_job_id = None existing.last_seen_grants_sync_at = None existing.external_ref = None existing.external_principal_id = None existing.external_principal_email = None return existing, True async def submit_access_request( self, *, scope_id: str, person_id: str, role_key: str, role_display_name: str | None, reason: str | None, ttl_days: int | None, requester_id: uuid.UUID, trace_id: str, extra_role_keys: list[str] | None = None, actor_role_names: list[str] | None = None, ) -> CmdbCloudAccessGrant: """Employee-initiated flow — status=pending until owner approves. Returns the *primary* grant; extra roles are inserted alongside in the same ``request_group_id``. Approve / reject / revoke in the service all walk the group and dispatch on each member. Scope with no owner → reject (``ScopeUnownedError``). Duplicate grant for the same (scope, person, role) on an active row → conflict. Duplicate inside ``[role_key, *extra_role_keys]`` → conflict (the form should de-dupe but defense-in-depth). Defense-in-depth (added 2026-05-06 launch audit): if the caller is requesting access for a *different* person, they must be a cloud admin. The MCP tool wrapper hardcodes ``person_id = rt.user.id`` so this never fires today, but a future refactor that exposes ``person_id`` to the caller would otherwise let any employee request access on behalf of anyone else. """ if str(person_id) != str(requester_id): from app.core.rbac import can_manage_cloud if not can_manage_cloud(actor_role_names or []): raise AppException( code="FORBIDDEN", message=( "Cannot request cloud access on behalf of another " "user without cloud-admin role." ), status_code=403, ) scope = await self.db.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) if scope.status != SCOPE_STATUS_ACTIVE: raise AppException( code="CLOUD_SCOPE_NOT_ACTIVE", message=f"Scope {scope_id} is {scope.status!r}; cannot request access", status_code=409, ) # Production gate runs before the unowned check so the # "go to Kidd.li" message wins on prod scopes that also # happen to be unowned (otherwise the user gets a misleading # ``ScopeUnownedError`` and emails IT to assign an owner that # would never become reachable through self-service anyway). _refuse_prod_self_service(scope) # …and before the unowned check for the same reason: a # cost-only RG row DOES carry an owner (the API-key requester), # so the ownerless branch would never catch it. _refuse_ungrantable_scope_type(scope) # Billing-account scopes (GCP) are intentionally ownerless — a # billing account has no owner-equivalent IAM role to mint, so the # approval authority is the cloud-admin team: the pending-request # notification below fans out to ``can_manage_cloud`` holders, and # ``approve_grant``'s router gate ``_require_scope_owner_or_admin`` # admits only ``can_manage_cloud`` when there is no owner. Every # OTHER scope type still requires an assigned owner to route the # approval — an ownerless project/subscription is a config gap, not # a self-service target. if ( not scope.owner_person_id and scope.scope_type != SCOPE_TYPE_GCP_BILLING_ACCOUNT ): raise ScopeUnownedError(scope_id) person = await self.db.get(IdentityPerson, uuid.UUID(person_id)) if person is None or person.deleted_at is not None: raise NotFoundError("IdentityPerson", person_id) # Build the canonical role list: primary first (so it's the row # returned to the caller), extras after, de-duplicated while # preserving order. all_roles: list[str] = [role_key] for k in extra_role_keys or []: k_norm = (k or "").strip() if not k_norm or k_norm == role_key: continue if k_norm in all_roles: continue all_roles.append(k_norm) if any(not r for r in all_roles): raise AppException( code="CLOUD_GRANT_INVALID_ROLE", message="role_key cannot be blank", status_code=400, ) # Role-governance gate (centralised here so HTTP + MCP both # enforce — the legacy is_admin_role check lived in the router / # MCP wrapper and could drift). ``ungrantable`` can never be # bound at this scope (provider rejects it) → refuse up front so # the user never gets a confusing approval-time failure. # ``restricted`` (owner-equivalent / full-IAM-admin) is not self- # serviceable — only a cloud admin can request it. ``elevated`` # IS self-requestable but its approval requires a cloud admin # (enforced in ``approve_grant``), so it's allowed to pend here. from app.core.rbac import can_manage_cloud from app.integrations.cloud_role_governance import ( TIER_ELEVATED, TIER_RESTRICTED, TIER_UNGRANTABLE, ) caller_is_cloud_admin = can_manage_cloud(actor_role_names or []) # Whether approving THIS request needs cloud-admin authority — any # elevated/restricted role in the group means a regular scope owner # can't approve it (gate in ``_approve_grant_locked``). Captured here # (reusing the same per-role classification the gate uses: primary # with its display name, extras with None) so the "[待审批]" push is # routed to people who can actually approve — see # ``_resolve_request_approval_recipients``. request_needs_admin_approval = False for r_key in all_roles: gov = await _classify_scope_role( self.db, scope, r_key, role_display_name if r_key == role_key else None, ) if gov.tier in (TIER_ELEVATED, TIER_RESTRICTED): request_needs_admin_approval = True if gov.tier == TIER_UNGRANTABLE: raise AppException( code="CLOUD_ROLE_UNGRANTABLE", message=( f"角色 {r_key} 不能授予在该 scope 上({gov.reason})。" "请选择适用于该 scope 的角色。" ), status_code=422, ) if gov.tier == TIER_RESTRICTED and not caller_is_cloud_admin: raise AppException( code="CLOUD_ROLE_RESTRICTED", message=( f"角色 {r_key} 为受限角色(owner 等价 / 全 IAM 管理)," "仅 IT / 云管理员可授予。请联系管理员。" ), status_code=403, ) # Driver-level shape pre-flight: refuse a role_key the eventual # grant_access would reject (e.g. RAM-shape key on a CloudSSO # AliCloud tenant) so the requester is told now instead of the # approver hitting the failure at grant time. Fails open. await _preflight_request_role_shapes(self.db, scope, all_roles) request_group_id = str(uuid.uuid4()) resurrection_summary: list[dict[str, Any]] = [] primary: CmdbCloudAccessGrant | None = None now_dt = datetime.now(UTC) for r_key in all_roles: r_display = ( role_display_name if r_key == role_key else None ) new_detail = { "reason": reason or "", "requested_ttl_days": ttl_days, "primary_role_key": role_key, } grant, was_resurrection = await self._reuse_or_create_grant_row( scope=scope, person=person, role_key=r_key, role_display_name=r_display, new_detail=new_detail, request_group_id=request_group_id, granted_by_person_id=None, now_dt=now_dt, conflict_message_builder=lambda status, role: ( f"Already have an {status} grant for role {role!r}" ), ) if was_resurrection: resurrection_summary.append({ "role_key": r_key, "grant_id": str(grant.id), "prior_status": (grant.detail or {}).get( "_resurrected_from_status", ), "prior_revoke_reason": (grant.detail or {}).get( "_resurrected_from_revoke_reason", ), }) if r_key == role_key: primary = grant await self.db.flush() # ``primary`` is guaranteed non-None because ``role_key`` is # always in ``all_roles``; assert for the type-checker. assert primary is not None await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=requester_id, action="cloud.grant.request", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(primary.id), detail={ "scope_id": str(scope.id), # Round-22 audit hardening: record the grant RECIPIENT # (``person_id``) separately from the actor # (``actor_user_id`` above). Today the HTTP layer # enforces actor == recipient, but a future MCP / # internal caller can pass a different ``person_id`` # via ``submit_access_request`` (the cloud-admin # "request on behalf of" four-eyes gate). Forensic # readers MUST be able to answer "who is this grant # actually for" without correlating against the # CmdbCloudAccessGrant row that may yet be revoked. "person_id": str(person_id), "requested_for_other_person": ( str(person_id) != str(requester_id) ), "primary_role_key": role_key, "extra_role_keys": [k for k in all_roles if k != role_key], "request_group_id": request_group_id, "reason": reason, # Forensic markers so a security auditor can answer # "why is this grant pending now when last week it # was revoked?" without correlating two audit rows # by grant_id manually. Empty list when no rows were # resurrected (all role_keys were brand-new). "resurrected_grants": resurrection_summary, }, ) # Notify the approval authority once for the whole group (one # message → approve or reject all roles together). For an OWNED # scope that's the owner; for an intentionally-ownerless # billing-account scope it's every cloud admin (``can_manage_cloud``) # — see ``_resolve_request_approval_recipients``. Without the # ownerless fan-out the request would land in a queue nobody is # paged about (and the old single-owner path would crash on # ``uuid.UUID(str(None))``). extra_lines = "\n".join( f" • {k}" for k in all_roles[1:] ) extras_block = ( f"\n附加角色:\n{extra_lines}" if extra_lines else "" ) approval_recipients = await self._resolve_request_approval_recipients( scope, requires_admin=request_needs_admin_approval, ) for recipient in approval_recipients: try: await self.notify.dispatch( NotificationEnvelope( recipient=recipient, title=( f"[待审批] {person.display_name or person.work_email} " f"申请 {scope.display_name or scope.external_id} 的云访问" ), body_text=( f"申请人: {person.display_name or person.work_email}\n" f"Scope: {scope.display_name or scope.external_id}\n" f"主角色: {role_display_name or role_key}" f"{extras_block}\n" f"理由: {reason or '(无)'}" ), # ``env.url`` is rendered absolute via # ``app.frontend_base_url`` and shown as the # interactive-card action button — no inline # path inside body_text (which Feishu would # render as plain unclickable text). # # Use the bare ``/cloud/scopes/{id}`` redirect # so the recipient's role decides the landing # page: ``global_admin`` lands on # ``/admin/cloud/scopes/{id}`` (full approve / # reject buttons unconditionally rendered); the # scope owner and everyone else lands on # ``/my/cloud/scopes/{id}`` (owner-aware # surface). Hard-coding ``/my/...`` here meant # an IT admin invited to help approve while the # owner is OOO landed on a page that hid the # approve buttons (``isOwner === false``), # forcing the round-trip workaround of # transferring scope ownership to themselves # just to surface the buttons. # # ``?tab=pending`` lands the approver directly on # the 待审批 queue tab (the redirect at # ``CloudScopeRedirect`` preserves the query # string into ``/admin`` or ``/my``); without it # the page opened on its 总览/成员 landing tab and # the approver had to hunt for the pending list. # ``grant`` is deliberately additive: desktop keeps # its role-aware scope page, while the mobile deep # link mapper promotes it to this exact approval's # one-tap decision screen. url=( f"/cloud/scopes/{scope.id}?tab=pending" f"&grant={primary.id}" ), channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=primary.id, ), trace_id=trace_id, ) except Exception as e: # noqa: BLE001 logger.warning("notify approver of grant request failed: %s", e) # Confirm submission to the requester so they don't wonder # whether the click did anything. Until 2026-05-19 the only # acknowledgement was the HTTP 200 + frontend toast — once the # tab was closed the requester had no record of submission, # and no way to tell "is it still pending or did it fail # silently?" without DM'ing IT. Feishu + in-app inbox so the # confirmation is durable across browser sessions; same # ``related_resource_id`` as the owner notification so a # reject / approve later threads from this card in the # requester's inbox. # # Suppressed when the requester is requesting on behalf of # someone else (cloud-admin four-eyes path) — the "you have a # request pending" message belongs to ``person``, not the # actor. The actor is shown the result via the HTTP response # and audit log. if ( person is not None and person.work_email and person.employment_status not in ("left", "leaver_pending") and str(person.id) == str(requester_id) ): extras_block_req = ( "\n附加角色:\n" + "\n".join(f" • {k}" for k in all_roles[1:]) if len(all_roles) > 1 else "" ) # Who reviews this — the scope owner for an owned scope, or # the cloud-admin team for an intentionally-ownerless # billing-account scope (``approval_recipients`` resolved # above is the authoritative recipient set for both). if scope.owner_person_id: owner_label = ( (approval_recipients[0].display_name or approval_recipients[0].work_email) if approval_recipients else "(未指派 Owner)" ) else: owner_label = "云管理员团队" try: await self.notify.dispatch( NotificationEnvelope( recipient=person, title=( f"[已提交] 你申请 {scope.display_name or scope.external_id} " f"的云访问已进入审批" ), body_text=( f"Scope: {scope.display_name or scope.external_id}\n" f"主角色: {role_display_name or role_key}" f"{extras_block_req}\n" f"理由: {reason or '(无)'}\n" f"待审批人: {owner_label}\n\n" f"审批通过后系统会自动开通访问;" "如需撤回，可在「我的子账号申请」里取消。" ), url=f"/cloud/scopes/{scope.id}", channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=primary.id, ), trace_id=trace_id, ) except Exception as e: # noqa: BLE001 logger.warning( "notify requester of grant submission failed: %s", e, ) return primary async def _load_request_group_pending( self, grant: CmdbCloudAccessGrant, ) -> list[CmdbCloudAccessGrant]: """Return every PENDING sibling sharing this grant's ``request_group_id`` (including the grant itself). Concurrency is now handled by the Redis advisory lock acquired in ``approve_grant`` / ``reject_grant`` — we no longer hold a ``FOR UPDATE`` lock on the sibling rows because the cloud HTTP call latency in the dispatch loop would block concurrent sync upserts on the same rows for tens of seconds. For grants without a group (legacy single-role requests, or cloud-side discovered grants that never had one), returns just ``[grant]`` — the caller's logic stays a no-op for the legacy path. """ if not grant.request_group_id: return [grant] siblings = list((await self.db.execute( select(CmdbCloudAccessGrant) .where( CmdbCloudAccessGrant.request_group_id == grant.request_group_id, CmdbCloudAccessGrant.status == GRANT_STATUS_PENDING, ) )).scalars().all()) # Defensive — if for some reason the grant we already locked # isn't in the sibling re-read (shouldn't happen, but guard # against driver bugs), include it. if grant not in siblings: siblings.append(grant) return siblings async def approve_grant( self, grant_id: str, *, approver_id: uuid.UUID, ttl_days: int | None = None, trace_id: str, approver_role_names: list[str] | None = None, ) -> LifecycleGrantOutcome: # Concurrency guard: two scope owners approving the same group # at the same time could both pass the ``status == pending`` # check and double-dispatch to the provider. We use a Redis # advisory lock keyed by the request_group_id (or grant_id when # there's no group) instead of holding a MySQL row lock — the # cloud HTTP calls inside the dispatch loop can run for several # seconds per role, and a FOR UPDATE held that long blocks # every concurrent sync upsert hitting the same grant rows. # The lock is released cleanly after the loop + audit; second # caller re-reads, sees status=active, raises ConflictError. # ``LockBusy`` surfaces from ``async with``, not from the bare # ``distributed_lock()`` call (it's an @asynccontextmanager). # # **Group-scoped lock key** (PR 2026-05-17 fix): the inner # ``_approve_grant_locked`` iterates EVERY pending sibling in # the same ``request_group_id`` and dispatches each. Locking # by ``grant_id`` only serialises duplicate clicks on the SAME # grant — two owners clicking approve on different sibling # rows X and Y of the same multi-role request would grab # different per-grant locks, both proceed, both iterate the # group, and double-dispatch every role. Pre-fetch the group # id and lock by group when present so concurrent approves # across the group serialise. Falls back to ``grant_id`` for # single-grant submissions (no group). group_id = await self.db.scalar( select(CmdbCloudAccessGrant.request_group_id) .where(CmdbCloudAccessGrant.id == uuid.UUID(grant_id)) ) lock_key = ( f"approve_group:{group_id}" if group_id else f"approve_grant:{grant_id}" ) try: # TTL=900s (was 300s, raised 2026-05-09 round 16): the # approve path iterates every PENDING sibling in the same # ``request_group_id`` and dispatches each role # SEQUENTIALLY. With 5 roles × ~60s worst-case provider # call (Identity Center cold start, AssumeRole + SCIM # round-trip) the loop alone consumed the prior 300s # window, releasing the lock before the loop finished # and re-opening the double-dispatch race the lock was # meant to close. 15 min is safe even for the largest # group + slowest provider; the cost of a longer TTL is # only "stuck process keeps lock longer" which is # desirable here (the second click sees the lock held # and 409s instead of trying to dispatch concurrently). async with distributed_lock(lock_key, ttl=900): return await self._approve_grant_locked( grant_id, approver_id=approver_id, ttl_days=ttl_days, trace_id=trace_id, approver_role_names=approver_role_names, ) except LockBusy as exc: raise ConflictError( f"Grant {grant_id} is already being approved", ) from exc async def _approve_grant_locked( self, grant_id: str, *, approver_id: uuid.UUID, ttl_days: int | None, trace_id: str, approver_role_names: list[str] | None = None, ) -> LifecycleGrantOutcome: grant = await self.db.scalar( select(CmdbCloudAccessGrant) .where(CmdbCloudAccessGrant.id == uuid.UUID(grant_id)) ) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) if grant.status != GRANT_STATUS_PENDING: raise ConflictError( f"Grant {grant_id} is {grant.status!r}, not pending" ) # Block self-approval. The router gate ``_require_scope_owner_or_admin`` # passes a scope owner approving any grant on their scope — # including a grant they themselves just requested. Without this # check the four-eyes intent of the approval flow collapses to # one-eye for any scope owner. (``owner_add_member`` exists for # the legitimate "owner gives themselves a role" path; force # callers through that path.) if str(grant.person_id) == str(approver_id): raise ConflictError( "Self-approval is not permitted; ask another scope " "owner or a global_admin to review your request, or " "use owner-add-member if you need to grant yourself " "a role on your own scope." ) # Round-42 audit: approve held the ``approve_group:`` lock while # decommission holds ``decommission_scope:`` — different # namespaces, so they don't mutex on each other. Without an # explicit row lock here, the parent REPEATABLE-READ snapshot # observed the scope's pre-decommission status even when the # decommission has already committed. ``SELECT … FOR UPDATE`` # forces a fresh read past the snapshot so the status check # below sees the latest committed value. scope_q = await self.db.execute( select(CmdbCloudScope) .where(CmdbCloudScope.id == str(grant.scope_id)) .with_for_update() ) scope = scope_q.scalar_one_or_none() if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", str(grant.scope_id)) # Hard-gate on scope status before dispatching to the cloud — # submit/owner_add_member already enforce this; approve was the # outlier. The grant stays in PENDING (no row mutation here) # so an admin can see the unblocked queue. if scope.status != SCOPE_STATUS_ACTIVE: raise AppException( code="CLOUD_SCOPE_NOT_ACTIVE", message=( f"Cannot approve a grant on scope {scope.id} — " f"current status is {scope.status!r}. " "The scope was likely decommissioned between " "submission and approval; the grant remains " "pending for visibility." ), status_code=409, ) # Group dispatch: if this grant came in as part of a multi-role # submission (``request_group_id`` set), approve every PENDING # sibling in the same group. Each provider call still runs in # sequence so a partial-failure leaves the failed grant in # PENDING for retry while the successful ones go ACTIVE — same # semantics as the single-grant path. siblings = await self._load_request_group_pending(grant) # Bucket-B approval gate: elevated / restricted roles (owner- # equivalent / IAM-write / broad write) can only be APPROVED by a # cloud admin — a scope owner who is a regular employee must not # rubber-stamp a privilege escalation. Checked across every # pending sibling BEFORE any dispatch, so a mixed group fails # atomically rather than half-granting. See # app/integrations/cloud_role_governance.py and DECISIONS 2026-06-06. from app.core.rbac import can_manage_cloud from app.integrations.cloud_role_governance import ( TIER_ELEVATED, TIER_RESTRICTED, ) if not can_manage_cloud(approver_role_names or []): for g in siblings: gov = await _classify_scope_role( self.db, scope, g.role_key, g.role_display_name, ) if gov.tier in (TIER_ELEVATED, TIER_RESTRICTED): raise AppException( code="CLOUD_APPROVAL_NEEDS_ADMIN", message=( f"角色 {g.role_key} 为高危角色({gov.tier})," "仅 IT / 云管理员可审批,scope owner 无法单独" "批准。请转交云管理员审批。" ), status_code=403, ) primary_outcome: LifecycleGrantOutcome | None = None partial_errors: list[str] = [] primary_race_status: str | None = None for g in siblings: # Defense-in-depth: re-read each sibling's status before # dispatch. The approve path locks on ``approve_group:`` and # the revoke / reject paths lock on ``grant_revoke:`` — # different namespaces, so a revoke fired between the SELECT # in ``_load_request_group_pending`` and reaching this # iteration won't have serialised against us. Without the # re-read, a sibling already flipped to REVOKED would be # re-dispatched as if still PENDING, writing a stale # ``cloud.grant.approve`` audit row + a confused # ``status=ACTIVE`` overwriting the prior ``revoked_at``. # # Why SELECT … FOR UPDATE: the parent ``self.db`` session # runs under MySQL REPEATABLE READ (Helix doesn't set the # isolation level so we get the InnoDB default). A # consistent-read snapshot was established at the first # SELECT in ``approve_grant`` and any same-session # ``session.refresh`` or non-locking ``select()`` is bound # to that snapshot — concurrent commits from a revoke in # another session are invisible. ``SELECT … FOR UPDATE`` is # a locking read that bypasses the snapshot AND acquires # the per-row write lock; held only until # ``_dispatch_grant`` (below) commits internally, so a # concurrent revoke is briefly serialised, not 60-second- # blocked. A fresh session would also work but tests with # an unpatched session_factory would hit the wrong engine. fresh_status = await self.db.scalar( select(CmdbCloudAccessGrant.status) .where(CmdbCloudAccessGrant.id == g.id) .with_for_update() ) if fresh_status != GRANT_STATUS_PENDING: if g.id == grant.id: # Primary flipped under us (concurrent revoke / # reject landed first). Record the race and let # the loop finish — siblings still in PENDING are # legitimately approve-able and any successful # dispatch is already committed by # ``_dispatch_grant``. We raise once at the end # so the caller sees a clean 409. primary_race_status = fresh_status or "unknown" continue partial_errors.append( f"{g.role_key}: skipped — status changed to " f"{fresh_status or 'unknown'} between approval queue " "and dispatch (concurrent revoke / reject)" ) continue outcome = await self._dispatch_grant( grant=g, scope=scope, ttl_days=ttl_days, actor_user_id=approver_id, trace_id=trace_id, action="cloud.grant.approve", ) if g.id == grant.id: primary_outcome = outcome if outcome.status == GRANT_STATUS_PENDING: # Driver returned ok=False → grant stayed PENDING. partial_errors.append( f"{g.role_key}: {outcome.warning or 'driver failed'}", ) # If the primary itself raced with a concurrent revoke, surface # ConflictError after the loop so any siblings still legitimately # approve-able have already been dispatched (their # ``_dispatch_grant`` calls each commit independently, so their # outcomes are durable past the raise). The caller's 409 now # reflects "your primary grant was revoked under you" without # silently losing the partial dispatch on the other siblings. if primary_race_status is not None: raise ConflictError( f"Grant {grant_id} is {primary_race_status!r}, not pending " "(concurrent revoke / reject landed first)" ) # The grant we locked at the top is always in ``siblings`` so # ``primary_outcome`` is always populated; assert for the # type-checker. assert primary_outcome is not None if partial_errors and len(siblings) > 1: existing_warning = primary_outcome.warning or "" primary_outcome.warning = ( f"{existing_warning}; partial group failures: " f"{'; '.join(partial_errors)}" if existing_warning else f"partial group failures: {'; '.join(partial_errors)}" ) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=approver_id, action="cloud.grant.approve", status="partial_failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "request_group_id": ( str(grant.request_group_id) if grant.request_group_id else None ), "total": len(siblings), "failed": len(partial_errors), "partial_errors": partial_errors, }, ) return primary_outcome async def reject_grant( self, grant_id: str, *, rejector_id: uuid.UUID, reason: str | None, trace_id: str, ) -> CmdbCloudAccessGrant: # Mirror ``approve_grant``: lock by ``approve_group:`` keyed by # ``request_group_id`` when present so an approve and a reject # on different siblings of the same group serialise via the # same Redis lock. Falls back to ``approve_grant:`` per-grant # for legacy single-grant submissions with no group id. Without # this, a concurrent reject and approve on sibling X and Y of # the same group would each grab different keys, both iterate # the group, and write conflicting state. group_id = await self.db.scalar( select(CmdbCloudAccessGrant.request_group_id) .where(CmdbCloudAccessGrant.id == uuid.UUID(grant_id)) ) lock_key = ( f"approve_group:{group_id}" if group_id else f"approve_grant:{grant_id}" ) try: async with distributed_lock(lock_key, ttl=120): return await self._reject_grant_locked( grant_id, rejector_id=rejector_id, reason=reason, trace_id=trace_id, ) except LockBusy as exc: raise ConflictError( f"Grant {grant_id} is being approved/rejected concurrently", ) from exc async def _reject_grant_locked( self, grant_id: str, *, rejector_id: uuid.UUID, reason: str | None, trace_id: str, ) -> CmdbCloudAccessGrant: grant = await self.db.scalar( select(CmdbCloudAccessGrant) .where(CmdbCloudAccessGrant.id == uuid.UUID(grant_id)) ) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) if grant.status != GRANT_STATUS_PENDING: raise ConflictError(f"Grant {grant_id} is not pending") # Reject every PENDING sibling in the same request group at # once — the user submitted them as a unit, the owner says no # to the unit. Each row gets its own audit entry so the # provenance is complete. # # Defense-in-depth (Round-3 review): mirror the per-sibling # fresh-status check that ``_approve_grant_locked`` performs. # ``_load_request_group_pending`` filters at SELECT time but # the parent session is on MySQL REPEATABLE READ — a concurrent # path that flipped a sibling's status (e.g. resurrection of a # row that happens to share ``request_group_id``) AFTER our # initial SELECT but BEFORE this loop wouldn't be visible. The # in-memory write would then overwrite a legitimate ACTIVE row # to REVOKED. ``SELECT … FOR UPDATE`` bypasses the snapshot AND # acquires the per-row write lock so a concurrent flipper is # briefly serialised; lock is released when the surrounding # request handler commits the session. siblings = await self._load_request_group_pending(grant) rejected_role_keys: list[str] = [] for g in siblings: fresh_status = await self.db.scalar( select(CmdbCloudAccessGrant.status) .where(CmdbCloudAccessGrant.id == g.id) .with_for_update() ) if fresh_status != GRANT_STATUS_PENDING: # Sibling already left PENDING (concurrent approve / # revoke / resurrect). Skip silently — the row's # current state is authoritative; we have no business # forcing a reject onto a non-pending row. continue g.status = GRANT_STATUS_REVOKED g.revoked_at = datetime.now(UTC) g.revoked_by_person_id = str(rejector_id) g.revoke_reason = GRANT_REVOKE_REASON_OWNER g.detail = {**(g.detail or {}), "reject_reason": reason or ""} rejected_role_keys.append(g.role_display_name or g.role_key) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=rejector_id, action="cloud.grant.reject", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(g.id), detail={"reason": reason, "request_group_id": grant.request_group_id}, ) # Notify the requester that their submission was rejected. The # approve path already notifies the member with "[已开通]"; # without a matching reject notification the requester sees no # signal at all and has to keep checking /my/cloud/grants # manually. Best-effort: the reject is durably persisted via # the route-handler commit upstream; failure here logs but # doesn't roll back. Suppressed for orphan-cleanup rows where # ``person_id`` is NULL. if rejected_role_keys and grant.person_id: requester = await self.db.get( IdentityPerson, uuid.UUID(str(grant.person_id)), ) scope = await self.db.get( CmdbCloudScope, uuid.UUID(str(grant.scope_id)), ) if ( requester is not None and scope is not None and requester.work_email and requester.employment_status not in ( "left", "leaver_pending", ) ): role_lines = "\n".join(f" • {k}" for k in rejected_role_keys) try: await self.notify.dispatch( NotificationEnvelope( recipient=requester, title=( f"[已驳回] 你申请 " f"{scope.display_name or scope.external_id} " f"的云访问被驳回" ), body_text=( f"Scope: {scope.display_name or scope.external_id}\n" f"被驳回的角色:\n{role_lines}\n" f"驳回理由: {reason or '(无)'}\n\n" "如有疑问可联系 scope owner;如仍需要访问，" "请修改理由后重新提交。" ), url=f"/cloud/scopes/{scope.id}", channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, ), trace_id=trace_id, ) except Exception as e: # noqa: BLE001 logger.warning( "notify requester of grant reject failed: %s", e, ) return grant async def provision_initial_owner_grant( self, *, scope_id: str, role_key: str, role_display_name: str, trace_id: str, is_backfill: bool = False, ) -> LifecycleGrantOutcome | None: """System-initiated auto-grant of an owner-equivalent role to the requester immediately after ``driver.create_scope``. Used by the post-provision background task (see ``lifecycle_router._post_provision_finalize``). Unlike :meth:`owner_add_member`, this path is allowed to grant the scope owner themselves a named role — the standard self-service flow explicitly forbids owner-self-add (avoid a scope owner picking some elevated named role without four-eyes review), but for the *initial* provisioning of a brand-new scope the requester IS supposed to start with cloud-side IAM equivalent to their Helix-side ownership, otherwise they can't actually use the project they just created. Idempotent: a re-run on a scope that already has a matching active grant adopts the existing row (the underlying ``_reuse_or_create_grant_row`` helper handles resurrection / no-op cases). Returns ``None`` if the scope is missing or has no ``owner_person_id`` (caller should treat as "skip — nothing to do"); otherwise returns the dispatch outcome. Best-effort by design: the caller is the post-provision background task; failures here log + return PENDING, the requester can re-attempt via the standard owner-add flow once an admin has set them up. """ scope = await self.db.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or scope.deleted_at is not None: return None # Billing-account scopes have no owner-equivalent IAM role to mint # (roles/owner isn't valid on a billing account) — the owner is an # approval authority, not an automatic grant holder. Skip. if scope.scope_type == SCOPE_TYPE_GCP_BILLING_ACCOUNT: return None if scope.status != SCOPE_STATUS_ACTIVE: logger.warning( "provision_initial_owner_grant: scope %s is %r — skipping", scope_id, scope.status, ) return None if not scope.owner_person_id: logger.warning( "provision_initial_owner_grant: scope %s has no owner — skipping", scope_id, ) return None owner = await self.db.get( IdentityPerson, uuid.UUID(str(scope.owner_person_id)), ) if owner is None or owner.deleted_at is not None: logger.warning( "provision_initial_owner_grant: scope %s owner %s missing — skipping", scope_id, scope.owner_person_id, ) return None # Leaver gate (parity with manual_provision + assign_scope_owner + # owner_add_member). Multi-day Feishu approval cycles can land the # owner in ``leaver_pending`` / ``left`` between submit + finalize; # the platform invariant "leavers cannot hold active grants" must # apply on this path too. The backfill cron also routes through # this function so one gate covers both call sites. if owner.employment_status in ("left", "leaver_pending"): logger.warning( "provision_initial_owner_grant: scope %s owner %s is %r " "— skipping (leaver gate)", scope_id, owner.id, owner.employment_status, ) return None # Reaching here means the caller resolved the owner role # successfully (this method is only ever called with a concrete # ``role_key``), so any ``auto_owner_grant_unresolvable_at`` # backoff marker the backfill cron stamped on a prior tick is now # stale — clear it so the scope drops out of the "skip until the # staleness window" branch of the candidate scan and the admin # UI stops showing it as unresolvable. The ``_dispatch_grant`` # call below commits this session, so the clear persists with the # grant row. Guarded to avoid a no-op ``updated_at`` churn. if scope.auto_owner_grant_unresolvable_at is not None: scope.auto_owner_grant_unresolvable_at = None person_label = owner.display_name or owner.work_email or str(owner.id) # Retry path (review-sweep #2/#7): a prior attempt may have # committed a PENDING owner-grant row whose driver bind failed # (``_dispatch_grant`` returns PENDING + ``detail.last_error`` on # a not-ok ``driver.grant_access`` — no cloud binding exists). # ``_reuse_or_create_grant_row`` would raise ``ConflictError`` on # such a row, which — now that the backfill cron's candidate scan # excludes only ACTIVE owner grants — would strand the scope in a # ConflictError-every-tick loop and never re-attempt the bind. # Re-dispatch the existing PENDING row directly so the driver call # is retried (idempotent: ``_dispatch_grant`` flips to ACTIVE only # on success, else leaves it PENDING for the next tick). existing_pending = await self.db.scalar( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.person_id == str(owner.id), CmdbCloudAccessGrant.role_key == role_key, CmdbCloudAccessGrant.status == GRANT_STATUS_PENDING, CmdbCloudAccessGrant.source == "helix", ) ) if existing_pending is not None: return await self._dispatch_grant( grant=existing_pending, scope=scope, ttl_days=None, actor_user_id=uuid.UUID(str(owner.id)), trace_id=trace_id, action="cloud.scope_request.auto_owner_grant", silent=is_backfill, force_perpetual=is_backfill, ) grant, _was_resurrection = await self._reuse_or_create_grant_row( scope=scope, person=owner, role_key=role_key, role_display_name=role_display_name, new_detail={ "auto_owner_grant": True, # ``backfill_origin`` distinguishes rows the backfill cron # wrote (owner already had cloud-side access; Helix only # caught up its own bookkeeping) from rows the inline # post-provision path wrote (a brand-new scope, the # requester needs cloud IAM bound). Read by the admin UI # / future audit queries to explain the source of the # row without parsing audit logs. **({"backfill_origin": True} if is_backfill else {}), }, request_group_id=None, granted_by_person_id=str(owner.id), now_dt=datetime.now(UTC), conflict_message_builder=lambda status, _role: ( f"{person_label} already has an {status} grant for that role" ), ) await self.db.flush() return await self._dispatch_grant( grant=grant, scope=scope, ttl_days=None, actor_user_id=uuid.UUID(str(owner.id)), trace_id=trace_id, action="cloud.scope_request.auto_owner_grant", silent=is_backfill, force_perpetual=is_backfill, ) async def owner_add_member( self, *, scope_id: str, person_id: str, role_key: str, role_display_name: str | None, ttl_days: int | None, actor_user_id: uuid.UUID, trace_id: str, actor_role_names: list[str] | None = None, ) -> LifecycleGrantOutcome: """Owner-direct add — no approval, audited, strictly checks scope ownership upstream (router layer enforces the RBAC gate). Creates a new grant row in ``active`` state and immediately dispatches to the driver. If the driver fails the row lands in ``pending`` so the owner can retry from the UI. """ scope = await self.db.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) if scope.status != SCOPE_STATUS_ACTIVE: raise AppException( code="CLOUD_SCOPE_NOT_ACTIVE", message=f"Scope {scope_id} is {scope.status!r}", status_code=409, ) # Same flat invariant as ``submit_access_request``: production # scopes are not granted through any Helix self-service flow, # owner-add included. Admins who legitimately need to provision # prod for someone go through the security team's intake # (Kidd.li). This block is BEFORE the owner check so the # "go to Kidd.li" message wins on prod scopes that also happen # to be unowned. Mirrored from submit_access_request via # ``_refuse_prod_self_service`` so the two paths can never drift. _refuse_prod_self_service(scope) # Ditto for sub-tenant (cost-only) scope rows — mirrored from # ``submit_access_request`` via ``_refuse_ungrantable_scope_type``. _refuse_ungrantable_scope_type(scope) # Billing-account scopes are intentionally ownerless (see # ``submit_access_request``); a cloud admin reaching here via # ``_require_scope_owner_or_admin`` may still directly add a member # (e.g. grant billing.viewer to finance) without an assigned owner. # The owner-self-add guard below is None-safe for this case. if ( not scope.owner_person_id and scope.scope_type != SCOPE_TYPE_GCP_BILLING_ACCOUNT ): raise ScopeUnownedError(scope_id) # A regular scope owner must not self-add: their ownership alone # cannot be used to choose a named role without a second review. # # Cloud/global admins are the exception. They need an auditable # break-glass repair path when a prior expiry/revoke bug leaves the # recorded owner with no actual grant. Blocking them merely repeats # the broken invariant (and makes the UI claim the owner has access # when the member list is empty). The role-governance gate below # still applies, and the dispatch is explicitly perpetual so the # repaired owner's access cannot be recycled by the expiry sweep. is_owner_self_add = str(scope.owner_person_id) == str(person_id) from app.core.rbac import can_manage_cloud actor_can_manage_cloud = can_manage_cloud(actor_role_names or []) if is_owner_self_add and not actor_can_manage_cloud: raise ConflictError( "Scope owner already has owner-equivalent access; " "owner-add-member is for ADDING others, not yourself. " "If you need a specific named cloud role, file a " "regular access request and have another reviewer " "approve it." ) # Role-governance gate (mirrors submit_access_request, centralised # off the router). ``ungrantable`` is refused for everyone — even # a cloud admin can't bind it at this scope. ``elevated`` / # ``restricted`` (owner-equivalent / IAM-write / broad write) may # only be direct-added by a cloud admin; a regular-employee scope # owner must not elevate a colleague through this no-approval path. from app.integrations.cloud_role_governance import ( TIER_ELEVATED, TIER_RESTRICTED, TIER_UNGRANTABLE, ) gov = await _classify_scope_role( self.db, scope, role_key, role_display_name, ) if gov.tier == TIER_UNGRANTABLE: raise AppException( code="CLOUD_ROLE_UNGRANTABLE", message=( f"角色 {role_key} 不能授予在该 scope 上({gov.reason})。" "请选择适用于该 scope 的角色。" ), status_code=422, ) if ( gov.tier in (TIER_ELEVATED, TIER_RESTRICTED) and not actor_can_manage_cloud ): raise AppException( code="CLOUD_ROLE_NEEDS_ADMIN", message=( f"角色 {role_key} 为高危角色,只能由 IT / 云管理员授予。" "请联系管理员。" ), status_code=403, ) # Driver-level shape pre-flight (mirror of submit_access_request): # refuse a role_key grant_access would reject before persisting the # active row + dispatching, so the owner gets the error inline # rather than the grant landing back in ``pending`` with a toast. await _preflight_request_role_shapes(self.db, scope, [role_key]) person = await self.db.get(IdentityPerson, uuid.UUID(person_id)) if person is None or person.deleted_at is not None: raise NotFoundError("IdentityPerson", person_id) # Leaver guard (2026-07-05): a scope owner (or the MCP dual) must not # grant cloud IAM to a colleague who is leaving/left — deleted_at is # still NULL for a leaver_pending/left person, so the None/deleted # check above does not catch them, and the UI's leaver filter is a # display-only guard the API/MCP path bypasses. Granting fresh access # to someone offboarding is actively re-opening access the leaver # loop is tearing down. if person.employment_status in ("left", "leaver_pending"): raise AppException( code="PERSON_NOT_ELIGIBLE_LEAVER", message=( f"{person.display_name or person.work_email} 正在离职或已离职," "不能被授予云访问权限。" ), status_code=409, ) # Reuse the shared resurrection helper so a re-add on a row # whose prior owner-add was later revoked / expired / disabled # mutates the existing row in-place (preserving id + # created_at) instead of triggering the unique-key 500 the # employee request flow used to hit. The dispatch step below # is identical in both branches. # ``person_label`` is interpolated via the callable not via # ``.format()``, so a colleague named ``A {b} C`` doesn't crash # the helper's downstream call. person_label = person.display_name or person.work_email grant, _was_resurrection = await self._reuse_or_create_grant_row( scope=scope, person=person, role_key=role_key, role_display_name=role_display_name, new_detail={"direct_add": True}, request_group_id=None, granted_by_person_id=str(actor_user_id), now_dt=datetime.now(UTC), conflict_message_builder=lambda status, _role: ( f"{person_label} already has an {status} grant for that role" ), ) await self.db.flush() return await self._dispatch_grant( grant=grant, scope=scope, ttl_days=ttl_days, actor_user_id=actor_user_id, trace_id=trace_id, action="cloud.grant.owner_add", force_perpetual=is_owner_self_add, ) # ── Dispatch (shared between approve and owner-direct) ── async def _resolve_request_approval_recipients( self, scope: CmdbCloudScope, requires_admin: bool = False, ) -> list[IdentityPerson]: """Who gets the ``[待审批]`` notification for a new access request. The recipients MUST be people who can actually APPROVE the request — paging someone who'll hit a 403 on click is pure noise. * **Owned scope, normal-tier request** → the scope owner (one approver, unchanged). The owner can approve normal-tier roles. * **Owned scope, high-tier request** (``requires_admin`` — any requested role is elevated/restricted): such a request can only be approved by a cloud admin (gate in ``_approve_grant_locked``). If the owner is themselves a cloud admin, page them (minimal noise); otherwise route to the cloud-admin team — paging a non-admin owner would dump an un-actionable card on them that 403s (``CLOUD_APPROVAL_NEEDS_ADMIN``) when they try to approve. * **Intentionally-ownerless scope** (GCP billing account) → every active cloud admin (``can_manage_cloud`` = global_admin / cloud_admin / legacy it_admin). The approval authority for these scopes IS the cloud-admin team — mirrored by ``approve_grant``'s router gate ``_require_scope_owner_or_admin`` which, with no owner, admits only ``can_manage_cloud``. Without this fan-out a billing-account request would page nobody (and the previous single-owner path would crash on ``uuid.UUID(str(None))``). Departed people (``left`` / ``leaver_pending``) and rows without a ``work_email`` are filtered out — they cannot action the request. """ if scope.owner_person_id: owner = await self.db.get( IdentityPerson, uuid.UUID(str(scope.owner_person_id)), ) owner_actionable = ( owner is not None and owner.work_email and owner.employment_status not in ("left", "leaver_pending") ) if owner_actionable: assert owner is not None # for the type-checker # Normal-tier: owner approves directly. High-tier: only a # cloud authority can approve, so page the owner only if # they hold a perpetual admin role (cloud/global admin) — # else fall through to the admin fan-out. if not requires_admin or await self._is_perpetual_grant_holder(owner.id): return [owner] # high-tier + non-admin owner → cloud-admin fan-out below elif not requires_admin: # Owner gone (left / no work_email) on a normal-tier request: # preserve prior behavior — no auto-page, the request sits # PENDING for an admin to find. (High-tier with an absent # owner falls through to the admin fan-out below so it still # reaches an approver.) return [] # Cloud-admin fan-out: an intentionally-ownerless scope, OR an owned # scope whose request needs cloud-admin authority the owner can't # provide. Same role tuple the inactivity sweep uses for its # privileged-role exemption (kept in lock-step). admin_role_names = ("it_admin", "global_admin", "cloud_admin") rows = await self.db.execute( select(IdentityPerson) .join(AuthUserRole, AuthUserRole.user_id == IdentityPerson.id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where( AuthRole.name.in_(admin_role_names), IdentityPerson.deleted_at.is_(None), IdentityPerson.employment_status.notin_( ("left", "leaver_pending"), ), IdentityPerson.work_email.isnot(None), ) .distinct() ) recipients = list(rows.scalars().all()) if not recipients: # No active cloud admin to notify — the request lands in a queue # nobody is paged about. It still persists as PENDING and an # admin can find it, but log loudly so ops notices the missing # approval authority (e.g. all cloud admins departed / lack a # work_email). Fires for an ownerless scope, or an owned scope # whose high-tier request the owner can't approve. logger.warning( "cloud access request on scope %s resolved to ZERO approval " "recipients (requires_admin=%s) — no active cloud admin to " "notify; request will sit PENDING until an admin reviews the " "queue", scope.id, requires_admin, ) return recipients async def _is_perpetual_grant_holder(self, person_id: uuid.UUID) -> bool: """Single-person form of the PERPETUAL predicate. Delegates to :func:`load_perpetual_person_ids` rather than running its own role join. It used to do the latter, and that is precisely the shape of drift this whole area was rewritten to remove on 2026-07-29: two independent implementations of "is this grant perpetual?" are two things a future author can widen only one of. Now there is one query in one place, and adding a role to the perpetual set is a one-line change that every caller — writers and sweepers, single-person and batch — picks up together. The set is a handful of people, so materializing it to answer one membership test is cheaper than the correctness risk of a second copy. """ return str(person_id) in await load_perpetual_person_ids(self.db) async def _dispatch_grant( self, *, grant: CmdbCloudAccessGrant, scope: CmdbCloudScope, ttl_days: int | None, actor_user_id: uuid.UUID, trace_id: str, action: str, silent: bool = False, force_perpetual: bool = False, ) -> LifecycleGrantOutcome: # ``silent`` suppresses the "[已开通] X 已生效" member notification. # Used by the auto-Owner-grant backfill path (cron # ``cmdb.auto_owner_grant_backfill``) — the people the backfill # writes rows for ALREADY have cloud-side Owner (their grants were # missed only because the original FastAPI BackgroundTask silently # dropped them, or because the scope was discovered via sync rather # than created through a scope_request). Sending them an # "[已开通]" card today would be a misleading lie ("we just opened # access for you" when in fact nothing changed cloud-side). # See PR #671 backfill incident (2026-06-02). # # ``force_perpetual`` writes ``expires_at = NULL`` regardless of # the cloud-admin gate / TTL math. Same backfill rationale: the # owner has been Owner since the scope existed; throwing a default # 90-day TTL on the bookkeeping row would kick them into the # renewal-nag flow (14/7/1-day reminders, then auto-revoke) for a # role they de facto held for months. NULL keeps these rows out of # ``sweep_expiring_grants`` entirely (it filters on # ``expires_at IS NOT NULL``). person = await self.db.get(IdentityPerson, uuid.UUID(str(grant.person_id))) if person is None: raise NotFoundError("IdentityPerson", str(grant.person_id)) # Cloud/global admins get perpetual grants. ``expires_at`` stays # NULL; the expiry sweeper's # ``expires_at IS NOT NULL`` filter (already in place) skips # them naturally without per-grant flag work. is_perpetual_holder = await self._is_perpetual_grant_holder(person.id) default_ttl = await _setting_int(self.db, "cloud.grant_default_ttl_days", 90) # Coerce each TTL candidate to a positive int (or None). ``ttl_days`` # and ``scope.default_grant_ttl_days`` are bounded ge=1 at the HTTP # surface, but ``requested_ttl_days`` lives in free-form ``detail`` # JSON — an MCP/future writer could land ``0`` (silently swallowed # by the old ``or`` chain → 90-day default) or a non-int string # (would crash ``int(effective_ttl)`` below, post-commit). Validate # so a bad candidate is skipped rather than mis-applied or fatal. def _pos_ttl(v: object) -> int | None: try: iv = int(v) # type: ignore[arg-type] except (TypeError, ValueError): return None return iv if iv > 0 else None effective_ttl = ( _pos_ttl(ttl_days) or _pos_ttl((grant.detail or {}).get("requested_ttl_days")) or _pos_ttl(scope.default_grant_ttl_days) or default_ttl ) if is_perpetual_holder or force_perpetual: expires_at: datetime | None = None else: # Store naive UTC to match the column's round-trip # representation (MySQL/TiDB strips tz). With # ``expire_on_commit=False`` the router serializes this same # in-memory object after commit; an aware value here forced # the serializer into a naive/aware subtraction crash. expires_at = datetime.now(UTC).replace(tzinfo=None) + timedelta( days=int(effective_ttl), ) # Durability: persist the PENDING grant row (and the prior # state-flip writes from approve_grant / owner_add_member / # batch approve) BEFORE the cloud-side call. Without this, a # crash between a successful ``driver.grant_access`` and the # outer route's commit() leaves the cloud-side grant durably # in place but rolls back the Helix-side row + audit, opening # a "ghost grant" inconsistency window. PR #190 closed this for # ``trigger_sync`` / ``trigger_sync_all`` / ``bulk_execute`` / # ``scope_request approve`` — this is the matching guard for # every grant-dispatch caller. await self.db.commit() integration, driver = await _load_driver_for_scope(self.db, scope) principal_not_found: CloudPrincipalNotFoundError | None = None try: _require_verified_gcp_scope_boundary(integration, scope) result = await driver.grant_access( CloudAccessContext( principal_email=person.work_email or "", principal_display_name=person.display_name, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), role_key=grant.role_key, role_display_name=grant.role_display_name, ttl_days=int(effective_ttl), # Tencent CAM uses a separate local sub-user rather than # Google SSO. Its API-key flow requires a verified mobile # number, so pass the HRIS-owned contact value only to the # IAM-mode driver; other drivers simply ignore ``extra``. extra={ "mobile_phone": person.mobile_phone, "mobile_country_code": ( person.mobile_country_code or person.country ), }, ), ) except CloudPrincipalNotFoundError as e: # Deterministic: the principal doesn't exist at the provider # IdP (deleted Google account, no Identity Center user, …). # Do the same durable failure bookkeeping as a generic # driver failure below, then raise a typed 409 instead of # returning PENDING — a blind retry with the same principal # can never succeed, and the blanket ``except Exception`` # here previously flattened this signal into an # every-10-min backfill-cron retry storm. logger.warning( "driver grant_access: principal not found for grant=%s " "scope=%s: %s", grant.id, scope.id, e, ) principal_not_found = e result = CloudGrantResult(ok=False, error_message=str(e)) except Exception as e: # noqa: BLE001 logger.exception("driver grant_access raised") result = CloudGrantResult(ok=False, error_message=str(e)) finally: try: await driver.close() except Exception: # noqa: BLE001 pass warning: str | None = None if not result.ok: grant.detail = { **(grant.detail or {}), "last_error": result.error_message, # Machine-readable marker so admin UI / audit queries can # tell "retry might work" from "principal must be fixed # first" without parsing the error string. Cleared by the # success path on a later (post-fix) dispatch. **({"principal_not_found": True} if principal_not_found else {}), } await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action=action, status="failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "error": result.error_message, **( {"reason": "principal_not_found"} if principal_not_found else {} ), }, ) # Persist the failure audit row + grant.detail update so a # post-dispatch route crash doesn't roll back the # forensic record. The PENDING row stays in PENDING so # admin retry is safe. await self.db.commit() if principal_not_found is not None: # Typed 409 AFTER the durable bookkeeping: HTTP callers # (approve / owner-add / renew) surface it via the # standard AppException handler; ``_post_provision_ # finalize`` catches it and stamps the backfill-cron # backoff marker. The row stays PENDING (not a terminal # status) so a restored account heals on the next # post-backoff retry. raise GrantPrincipalNotFoundError( str(principal_not_found), ) from principal_not_found return LifecycleGrantOutcome( grant_id=str(grant.id), status=GRANT_STATUS_PENDING, driver_result=result, warning=result.error_message, ) grant.status = GRANT_STATUS_ACTIVE grant.expires_at = expires_at grant.granted_at = datetime.now(UTC) grant.external_ref = result.external_ref grant.last_notified_stage = None # Scrub AK/SK — deliver them once via the outcome.detail then # store only a redacted reference. Never persist access keys in # plain DB. Round-20 audit (A1F12): switched from denylist to # allowlist. Previous shape only filtered out # ``(secret_access_key, access_key_id, secret)`` so a future # driver returning ``session_token`` / ``private_key`` / # ``client_secret`` / ``password`` / ``service_account_key`` # would silently durably store the secret in # ``cmdb_cloud_access_grants.detail`` and surface it in the # admin grant-detail tab. The allowlist below names the few # fields the UI / audit actually need; everything else stays # out of DB regardless of how it was named. _ALLOWED_EXTERNAL_DETAIL_KEYS = frozenset({ "account_id", "principal_id", "principal_email", "principal_name", "external_principal_id", "external_principal_arn", "role_arn", "role_name", "external_id", "subscription_id", "tenant_id", "resource_group", "resource_group_name", "account_arm_id", "user_arn", "user_name", "iam_user_name", "endpoint", "region", "location", "console_url", "access_key_id", # public half of an AK/SK pair — fingerprint only # AliCloud CloudSSO grant detail (PR #565): "access_configuration_id", "access_configuration_name", "target_id", "target_type", }) persisted_detail = {**(grant.detail or {}), "last_error": None} # A restored / re-provisioned account dispatched fine — drop the # stale principal-not-found marker a prior failed attempt wrote. persisted_detail.pop("principal_not_found", None) if result.detail: persisted_detail["external_detail"] = { k: v for k, v in result.detail.items() if k in _ALLOWED_EXTERNAL_DETAIL_KEYS } grant.detail = persisted_detail if result.already_exists: warning = "grant already existed at provider — adopted" await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action=action, status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "external_ref": result.external_ref, "expires_at": ( expires_at.isoformat() if expires_at else None ), # Track the PERSISTED expiry semantics, not just the # cloud-admin gate: ``force_perpetual`` (backfill path) # also writes expires_at=NULL for a non-cloud-admin owner. # the role predicate alone mislabels those backfill rows # as non-perpetual, so a forensic query on detail.perpetual # would miss them (review-sweep #11/#18). "perpetual": expires_at is None, "already_exists": result.already_exists, }, ) # Durably persist the ACTIVE flip + success audit BEFORE we # fire any further side effects (member notification). The # cloud-side grant is already live; if the route handler dies # between here and its trailing commit() the success state is # still recorded, eliminating the "ghost grant" window. await self.db.commit() # Tencent CAM provides no reset URL. A newly-created CAM user does # return a generated, first-login-reset password; send it directly # by email only (never Feishu / inbox) and persist it solely in the # existing masked, non-retryable secret-notification record. is_new_tencent_cam_user = ( integration.provider == "tencent" and bool((result.detail or {}).get("created")) ) cam_name = str((result.detail or {}).get("user_name") or "") initial_password = str( (result.detail or {}).get("initial_console_password") or "", ) initial_credential_email_sent = False if initial_password and is_new_tencent_cam_user: # Flips once a terminal outbox row exists for this credential # (ok=True *or* ok=False). Anything that leaves it False means the # generated password is live on the CAM user with no durable trace # anywhere — the dead-letter fallback at the end of this block is # that trace. credential_mail_recorded = False credential_mail_error: str | None = None try: from app.integrations.driver_base import EmailDriver, EmailMessage from app.integrations.service import build_primary_driver email_driver = await build_primary_driver("email", session=self.db) if not isinstance(email_driver, EmailDriver): raise RuntimeError("primary email integration is not an EmailDriver") # Copy comes from the ``tencent_cam_initial_password`` # notification template (all three languages) rather than the # Chinese-only literals this used to carry — a Japanese or # English-reading hire received the Chinese credential mail. # Rendered here rather than through ``dispatch`` because this # send bypasses the pipeline entirely (see below); builtin-only, # so an admin's ``platform_settings`` edit can never break the # one mail that carries a password. lang = resolve_lang(person) fmt = { "person_name": person.display_name or person.work_email, "cam_name": cam_name, "main_account_id": scope.external_id, "temp_password": initial_password, } subject = ( pick_localized(_CAM_MAIL_TEMPLATE, "subject", lang) or "" ).format(**fmt) body_text = ( pick_localized(_CAM_MAIL_TEMPLATE, "body_text", lang) or "" ).format(**fmt) if not subject or not body_text: # ``pick_localized`` 返回 None 时上面的 ``or ""`` 会把它 # 变成空串 —— 一封没有正文的密码信照样发得出去, 收件人 # 拿到一个空信封, 而 outbox 里那条记录还是 ok=True。 # (模板整个被删 / 字段名被改都会走到这里。) 当成发送失败, # 走和退信同一条路径: warning + 下面那条飞书登录指引改用 # 「凭证未送达, 请联系 IT 重发」措辞, 由 IT 线下交付 —— # 交付所需的凭证由本块末尾那条 ok=False 的死信 outbox 行 # 留档 (raise 走到 ``credential_mail_recorded`` 判定)。 raise RuntimeError( "tencent_cam_initial_password template rendered empty", ) email_result = await email_driver.send(EmailMessage( to=[person.work_email], subject=subject, body_text=body_text, )) # Decide "did the mail go out" the moment ``send`` returns, # BEFORE any bookkeeping. This used to be derived after # ``record_external_email``, so a *recording* failure (FK, # poisoned txn, connection timeout) was caught by the outer # ``except`` with the flag still False — and the Feishu guide # below then told a new hire the credential "未能送达，请联系 # IT 重新发送" about a mail already sitting in their inbox. IT # resets the password, the delivered one dies, both sides # burn a round trip. Delivery truth comes from the driver; # the outbox row is a record OF it, never its source. initial_credential_email_sent = bool(email_result.ok) error_text = (email_result.error_message or "").replace( initial_password, "[REDACTED]", ) or None if not initial_credential_email_sent: credential_mail_error = ( error_text or "email driver returned ok=False" ) warning = "Tencent CAM account created but credential email failed" # Best-effort, but SAVEPOINT-isolated: a bare swallowed flush # failure deactivates the whole transaction, and the Feishu # login guide dispatched right below still needs it. # (``best_effort`` logs ``format_exception_for_log(..., 300)``; # a DBAPI error's password-bearing ``[parameters: …]`` tail # sits far past that, behind the multi-hundred-char # ``notification_outbox`` INSERT statement.) async with best_effort( self.db, label="cloud: record Tencent CAM credential mail", log=logger, ): await self.notify.record_external_email( recipient=person, recipient_email=person.work_email, subject=subject, body_text=body_text, ok=initial_credential_email_sent, template_key="tencent_cam_initial_password", error=error_text, related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, ) # True even when that savepoint rolled back: the fallback row # below is the same INSERT into the same table and would fail # for the same reason. Re-attempting it buys nothing. credential_mail_recorded = True except Exception as exc: # noqa: BLE001 credential_mail_error = format_exception_for_log(exc).replace( initial_password, "[REDACTED]", ) logger.warning( "Tencent CAM initial credential delivery failed: %s", credential_mail_error, ) warning = "Tencent CAM account created but credential email failed" if not credential_mail_recorded: # No mail went out AND no outbox row exists — the empty-template # guard above, a missing/mistyped email integration, or ``send`` # itself raising. The generated password is already set on the # CAM user and this outbox row is its ONLY persistence (see the # block comment above ``is_new_tencent_cam_user``), so without # this the credential is simply gone: ops gets a worker warning # and nothing queryable, and IT's only remaining move is a # password reset. Write the terminal ok=False row ourselves, # carrying the credentials in the body exactly like the # delivered variant does — same readership, same masked # notification log, same bounded exposure (the password forces # a change at first login) — so IT can finish the delivery. # Copy is deliberately in-code and zh-only: this row is an # IT-facing dead letter, and the reason we are here at all may # be that the template no longer renders. async with best_effort( self.db, label="cloud: dead-letter Tencent CAM credential", log=logger, ): await self.notify.record_external_email( recipient=person, recipient_email=person.work_email or "", subject=( "[未送达] 腾讯云 CAM 初始登录凭证 - " f"{person.display_name or person.work_email or person.id}" ), body_text=( "这封凭证信未能发出，以下凭证仅留档于此，" "请 IT 线下交付给本人。\n\n" f" · 收件人: {person.work_email or '(无公司邮箱)'}\n" f" · 子用户名: {cam_name or '(未知)'}\n" f" · 主账号 ID: {scope.external_id}\n" f" · 临时密码: {initial_password}\n" " · 登录入口: https://cloud.tencent.com/login" "（选择“子账号登录”）\n\n" "首次登录后必须立即修改密码。" ), ok=False, template_key="tencent_cam_initial_password", error=( credential_mail_error or "credential mail was never sent" ), related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, ) # A CAM account is not reachable through Google SSO. The ordinary # grant card only says "go to the cloud console", which leaves a # first-time user without the Tencent entry point or the required # account identifiers. Send a separate Feishu-only login guide. It # deliberately contains no password: the one-time credential stays # in the direct work-email delivery above. if is_new_tencent_cam_user and person.work_email and not silent: credential_line = ( "初始密码已单独发送至你的公司邮箱；首次登录后必须立即修改。" if initial_credential_email_sent else ( "初始登录凭证未能送达你的公司邮箱，请联系 Helix IT 管理员" "重新发送。" ) ) try: await self.notify.dispatch( NotificationEnvelope( recipient=person, title="[账号已创建] 请登录腾讯云完成首次设置", body_text=( "你的腾讯云 CAM 子用户已创建。请按以下信息登录：\n\n" "1. 点击下方“登录腾讯云”，选择“子账号登录”\n" f"2. 主账号 ID：{scope.external_id}\n" f"3. 子用户名：{cam_name or '请查看公司邮箱'}\n" f"4. {credential_line}" ), channels=["feishu"], card_actions=[ CardAction( label=LocalizedText( zh="登录腾讯云", en="Sign in to Tencent Cloud", ja="Tencent Cloud にログイン", ), url="https://cloud.tencent.com/login", style="primary", ), ], related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, ), trace_id=trace_id, ) except Exception as exc: # noqa: BLE001 logger.warning("notify Tencent CAM login guide failed: %s", exc) # Notify the member — unless the caller asked for silent (backfill). if person.work_email and not silent: expiry_line = ( f"有效期至: {expires_at:%Y-%m-%d}\n\n" "请到云端控制台登录使用。过期前 14/7/1 天会提醒续期。" if expires_at is not None else ( "有效期: 永久 (Cloud Admin 永久授权)\n\n" "请到云端控制台登录使用。" ) ) try: await self.notify.dispatch( NotificationEnvelope( recipient=person, title=( f"[已开通] {scope.display_name or scope.external_id} " f"的云访问已生效" ), body_text=( f"云账号: {integration.name}\n" f"Scope: {scope.display_name or scope.external_id}\n" f"角色: {grant.role_display_name or grant.role_key}\n" f"{expiry_line}" ), channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, ), trace_id=trace_id, ) except Exception as e: # noqa: BLE001 logger.warning("notify grant dispatch success failed: %s", e) return LifecycleGrantOutcome( grant_id=str(grant.id), status=GRANT_STATUS_ACTIVE, driver_result=result, warning=warning, ) # ── Renew ────────────────────────────────────────────── async def renew_grant( self, grant_id: str, *, actor_user_id: uuid.UUID, trace_id: str, actor_role_names: list[str] | None = None, ttl_days_override: int | None = None, approved_request_id: str | None = None, ) -> CmdbCloudAccessGrant: """Extend ``expires_at``, never shortening it. Works on ``active`` and ``expired`` grants — reviving an expired one re-dispatches to the provider. ``revoked`` / ``disabled`` grants cannot be renewed; submit a new access request instead. **Authorization.** Since 2026-07-29 employee renewal is not self-service: the holder raises a renewal REQUEST and the scope owner decides it (``renewal_service``). This method is the shared primitive underneath, reachable two ways: * ``approved_request_id`` set — the caller is ``renewal_service.decide_renewal_request`` applying an approved decision. The approval IS the authorization, so no holder / admin check is applied here; the renewal service has already verified the approver's authority and refused self-approval. * ``approved_request_id`` unset — a direct renewal, restricted to the holder themselves or a cloud admin (break-glass, e.g. an on-call engineer locked out mid-incident). Not reachable from the employee UI. ``ttl_days_override`` carries the TTL the approver signed off on; without it the platform default ``cloud.grant_renewal_days`` applies. Concurrency: serialized with ``revoke_grant`` and self via the same Redis advisory lock keyed by ``grant_id``. Otherwise a renew + revoke racing produces two cloud-side calls + two audit rows with last-writer-wins on ``revoked_at`` / ``expires_at``. """ try: async with distributed_lock( f"grant_revoke:{grant_id}", ttl=120, ): return await self._renew_grant_locked( grant_id, actor_user_id=actor_user_id, trace_id=trace_id, actor_role_names=actor_role_names, ttl_days_override=ttl_days_override, approved_request_id=approved_request_id, ) except LockBusy as exc: raise ConflictError( f"Grant {grant_id} is being modified concurrently", ) from exc async def _renew_grant_locked( self, grant_id: str, *, actor_user_id: uuid.UUID, trace_id: str, actor_role_names: list[str] | None = None, ttl_days_override: int | None = None, approved_request_id: str | None = None, ) -> CmdbCloudAccessGrant: grant = await self.db.get(CmdbCloudAccessGrant, uuid.UUID(grant_id)) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) # Defense-in-depth (added 2026-05-06 launch audit): only the # grant holder or a cloud admin may renew directly. The MCP tool # wrapper gates this at the controller; mirroring the check here # keeps a future REST endpoint or refactor from regressing # silently. # # Skipped only on the approval path, where the authority that # matters is the APPROVER's (verified in # ``renewal_service.decide_renewal_request``, which also refuses # self-approval) — the holder is by definition not the actor. if approved_request_id is None and ( str(grant.person_id) != str(actor_user_id) ): from app.core.rbac import can_manage_cloud if not can_manage_cloud(actor_role_names or []): raise AppException( code="FORBIDDEN", message="Only the grant holder or a cloud admin may renew this grant.", status_code=403, ) if grant.status not in (GRANT_STATUS_ACTIVE, GRANT_STATUS_EXPIRED): raise ConflictError( f"Grant status {grant.status!r} cannot be renewed; submit a new request", ) # The approver-signed TTL wins when present; otherwise the # platform default. Clamped to the hard cap so a bad override can # never mint access longer than policy allows, no matter which # caller supplied it. default_renewal_days = await _setting_int( self.db, "cloud.grant_renewal_days", 90, ) renewal_days = ( min(int(ttl_days_override), GRANT_TTL_HARD_CAP_DAYS) if ttl_days_override and int(ttl_days_override) > 0 else default_renewal_days ) # Cloud/global admins renew to NULL (perpetual). For everyone else # this is the standard +90-day bump from NOW. is_perpetual_holder = await self._is_perpetual_grant_holder( uuid.UUID(str(grant.person_id)) ) # Naive UTC — same reason as the approve-dispatch path: the # router re-serializes this object post-commit and must not # hit a naive/aware datetime subtraction. candidate = ( datetime.now(UTC).replace(tzinfo=None) + timedelta(days=renewal_days) ) new_expires: datetime | None if is_perpetual_holder: new_expires = None else: # RENEWAL MUST NEVER SHORTEN. The old code assigned # ``now + renewal_days`` unconditionally, so renewing a grant # that still had MORE than ``renewal_days`` left silently cut # it down. Four grants were actually damaged this way: three # of nancy.wu's 365-day grants on aws-account 470438934515 and # one of yishuai.li's on 604175033442 dropped from ~336 days # remaining to 90 on 2026-07-15 — all four via the # 「全部续期」 kebab item, which fires one renew per active # grant with no confirmation step. The user pressed a button # labelled "renew" and lost 8 months of validity, while the # toast cheerfully reported 「已续期 +90 天」. # # Take the later of (current expiry, now + renewal_days). A # renewal is a floor on remaining validity, never a ceiling. existing = grant.expires_at if existing is not None and existing.tzinfo is not None: existing = existing.astimezone(UTC).replace(tzinfo=None) new_expires = ( candidate if existing is None or existing < candidate else existing ) # True when the row already had at least ``renewal_days`` of # runway, so this call did not move ``expires_at`` at all. The # caller surfaces it so the UI can say 「有效期已长于 90 天,无需 # 续期」 instead of falsely claiming a bump. already_longer = ( not is_perpetual_holder and grant.expires_at is not None and new_expires == grant.expires_at ) previous_status = grant.status previously_expired = grant.status == GRANT_STATUS_EXPIRED grant.status = GRANT_STATUS_ACTIVE grant.expires_at = new_expires grant.last_renewed_at = datetime.now(UTC) grant.last_notified_stage = None # If the grant had expired and been revoked at the provider, we # need to re-dispatch. Flip to active first; if dispatch fails # leave the row expired and surface the warning. if previously_expired: scope = await self.db.get(CmdbCloudScope, uuid.UUID(str(grant.scope_id))) # Round-42 audit: same gap as approve — re-dispatching a # grant onto a closed/deleted scope writes ACTIVE state on # a scope that's already gone. Block at the boundary; the # caller already validated the user, this is the second # guard on the resource. if scope is None or scope.deleted_at is not None: grant.status = GRANT_STATUS_EXPIRED raise NotFoundError("CmdbCloudScope", str(grant.scope_id)) if scope.status != SCOPE_STATUS_ACTIVE: grant.status = GRANT_STATUS_EXPIRED raise AppException( code="CLOUD_SCOPE_NOT_ACTIVE", message=( f"Cannot reinstate a grant on scope {scope.id} — " f"current status is {scope.status!r}. " "Re-submit a new access request on an active scope." ), status_code=409, ) if scope.owner_person_id: outcome = await self._dispatch_grant( grant=grant, scope=scope, ttl_days=renewal_days, actor_user_id=actor_user_id, trace_id=trace_id, action="cloud.grant.renew_reinstate", ) if outcome.status != GRANT_STATUS_ACTIVE: # Dispatch failed — mark expired again and surface. grant.status = GRANT_STATUS_EXPIRED raise GrantDispatchError(outcome.warning or "reinstate failed") # Clear the closure bookkeeping now that the revival is committed # to. Deliberately AFTER the dispatch guards above: every one of # them puts the row back to ``expired``, and a row that is expired # again must keep the ``revoked_at`` / ``revoke_reason`` that # explains WHY — clearing earlier would have laundered a failed # reinstate into a closure with no recorded cause. # # Without this, a revived row stays ACTIVE while still carrying # ``revoked_at`` + ``revoke_reason='expired'`` from the sweep that # closed it. 24 production rows were in exactly that state on # 2026-07-29. Nothing crashes (``ended_at`` / ``ended_reason`` are # derived from ``status``, so the 已结束 tab renders correctly), # but every compliance query that counts # ``revoke_reason='expired'`` — reclaim scorecards, auditor # exports — silently counts live access as reclaimed. if previously_expired and grant.status == GRANT_STATUS_ACTIVE: grant.revoked_at = None grant.revoke_reason = None grant.revoked_by_person_id = None await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.grant.renew", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "previous_status": previous_status, "new_expires_at": ( new_expires.isoformat() if new_expires else None ), "perpetual": is_perpetual_holder, # Distinguishes "we extended you" from "you already had # more runway than a renewal grants, so nothing moved". # Without this the audit trail cannot tell the two apart # and a reader assumes every renew row bumped the clock. "no_change_already_longer": already_longer, "ttl_days": renewal_days, # Which authority this renewal rests on. ``None`` means a # direct holder / cloud-admin break-glass renewal; a value # is the approved renewal request that authorised it, so # forensics can always get from a renewal to the person # who signed it off. "approved_request_id": approved_request_id, }, ) return grant # ── Revoke ───────────────────────────────────────────── async def revoke_grant( self, grant_id: str, *, reason: str, actor_user_id: uuid.UUID, trace_id: str, suppress_driver_error: bool = False, actor_role_names: list[str] | None = None, ) -> CmdbCloudAccessGrant: """Revoke an active grant. Calls driver.revoke_access; marks row revoked + sets ``revoked_at`` regardless of driver success — if the driver returns error and ``suppress_driver_error=False``, raises :class:`GrantDispatchError` so the caller can choose to rollback. Offboarding passes ``suppress_driver_error=True``: the grant row MUST flip to revoked even if Azure / GCP / AWS reject us — the Helix-side invariant is more important than the provider-side one, and the sweep will retry. Concurrency: we serialize concurrent revoke clicks (admin double-click; offboarding sweep + admin manual revoke racing) via a Redis advisory lock keyed by ``grant_id``. Without it, two transactions both pass the terminal-status check, both call the cloud driver (most are idempotent — wasted call), and both write ``cloud.grant.revoke`` audit rows that confuse forensics. """ try: async with distributed_lock( f"grant_revoke:{grant_id}", ttl=120, ): return await self._revoke_grant_locked( grant_id, reason=reason, actor_user_id=actor_user_id, trace_id=trace_id, suppress_driver_error=suppress_driver_error, actor_role_names=actor_role_names, ) except LockBusy: # Another caller is currently revoking; treat as no-op # success so the admin's UI doesn't spuriously error on # the second click. The first caller's audit + DB write # is the source of truth. grant = await self.db.get( CmdbCloudAccessGrant, uuid.UUID(grant_id), ) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) # Pre-launch hardening 2026-05-16 (LS4 P2): mirror the # RBAC check that lives inside _revoke_grant_locked so a # caller without permission can't read back the grant # row just by hitting the endpoint while a legitimate # revoke is in flight. Without this, an employee whose # peer's grant happens to be mid-revoke would receive # 200 + the grant payload (which they're not authorised # to see) instead of 403. if not suppress_driver_error and str(grant.person_id) != str(actor_user_id): from app.core.rbac import ( can_manage_cloud, is_cloud_scope_owner_of, ) authorized = ( can_manage_cloud(actor_role_names or []) or await is_cloud_scope_owner_of( self.db, str(actor_user_id), str(grant.scope_id), ) ) if not authorized: raise AppException( code="FORBIDDEN", message=( "Only the grant holder, scope owner, or a cloud " "admin may revoke this grant." ), status_code=403, ) return grant async def _revoke_grant_locked( self, grant_id: str, *, reason: str, actor_user_id: uuid.UUID, trace_id: str, suppress_driver_error: bool, actor_role_names: list[str] | None = None, ) -> CmdbCloudAccessGrant: grant = await self.db.get(CmdbCloudAccessGrant, uuid.UUID(grant_id)) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) # Defense-in-depth (added 2026-05-06 launch audit, widened # 2026-05-09 round 16): only the grant holder, a cloud admin, # the scope owner, or the offboarding sweep (suppress_driver_ # error=True signals system-context) may revoke. MCP tool # wrapper enforces this at the controller; mirror here so a # future REST endpoint or refactor can't regress silently. # The HTTP route's ``_require_scope_owner_or_admin`` is the # primary gate; this is a secondary check that also accepts # scope owners — necessary because scope owners legitimately # remove other employees' grants on scopes they own (review # cycles, leaver follow-ups). Without the scope-owner branch # below, switching the router's ``suppress_driver_error`` to # False — needed so cloud failures don't masquerade as # successful Helix-side revokes — would 403 every legitimate # scope-owner revoke action. if not suppress_driver_error and str(grant.person_id) != str(actor_user_id): from app.core.rbac import ( can_manage_cloud, is_cloud_scope_owner_of, ) authorized = ( can_manage_cloud(actor_role_names or []) or await is_cloud_scope_owner_of( self.db, str(actor_user_id), str(grant.scope_id), ) ) if not authorized: raise AppException( code="FORBIDDEN", message=( "Only the grant holder, scope owner, or a cloud " "admin may revoke this grant." ), status_code=403, ) if grant.status in (GRANT_STATUS_REVOKED, GRANT_STATUS_EXPIRED): return grant scope = await self.db.get(CmdbCloudScope, uuid.UUID(str(grant.scope_id))) if scope is None: raise NotFoundError("CmdbCloudScope", str(grant.scope_id)) person = await self.db.get(IdentityPerson, uuid.UUID(str(grant.person_id))) if person is None: raise NotFoundError("IdentityPerson", str(grant.person_id)) integration, driver = await _load_driver_for_scope(self.db, scope) try: _require_verified_gcp_scope_boundary(integration, scope) result = await driver.revoke_access( CloudAccessContext( principal_email=person.work_email or "", principal_display_name=person.display_name, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), role_key=grant.role_key, role_display_name=grant.role_display_name, existing_external_ref=grant.external_ref, ), ) except Exception as e: # noqa: BLE001 logger.exception("driver revoke_access raised") result = CloudGrantResult(ok=False, error_message=str(e)) finally: try: await driver.close() except Exception: # noqa: BLE001 pass grant.status = GRANT_STATUS_REVOKED grant.revoked_at = datetime.now(UTC) grant.revoked_by_person_id = str(actor_user_id) if actor_user_id else None grant.revoke_reason = reason # Flag the suppress-driver-error compliance gap explicitly when # the local row is being marked revoked despite a real cloud # failure (5xx, throttle, IAM lock — anything that is NOT # ``already_absent``). Without this flag, the row + audit # disagree quietly: status='revoked' / audit='failure' / # cloud-side still active. ``requires_manual_cloud_revoke`` # gives the next reconciliation pass an explicit handle to # re-attempt instead of trusting the row's ``revoked`` # status. (Round-17 audit.) requires_manual = bool( suppress_driver_error and not result.ok and not result.already_absent, ) grant.detail = { **(grant.detail or {}), "last_error": None if result.ok else result.error_message, "revoke_result": { "already_absent": result.already_absent, "ok": result.ok, }, "requires_manual_cloud_revoke": requires_manual, } await self.audit.log( trace_id=trace_id, # Sweeps / offboarding pass actor_user_id=None — surface that # as actor_type="system" so the audit log doesn't look like # a mystery "user with null id" revoke. actor_type="system" if actor_user_id is None else "user", actor_user_id=actor_user_id, action="cloud.grant.revoke", # Round-42 audit: ``already_absent`` is idempotent success # — the row is correctly flipped to REVOKED and the caller # treats it as success (see the suppression at line 2904). # The prior ``failure`` label here confused forensics and # alert dashboards that count failures by status. status="success" if (result.ok or result.already_absent) else "failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "reason": reason, "driver_ok": result.ok, "driver_error": result.error_message, "already_absent": result.already_absent, }, ) # ``already_absent=True`` is idempotent success — drivers raise # this when the cloud-side principal/grant is already gone (404 # on AWS / GCP / Azure / Aliyun). Treating it as a hard failure # would 502 every legitimate "user revoked, then offboarding # sweeper revoked again" double-fire. Pre-launch hardening # 2026-05-09 (round 16 audit). if not result.ok and not result.already_absent and not suppress_driver_error: # Round-16 review follow-up: write a DURABLE audit row # before raising. The session-scoped ``self.audit.log()`` # call above is rolled back together with the grant.status # mutation when the GrantDispatchError propagates out # through the FastAPI ``get_db`` cleanup, so a forensic # trail of "tried, cloud rejected" would otherwise vanish. # The new row uses an independent session so it survives # the rollback. Best-effort — if the audit write itself # fails, log loudly but do not mask the original cloud # failure (the user still needs to see the 502). try: from app.audit.service import AuditService from app.core.database import async_session_factory async with async_session_factory() as audit_db: await AuditService(audit_db).log( trace_id=trace_id, actor_type=( "system" if actor_user_id is None else "user" ), actor_user_id=actor_user_id, action="cloud.grant.revoke_attempt_failed", status="failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "reason": reason, "driver_error": result.error_message, "scope_id": str(grant.scope_id), "person_id": ( str(grant.person_id) if grant.person_id else None ), }, ) await audit_db.commit() except Exception: # noqa: BLE001 logger.exception( "durable revoke-failure audit write failed for " "grant_id=%s — original cloud error still raises", grant.id, ) raise GrantDispatchError( result.error_message or "unknown revoke failure", ) # Cascade-close any open renewal request against this grant. # # An open renewal must not survive a deliberate closure: the row # would stay approvable, and an owner clicking 「批准」 next week # would silently RESURRECT access that a human, the offboarding # pipeline, or a drift sweep had closed on purpose. Expiry is the # one closure that does NOT cascade — a grant reaching # ``expires_at`` with a renewal pending is the normal case, and # approving it afterwards is exactly the intended revival path # (the expiry sweep passes ``GRANT_REVOKE_REASON_EXPIRED``, which # ``close_open_requests_for_grant`` is never called for). # # Imported inline: ``renewal_service`` imports this module, so a # top-level import here would be circular. if reason != GRANT_REVOKE_REASON_EXPIRED: from app.domains.cmdb.renewal_service import ( CloudRenewalService, ) grant_id_for_log = grant.id # SAVEPOINT, not a bare ``except``: ``close_open_requests_for_grant`` # writes and flushes. Swallowing a failed flush would deactivate the # transaction the REVOKE itself lives in — so "never let renewal # bookkeeping fail a revoke" would achieve the exact opposite, and # the caller's commit would raise instead of closing the access. closed = 0 async with best_effort( self.db, label=f"cascade-close renewal requests for grant {grant_id_for_log}", log=logger, ) as renewal_step: closed = await CloudRenewalService( self.db, ).close_open_requests_for_grant( grant_id=str(grant.id), reason=reason, trace_id=trace_id, ) if not renewal_step.ok: # A surviving open request is caught by the lapse sweep within # RENEWAL_REQUEST_LAPSE_DAYS, and the decide path re-validates # grant status before applying anything. logger.warning( "renewal cascade-close failed for grant %s — revoke " "itself succeeded: %s", grant_id_for_log, renewal_step.reason(), ) elif closed: logger.info( "revoke(%s) cascade-closed %d open renewal " "request(s) on grant %s", reason, closed, grant_id_for_log, ) return grant async def retry_manual_cloud_revoke( self, grant_id: str, *, trace_id: str, ) -> bool: """Re-attempt the cloud-side revoke for a grant whose local row is REVOKED but ``detail.requires_manual_cloud_revoke=True``. Returns ``True`` if the cloud now reports the principal absent (success or already_absent) and the flag was cleared; ``False`` if the driver still fails (flag left set for the next sweep). Called by :func:`sweep_pending_manual_cloud_revokes`. Refuses to run on grants that are not in the expected post-revoke state so a misfiled job_id can't reopen a healthy grant. """ grant = await self.db.get(CmdbCloudAccessGrant, uuid.UUID(grant_id)) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) if grant.status != GRANT_STATUS_REVOKED: # Sweep guard against picking up a grant whose state changed # under us between the discovery query and per-grant retry. return False detail = grant.detail or {} if not detail.get("requires_manual_cloud_revoke"): return True # already cleared, nothing to do scope = await self.db.get(CmdbCloudScope, uuid.UUID(str(grant.scope_id))) if scope is None: raise NotFoundError("CmdbCloudScope", str(grant.scope_id)) # Round-2 review follow-up (SS1 P2-2): if the scope was # decommissioned (status != active) after the original revoke # marked the grant requires_manual_cloud_revoke, the driver # call is unavoidably futile — the cloud sub-account itself # is closed. Clear the flag so the hourly sweep doesn't churn # against it forever, and write a distinct audit row so # forensics can tell "scope closed under us" from "driver # actually succeeded". if scope.status != SCOPE_STATUS_ACTIVE: new_detail = dict(detail) new_detail["requires_manual_cloud_revoke"] = False new_detail["last_error"] = None new_detail["retry_skipped_scope_closed_at"] = ( datetime.now(UTC).isoformat() ) grant.detail = new_detail flag_modified(grant, "detail") await self.audit.log( trace_id=trace_id, actor_type="system", actor_user_id=None, action="cloud.grant.revoke_retry_skip_scope_closed", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "scope_id": str(scope.id), "scope_status": scope.status, "previous_error": detail.get("last_error"), }, ) return True person = await self.db.get( IdentityPerson, uuid.UUID(str(grant.person_id)), ) if person is None: raise NotFoundError("IdentityPerson", str(grant.person_id)) _integration, driver = await _load_driver_for_scope(self.db, scope) try: result = await driver.revoke_access( CloudAccessContext( principal_email=person.work_email or "", principal_display_name=person.display_name, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), role_key=grant.role_key, role_display_name=grant.role_display_name, existing_external_ref=grant.external_ref, ), ) except Exception as e: # noqa: BLE001 logger.exception( "retry_manual_cloud_revoke: driver.revoke_access raised " "for grant_id=%s", grant_id, ) # Round-2 review follow-up (SS2 P2-1): redact driver # exception string before storing in any persisted field. # boto3 / Azure / GCP ClientError stringifications can # embed ARNs, IAM principal names, subscription IDs, signed # URL tokens — same redactor every other audit/error # site already uses (sync.py:495-520, providers/*). from app.domains.cmdb.providers.exceptions import ( redact_provider_error, ) result = CloudGrantResult( ok=False, error_message=redact_provider_error(e), ) finally: try: await driver.close() except Exception: # noqa: BLE001 pass # Round-2 review follow-up (SS2 P2-1): even when the driver # returns ``ok=False`` via its own internal error formatting # (without raising up to the except above), the # ``error_message`` may still embed redactable tokens that # leak into the audit detail / grant.detail JSON. Wrap once # more here as a belt-and-suspenders. from app.domains.cmdb.providers.exceptions import ( redact_provider_error as _redact, ) redacted_error = ( _redact(result.error_message) if result.error_message else None ) # Audit every retry attempt — success OR failure — so the trail # tracks the long-tail of stuck grants. await self.audit.log( trace_id=trace_id, actor_type="system", actor_user_id=None, action="cloud.grant.revoke_retry", status="success" if (result.ok or result.already_absent) else "failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "driver_ok": result.ok, "already_absent": result.already_absent, "driver_error": redacted_error, "previous_error": _redact(detail.get("last_error") or "") or None, }, ) if result.ok or result.already_absent: new_detail = dict(detail) new_detail["requires_manual_cloud_revoke"] = False new_detail["last_error"] = None # Reset the storm-cap bookkeeping — a future re-flag (the # grant can't be re-revoked, but keep the state machine # clean) starts from zero. new_detail.pop("manual_revoke_retry_count", None) new_detail.pop("manual_revoke_escalated_at", None) new_detail["retry_succeeded_at"] = datetime.now(UTC).isoformat() grant.detail = new_detail # Round-1 review follow-up (Review-B P0): JSON columns need # an explicit ``flag_modified`` to enter the dirty set # reliably. Rebinding the attribute with a new dict usually # works via attribute history but the codebase has standardised # on flag_modified for JSON to defend against MutableDict # opt-in drift — see provisioning/service.py:716 + # ai_spending_handlers.py:261-268. flag_modified(grant, "detail") await self.db.flush() return True # Failure: keep the flag, update last_error so the operator UI # surfaces the latest reason rather than a stale one. # Round-2 review follow-up (SS2 P2-1): store the redacted # version so the operator UI doesn't render boto3 raw ARNs. # # Retry-storm cap (2026-06-10, execution-plane PR2): the hourly # sweep used to re-hammer a structurally-broken grant forever — # the May driver-bug storm produced 463 failed attempts over # ~3 weeks with nobody paged until the 7-day aging compliance # incident. Now each failed retry increments a counter; at # ``_MANUAL_REVOKE_ESCALATE_AFTER`` consecutive failures the # grant is ESCALATED: cloud_admin + global_admin get one # notification (deep-linked to the scope), the sweep stops # picking the grant up, and the admin "重试云侧回收" button # (which resets the counter) becomes the resume path. The >7d # aging incident keeps firing independently — escalation is the # fast page, the incident is the compliance backstop. new_detail = dict(detail) new_detail["last_error"] = redacted_error retry_count = int(new_detail.get("manual_revoke_retry_count") or 0) + 1 new_detail["manual_revoke_retry_count"] = retry_count escalated_now = False if ( retry_count >= _MANUAL_REVOKE_ESCALATE_AFTER and not new_detail.get("manual_revoke_escalated_at") ): new_detail["manual_revoke_escalated_at"] = ( datetime.now(UTC).isoformat() ) escalated_now = True grant.detail = new_detail flag_modified(grant, "detail") await self.db.flush() if escalated_now: # Audit BEFORE the notification fan-out (post-review fix # 2026-06-10): if a dispatch inside the fan-out poisons the # session, the durable escalation record is already # flushed — the previous order could lose both the stamp # and the audit while the Feishu DM had already gone out, # re-paging every subsequent tick. await self.audit.log( trace_id=trace_id, actor_type="system", actor_user_id=None, action="cloud.grant.manual_revoke_escalated", status="failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "retry_count": retry_count, "last_error": redacted_error, "scope_id": str(grant.scope_id), }, ) try: await self._notify_admins_manual_revoke_escalated( grant=grant, scope=scope, person=person, last_error=redacted_error, retry_count=retry_count, trace_id=trace_id, ) except Exception: # noqa: BLE001 — page is best-effort logger.exception( "manual-revoke escalation fan-out failed for " "grant=%s — stamp + audit already flushed", grant.id, ) return False async def _notify_admins_manual_revoke_escalated( self, *, grant: CmdbCloudAccessGrant, scope: CmdbCloudScope, person: IdentityPerson, last_error: str | None, retry_count: int, trace_id: str, ) -> None: """One-shot page to the cloud-ops cohort (cloud_admin + global_admin) when a cloud-side revoke has failed ``_MANUAL_REVOKE_ESCALATE_AFTER`` consecutive sweep retries. Fires at most once per grant — the ``manual_revoke_escalated_at`` stamp both gates this call and removes the grant from the sweep, so there is no per-tick re-page. Best-effort: a notification failure must not fail the sweep (the audit row + compliance incident remain the durable trail). """ from app.auth.models import AuthRole, AuthUserRole admin_ids = list((await self.db.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthRole.name.in_( ("global_admin", "it_admin", "cloud_admin"), )) )).scalars().all()) title = ( f"[云侧回收卡死] {person.display_name or person.work_email} " f"在 {scope.display_name or scope.external_id} 的权限 " f"{retry_count} 次自动回收均失败" ) body = ( f"离职/撤销动作在 Helix 侧已落盘,但云侧回收连续 " f"{retry_count} 次失败,自动重试已停止。\n\n" f" · 当事人: {person.display_name} ({person.work_email})\n" f" · 云范围: {scope.display_name or scope.external_id}\n" f" · 角色: {grant.role_display_name or grant.role_key}\n" f" · 最后错误: {(last_error or '(unknown)')[:300]}\n\n" "请在云控制台手动处理后,到 Helix 该云范围「已结束」成员列表" "点「重试云侧回收」确认收敛(按钮会重置计数并立即重试)。" ) notify = NotificationService(self.db) seen: set[str] = set() for aid in admin_ids: key = str(aid) if key in seen: continue seen.add(key) admin_person = await self.db.get(IdentityPerson, aid) if admin_person is None or not admin_person.work_email: continue try: await notify.dispatch( NotificationEnvelope( recipient=admin_person, title=title, body_text=body, template_key="cloud.grant.manual_revoke_escalated", url=f"/admin/cloud/scopes/{grant.scope_id}", channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=grant.id, context={ # Same fallback the title uses. A person with # no display_name (HRIS rows do occur without # one) would otherwise render the template's # "当事人：（addr）" with an empty name. "person_name": ( person.display_name or person.work_email or "" ), "person_email": person.work_email or "", "scope_name": ( scope.display_name or scope.external_id ), "role": ( grant.role_display_name or grant.role_key ), "retry_count": retry_count, "last_error": (last_error or "")[:300], }, ), trace_id=trace_id, ) except Exception: # noqa: BLE001 logger.exception( "manual-revoke escalation notify failed: admin=%s " "grant=%s", aid, grant.id, ) async def admin_retry_cloud_revoke( self, grant_id: str, *, actor_user_id: uuid.UUID, trace_id: str, ) -> dict[str, Any]: """Admin-initiated reset + immediate retry of a stuck cloud-side revoke (the resume path after an escalation page, or an early manual kick before the hourly sweep gets there). Resets ``manual_revoke_retry_count`` / ``manual_revoke_escalated_at`` (re-arming ``_MANUAL_REVOKE_ESCALATE_AFTER`` automated attempts if this retry also fails), then runs one ``retry_manual_cloud_revoke`` under the same ``grant_revoke:{id}`` lock the sweep holds, so a concurrent sweep tick can't double-fire the driver. Post-review fix (2026-06-10): the read-detail → reset → retry sequence ALL happens inside the lock — previously the reset flushed before lock acquisition, so two near-simultaneous admin clicks could interleave (B's stale-snapshot reset overwriting A's fresh retry bookkeeping, then a second driver call). The driver's idempotency made that benign in effect, but the lock's promise should hold for humans too. Returns ``{"cleared": bool, "last_error": str | None}``. """ try: async with distributed_lock( f"grant_revoke:{grant_id}", ttl=120, ): grant = await self.db.get( CmdbCloudAccessGrant, uuid.UUID(str(grant_id)), ) if grant is None: raise NotFoundError( "CmdbCloudAccessGrant", str(grant_id), ) detail = dict(grant.detail or {}) if not detail.get("requires_manual_cloud_revoke"): # Nothing stuck — idempotent no-op so a double-click # or a stale page doesn't error. return { "cleared": True, "last_error": None, "noop": True, } detail.pop("manual_revoke_retry_count", None) detail.pop("manual_revoke_escalated_at", None) grant.detail = detail flag_modified(grant, "detail") await self.db.flush() await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.grant.manual_revoke_admin_retry", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={"scope_id": str(grant.scope_id)}, ) cleared = await self.retry_manual_cloud_revoke( str(grant_id), trace_id=trace_id, ) refreshed = dict(grant.detail or {}) except LockBusy as exc: raise AppException( code="GRANT_REVOKE_IN_PROGRESS", message=( "Another revoke attempt on this grant is in flight; " "retry in a few seconds." ), status_code=409, ) from exc return { "cleared": bool(cleared), "last_error": refreshed.get("last_error"), "noop": False, } # ── Inactivity disable (Stage 2 of sweep_inactive_grants) ── async def disable_grant_due_to_inactivity( self, grant_id: str, *, trace_id: str, expected_updated_at_le: datetime | None = None, ) -> tuple[CmdbCloudAccessGrant, bool]: """Auto-disable a grant on the inactivity sweep's Stage 2. Semantically distinct from ``revoke_grant``: * ``revoke_grant`` is terminal — admin / owner / offboarding decided this access is gone for good. Sets ``status=revoked`` + stamps ``revoked_at`` / ``revoked_by_person_id`` / ``revoke_reason``. Audit: ``cloud.grant.revoke``. * ``disable_grant_due_to_inactivity`` is reversible — the person could legitimately come back, ask the owner to re-grant, and resume work. Sets ``status=disabled`` and leaves the ``revoked_*`` columns NULL. Audit: ``cloud.grant.disable``. Reason is implicitly "inactive" (this entry point only fires from the inactivity sweep). Pre-2026-05-05 the sweep called ``revoke_grant`` and then overwrote ``status`` to ``disabled``, leaving the audit trail saying "revoked / reason=inactive" while the row said "disabled" with ``revoked_at`` filled in — three sources of truth in disagreement. This helper is the single source. ``expected_updated_at_le``: optional in-lock TOCTOU guard. The sweep reads ``updated_at`` outside the per-grant lock, decides "this is stale enough to disable", and then competes for the lock with admin / self-service callers (``renew_grant`` / ``revoke_grant``) on the same key. If a renewal lands in the microseconds between the sweep's read and our lock acquire, the row's ``updated_at`` moves forward; without this guard we would disable a freshly-renewed grant. When provided, the helper aborts (returning the grant unchanged) if the in-lock ``updated_at`` strictly exceeds the value the caller observed. Driver-side errors are ALL suppressed end-to-end: a failure in ``_load_driver_for_scope`` (missing Integration row, unknown driver class, credential decryption fault) OR in ``driver.revoke_access`` itself leaves the Helix row at ``status=disabled`` regardless. The Helix-side invariant matters more: a stuck-active row that the policy decided should be paused is worse than a row paused on Helix with a possibly-still-live cloud-side ACL (the next sweep tick or drift-removed sweep re-attempts the cloud side). Production refusal: raises ``AppException`` with code ``CLOUD_SCOPE_PROD_BLOCKED`` (403) when the grant's scope is a production environment (``PROD_ENVIRONMENTS``). Prod cloud access is not managed by Helix lifecycle automation — the sweep never routes prod groups here, and this guard keeps any future caller from auto-disabling prod either. Returns ``(grant, did_disable)``. ``did_disable=False`` on every early-out path (already terminal, TOCTOU guard fired, lock contended). The sweep uses the boolean to keep ``summary["disabled"]`` / ``summary["groups_disabled"]`` honest — pre-2026-05-05 review the sweep blindly bumped the counters on every helper call, including the no-op early-out paths. """ try: async with distributed_lock( f"grant_revoke:{grant_id}", ttl=120, ): return await self._disable_grant_due_to_inactivity_locked( grant_id, trace_id=trace_id, expected_updated_at_le=expected_updated_at_le, ) except LockBusy: grant = await self.db.get( CmdbCloudAccessGrant, uuid.UUID(grant_id), ) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) return grant, False async def _disable_grant_due_to_inactivity_locked( self, grant_id: str, *, trace_id: str, expected_updated_at_le: datetime | None, ) -> tuple[CmdbCloudAccessGrant, bool]: grant = await self.db.get(CmdbCloudAccessGrant, uuid.UUID(grant_id)) if grant is None: raise NotFoundError("CmdbCloudAccessGrant", grant_id) # Pull fresh row state from DB inside the lock — the session # identity map may hold a stale instance from the sweep's # initial SELECT, and a concurrent ``renew_grant`` / # ``revoke_grant`` that finished between then and now would # otherwise be invisible to us. await self.db.refresh(grant) if grant.status in ( GRANT_STATUS_REVOKED, GRANT_STATUS_EXPIRED, GRANT_STATUS_DISABLED, ): return grant, False if expected_updated_at_le is not None: current_upd = grant.updated_at if current_upd is not None and current_upd.tzinfo is None: current_upd = current_upd.replace(tzinfo=UTC) expected = expected_updated_at_le if expected.tzinfo is None: expected = expected.replace(tzinfo=UTC) if current_upd is not None and current_upd > expected: # Concurrent admin renewal / revoke / other mutation # bumped the row between the sweep's snapshot read and # our lock acquire. Defer — the next sweep tick will # re-evaluate against the new state. return grant, False scope = await self.db.get(CmdbCloudScope, uuid.UUID(str(grant.scope_id))) if scope is None: raise NotFoundError("CmdbCloudScope", str(grant.scope_id)) if _is_prod_environment(scope.environment): # Production cloud access is NOT managed by Helix lifecycle # automation (PR #1961 invariant). The inactivity sweep # skips prod groups upstream; this hard refusal is # belt-and-braces so no FUTURE caller can auto-disable a # prod grant — the ``driver.revoke_access`` below would # mutate a production account, bypassing the prod write # lock (which is router/MCP-scoped, not driver-scoped). # Same typed error as the self-service prod block # (``_refuse_prod_self_service``) so callers/frontends # already translating ``CLOUD_SCOPE_PROD_BLOCKED`` handle # it uniformly. raise AppException( code="CLOUD_SCOPE_PROD_BLOCKED", message=( "Production cloud access is not managed by Helix " "lifecycle automation; the R&D security team owns " "prod access end-to-end." ), status_code=403, ) person = await self.db.get(IdentityPerson, uuid.UUID(str(grant.person_id))) if person is None: raise NotFoundError("IdentityPerson", str(grant.person_id)) # Driver-side failures are ALL suppressed end-to-end so the # Helix-side invariant ("inactivity policy decided → row is # disabled") holds even when the cloud side is misconfigured. # That includes ``_load_driver_for_scope`` itself (missing # Integration row, unknown driver class, credential decryption # failure, …) — pre-2026-05-05 review-pass-2 those propagated # and left the row stuck at ``status=active`` for every sweep # tick until an operator reconciled. Worse silent-state than # "row disabled, cloud-side may still be live; next drift # sweep will catch it." driver = None try: integration, driver = await _load_driver_for_scope(self.db, scope) result = await driver.revoke_access( CloudAccessContext( principal_email=person.work_email or "", principal_display_name=person.display_name, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), role_key=grant.role_key, role_display_name=grant.role_display_name, existing_external_ref=grant.external_ref, ), ) except Exception as e: # noqa: BLE001 logger.exception( "driver/scope load or revoke_access raised during " "inactivity disable", ) result = CloudGrantResult(ok=False, error_message=str(e)) finally: if driver is not None: try: await driver.close() except Exception: # noqa: BLE001 pass now = datetime.now(UTC) grant.status = GRANT_STATUS_DISABLED grant.detail = { **(grant.detail or {}), "last_error": None if result.ok else result.error_message, "disable_result": { "already_absent": result.already_absent, "ok": result.ok, "reason": GRANT_REVOKE_REASON_INACTIVE, "at": now.isoformat(), }, } await self.audit.log( trace_id=trace_id, actor_type="system", actor_user_id=None, action="cloud.grant.disable", status="success" if result.ok else "failure", resource_type="cmdb_cloud_access_grant", resource_id=str(grant.id), detail={ "reason": GRANT_REVOKE_REASON_INACTIVE, "driver_ok": result.ok, "driver_error": result.error_message, "already_absent": result.already_absent, }, ) return grant, True # ── Grant utilities for offboarding / sweeps ───────────── async def list_active_grants_for_person( self, person_id: str, ) -> list[CmdbCloudAccessGrant]: rows = (await self.db.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.person_id == person_id, CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ) )).scalars().all() return list(rows) async def list_owned_scopes_for_person( self, person_id: str, ) -> list[CmdbCloudScope]: rows = (await self.db.execute( select(CmdbCloudScope).where( CmdbCloudScope.owner_person_id == person_id, CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ) )).scalars().all() return list(rows) # ── Alert rules (per-scope daily/monthly cost alerts) ──────── async def set_scope_alert_rules( self, scope_id: str, rules: list[dict[str, Any]], *, actor_user_id: uuid.UUID, actor_role_names: list[str] | None = None, trace_id: str, ) -> CmdbCloudScope: """Replace the scope's alert rules wholesale. Each rule is ``{type, threshold_usd, cc_person_ids, active}``; fields are normalized + validated below. The rule's ``last_fired`` map is preserved across updates keyed by (type, threshold) — so editing the Cc list doesn't reset the anti-spam state. ``actor_role_names`` is the author's roles when the caller knows them (the HTTP route does — ``user.role_names``, honouring ``X-View-Role`` narrowing); omitted, the STORED roles are read. Used ONLY to derive the cost basis of a brand-new absolute threshold — see the ``measure`` block below. Narrowing therefore lands on the list basis, which is the everyone-visible side. """ scope = await self.db.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or scope.deleted_at is not None: raise NotFoundError("CmdbCloudScope", scope_id) # Build the existing-fired index keyed primarily by # (type, threshold_pct) when present and falling back to # (type, threshold_usd) — same key shape used at write time # below. This lets the admin retune `budget_usd_monthly` (which # changes the derived threshold_usd of a percentage rule) # without resetting `last_fired` and double-firing the same # monthly alert. existing_fired: dict[str, dict[str, bool]] = {} # ``measure`` marker preservation (2026-07-28 dual-measure # cutover): the conversion script stamps ``measure=="list"`` on # rules whose ``threshold_usd`` it re-denominated to the # pre-discount list basis; the billing evaluator compares each rule # against the aggregate in that rule's own basis. The API input # model (``AlertRuleInput``) doesn't expose ``measure``, so a # wholesale replace from the admin UI would silently strip it and # flip evaluation back to net against list-denominated thresholds. # Preserve it across edits keyed the same way as ``last_fired`` — # an UNCHANGED absolute rule keeps its stamp; a rule whose threshold # the admin retunes is a NEW identity and gets a freshly DERIVED # stamp (see the ``measure`` block below), because the new number is # denominated in whatever basis its author was reading. # # Trust boundary (2026-07-29 review, F14): the stamp is accepted # ONLY from the DB side (this map) or DERIVED from the scope's # ``budget_measure`` for percentage rules — never from the caller's # ``rules`` payload. ``AlertRuleInput`` happens to strip an # attacker-supplied ``measure`` today, but a schema field added # later must not silently become a way for a scope OWNER to # re-denominate their own alert thresholds. # # The sibling write that CAN move a basis (``apply_scope_budget``, # reached from ``PATCH /cmdb/scopes/{id}`` and the open API) is gated # separately: naming the scope's existing basis is open to the owner, # CHANGING it requires ``can_manage_cloud``. So this route is not the # last line of defence for that invariant — but it remains the only # place a per-rule stamp could be forged directly, and it must keep # refusing. existing_measure: dict[str, str] = {} # "Did a rule with this identity already exist?" — NUMERICALLY keyed, # unlike the two maps above whose string keys are built from the raw # JSON value (``250`` and ``250.0`` are different keys there). Only # the derivation of a BRAND-NEW rule's stamp reads this set, and there # a false "new" would re-denominate an existing threshold, which is # the one thing that must not happen (ADR 2026-07-29 #3). existing_identities: set[str] = set() for r in scope.alert_rules or []: pct_existing = r.get("threshold_pct") if pct_existing is not None: key = f"{r.get('type')}:pct:{pct_existing}" else: key = f"{r.get('type')}:{r.get('threshold_usd')}" existing_fired[key] = r.get("last_fired") or {} if r.get("measure") in ("list", "net"): existing_measure[key] = r["measure"] try: existing_identities.add( f"{r.get('type')}:pct:{pct_existing}" if pct_existing is not None else f"{r.get('type')}:{float(r.get('threshold_usd') or 0)}" ) except (TypeError, ValueError): continue # Cache the scope's current monthly budget so percentage rules # can be materialised into an absolute threshold below. None / # 0 is treated as "no budget set" — a percentage rule is # rejected (we have nothing to multiply against). scope_budget = ( float(scope.budget_usd_monthly) if scope.budget_usd_monthly else 0.0 ) # Resolved at most once, and only if some rule actually needs it. author_reads_invoice: bool | None = None normalized: list[dict[str, Any]] = [] for r in rules: rule_type = str(r.get("type") or "").strip() if rule_type not in ("daily", "monthly"): raise AppException( code="CLOUD_ALERT_BAD_TYPE", message=f"Alert type must be 'daily' or 'monthly', got {rule_type!r}", status_code=400, ) # Two ways to set the threshold: # 1. ``threshold_pct`` — % of scope.budget_usd_monthly. # We materialise into ``threshold_usd`` immediately so # the alert evaluator hot path stays simple (no # percentage math at billing-sweep time). # 2. ``threshold_usd`` — absolute. Survives even when the # scope has no monthly budget set. # If both are supplied, the percentage wins (it's the more # admin-friendly intent — "always 50 % of whatever I budget # this month for"). The original pct is kept on the rule so # the UI can render "50 % of budget" instead of bare USD. pct_raw = r.get("threshold_pct") pct: float | None = None if pct_raw is not None: try: pct = float(pct_raw) except (TypeError, ValueError) as e: raise AppException( code="CLOUD_ALERT_BAD_THRESHOLD", message="threshold_pct must be a number", status_code=400, ) from e if pct <= 0: raise AppException( code="CLOUD_ALERT_BAD_THRESHOLD", message="threshold_pct must be positive", status_code=400, ) if scope_budget <= 0: raise AppException( code="CLOUD_ALERT_BUDGET_REQUIRED", message=( "Cannot use a percentage threshold on a scope " "with no monthly budget — set budget_usd_monthly " "first or switch this rule to an absolute USD " "threshold." ), status_code=400, ) threshold = round(scope_budget * pct / 100.0, 2) else: try: threshold = float(r.get("threshold_usd") or 0) except (TypeError, ValueError) as e: raise AppException( code="CLOUD_ALERT_BAD_THRESHOLD", message="threshold_usd must be a number", status_code=400, ) from e if threshold <= 0: raise AppException( code="CLOUD_ALERT_BAD_THRESHOLD", message=( "threshold_usd must be positive (or supply " "threshold_pct instead)" ), status_code=400, ) cc = r.get("cc_person_ids") or [] if not isinstance(cc, list): raise AppException( code="CLOUD_ALERT_BAD_CC", message="cc_person_ids must be a list of person id strings", status_code=400, ) cc_strs = [str(x) for x in cc if x] if len(cc_strs) > 50: raise AppException( code="CLOUD_ALERT_TOO_MANY_CC", message=( "cc_person_ids capped at 50 — large lists are abuse-vector " "for weaponized C-suite alert spam" ), status_code=400, ) # Verify every cc id is a real, active IdentityPerson. Without # this a scope owner could stuff arbitrary UUIDs into the cc # list and either (a) target the C-suite for harassment via # synthetic budget alerts, or (b) leak that a guessed UUID # resolves to a real person via the resulting notification. if cc_strs: rows = await self.db.execute( select(IdentityPerson.id).where( IdentityPerson.id.in_([uuid.UUID(x) for x in cc_strs]), IdentityPerson.deleted_at.is_(None), ) ) live_ids = {str(rid) for (rid,) in rows.all()} missing = [x for x in cc_strs if x not in live_ids] if missing: # Return only the count, never the actual UUIDs. # The prior wording named the invalid UUIDs back # to the caller, which is a UUID-enumeration oracle: # an attacker who already has a UUID from a prior # phish can verify "yes, this person exists" before # pivoting. UUIDs are 122-bit so brute force isn't # feasible — but defense-in-depth. raise AppException( code="CLOUD_ALERT_CC_NOT_FOUND", message=( f"cc_person_ids contains {len(missing)} " f"unknown or inactive person reference(s)" ), status_code=400, ) # Dedup by (rule_type, threshold_pct or threshold_usd) so an # admin retuning a percentage rule's underlying threshold_usd # (e.g. by raising the budget) doesn't reset last_fired and # spuriously re-fire the same monthly alert. dedupe_key = ( f"{rule_type}:pct:{pct}" if pct is not None else f"{rule_type}:{threshold}" ) entry: dict[str, Any] = { "type": rule_type, "threshold_usd": threshold, "cc_person_ids": cc_strs, "active": bool(r.get("active", True)), "last_fired": existing_fired.get(dedupe_key) or {}, } if pct is not None: entry["threshold_pct"] = pct # Measure stamp — DERIVED, never from ``r`` (see the trust # boundary note above; the conversion script writes # ``scope.alert_rules`` directly and does NOT round-trip here). # # Percentage rules: DERIVED from the budget's basis, not # inherited. ``threshold_usd ≡ pct × budget``, so a pct rule's # basis IS the budget's basis by construction. Inheriting instead # (the dedupe key is ``type:pct:N``, deliberately invariant under # budget changes) let an admin "correct" a list-inflated budget # back to the invoice number and keep a ``measure: "list"`` stamp # on a now-net-denominated threshold — #2079 reproduced exactly. # # Absolute rules: # * identity unchanged and stamped → keep the stamp; # * identity unchanged and UNSTAMPED → leave it unstamped. # Unstamped ≡ net (ADR 2026-07-29 #2) is the right reading for # a threshold whose authoring basis is unknown, and writing # that reading down here would silence the ``measure # ambiguity`` warning — the only signal that a list-converted # scope still carries net-evaluated leftovers; # * BRAND-NEW identity → derive it. This used to be a ``.get()`` # miss that silently meant net: a scope owner with no role at # all (``PUT /scopes/{id}/alerts`` is owner-or-admin) created a # net-denominated rule by typing a number they had read off a # LIST-basis surface — the only cost basis they can see. That # is wrong twice over: the threshold fires ~1/discount too # late, and the card it eventually fires carries an # invoice-basis figure to a non-circle recipient. So a new # absolute rule authored by someone outside the invoice circle # is stamped ``list``; inside the circle (who reads both) the # scope's own declared ``budget_measure`` decides, exactly as # for pct rules. Nothing here re-denominates an EXISTING # number, so the "owner may not move a basis" line (ADR # 2026-07-29 #3) is untouched. if pct is not None: measure: str | None = scope_budget_measure(scope) elif dedupe_key in existing_measure: measure = existing_measure[dedupe_key] elif dedupe_key in existing_identities: measure = None # pre-existing, unstamped — leave it so. else: if author_reads_invoice is None: author_reads_invoice = can_view_invoice_measure( actor_role_names if actor_role_names is not None else await _stored_role_names(self.db, actor_user_id) ) measure = ( scope_budget_measure(scope) if author_reads_invoice else MEASURE_LIST ) if measure in (MEASURE_LIST, MEASURE_NET): entry["measure"] = measure normalized.append(entry) scope.alert_rules = normalized scope.updated_at = datetime.now(UTC) await self.audit.log( trace_id=trace_id, actor_type="user", actor_user_id=actor_user_id, action="cloud.scope.update_alerts", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "rule_count": len(normalized), "rules": [ {k: v for k, v in r.items() if k != "last_fired"} for r in normalized ], }, ) return scope async def strip_person_from_alert_cc( session: AsyncSession, person_id: str, *, actor_user_id: uuid.UUID | None, trace_id: str, via: str = "unknown", ) -> int: """Remove ``person_id`` from every ``cmdb_cloud_scopes.alert_rules[*].cc_person_ids``. Hooked into ``LeaverBlockedGuard.flip_to_left`` so a leaver's UUID doesn't linger in cc lists after their ``identity_persons`` row flips to ``status='left'``. Without this hook, a half-year-old leaver still surfaces in the alert config UI as ``未知用户 · 55f3764a…`` because ``/scopes/{id}/pickable-people`` filters out ``status='left'`` and the frontend can't resolve the UUID to a name. Same root-cause class as the AWS-Identity-Center "User not found" residue — the platform mutated the principal's lifecycle but didn't sweep the references that pointed at it. Returns the number of scope rows actually modified. Idempotent — a second call after the leaver is already gone from every cc list is a no-op (returns 0). Safe to call from any code path; failure is logged + swallowed by the caller because cc cleanup is a hygiene side effect, not a correctness invariant — the runtime alert routing already de-duplicates against active employees only via the IdentityPerson row's existence + ``work_email`` non-null checks in :func:`_resolve_billing_alert_recipients`. Writes one ``cloud.alert.cc_leaver_stripped`` audit row per modified scope so investigators can later answer "when did Helix remove this person from this scope's cc list". """ pid = str(person_id).strip() if not pid: return 0 rows = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.alert_rules.is_not(None), ) )).scalars().all() if not rows: return 0 audit = AuditService(session) now = datetime.now(UTC).replace(tzinfo=None) modified = 0 for scope in rows: rules = scope.alert_rules or [] if not isinstance(rules, list): continue new_rules: list[dict[str, Any]] = [] scope_modified = False wiped_per_rule: list[dict[str, Any]] = [] for rule in rules: if not isinstance(rule, dict): new_rules.append(rule) continue cc = rule.get("cc_person_ids") or [] if not isinstance(cc, list) or pid not in [str(x) for x in cc]: new_rules.append(rule) continue # Strip every occurrence (defensive — a corrupted rule # might list the same id twice; we don't want one to # remain). stripped = [str(x) for x in cc if str(x) != pid] wiped_per_rule.append({ "type": rule.get("type"), "threshold_pct": rule.get("threshold_pct"), "threshold_usd": rule.get("threshold_usd"), "before_cc_count": len(cc), "after_cc_count": len(stripped), }) new_rules.append({**rule, "cc_person_ids": stripped}) scope_modified = True if not scope_modified: continue scope.alert_rules = new_rules scope.updated_at = now modified += 1 await audit.log( trace_id=trace_id, actor_type="user" if actor_user_id else "system", actor_user_id=actor_user_id, action="cloud.alert.cc_leaver_stripped", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "person_id": pid, "scope_external_id": scope.external_id, "scope_display_name": scope.display_name, "via": via, "wiped_per_rule": wiped_per_rule, }, ) if modified: await session.flush() return modified async def _resolve_billing_alert_recipients( session: AsyncSession, scope: CmdbCloudScope, *, extra_person_ids: list[str] | None = None, ) -> list[IdentityPerson]: """Cloud-billing alert recipient set. Three sources, unioned and de-duplicated by ``IdentityPerson.id``: 1. **Scope owner** (``scope.owner_person_id``) — always paged. 2. **Billing-class grant holders** — anyone with an *active* grant on this scope whose ``role_key`` / ``display_name`` classifies as billing per the keyword sieve in :mod:`app.domains.cmdb.role_classifier` (Aliyun ``AliyunBillingFullAccess`` / ``AliyunBSSReadOnlyAccess``, Azure ``Cost Management Reader``, GCP ``roles/billing.viewer``, AWS ``Billing*``, Tencent ``QcloudFinanceFullAccess``, custom ``*财务*`` / ``*账单*`` / ``*计费*`` roles). 3. **Owner-curated CC list** (``extra_person_ids``, sourced from the per-rule ``cc_person_ids`` array). Lets the scope owner ping people who don't hold a billing-class cloud grant — finance partners, the team lead, the FinOps Slack-channel owner — without having to provision them a cloud role just to receive the alert. The default routing covers the common case; this is the explicit override knob the owner maintains in the alert config UI. Rows missing ``work_email`` are skipped (no channel to deliver to). Order is owner → billing-role holders (DB order) → CC additions (insertion order); only matters for human-readable downstream logs. **Prod gate (2026-07-28)**: for a production scope (:func:`_is_prod_environment`), the resolved set is narrowed to admin-tier holders (``global_admin`` / legacy ``it_admin`` / ``cloud_admin``) — plain employees must NOT be paged about prod billing, mirroring the platform-wide "prod is owned by the R&D security/IT side, Helix does not push prod ops noise at employees" posture (same posture as the expiry/inactivity sweeps' prod exclusion). Incident 2026-07-27: an ``employee`` holding ``roles/recommender.billingAccountCudViewer`` (billing keyword sieve match) was paged a prod daily-spike card. """ out: list[IdentityPerson] = [] seen_ids: set[str] = set() def _is_leaver(p: IdentityPerson) -> bool: # Round-20 audit (A1F6): leavers must NOT receive billing # alerts. Their corporate email is about to be revoked; the # alert routes critical ops info to a soon-to-be-stranger and # confuses the recipient set. Mirror the same employment_status # check the scope-owner-rebind path uses (line ~458). # Pre-launch hardening 2026-05-16 (LS3 P2): also exclude # soft-deleted persons (``deleted_at IS NOT NULL``); historically # soft-deleted persons retained employment_status='active' and # would still page. return ( p.employment_status in ("left", "leaver_pending") or p.deleted_at is not None ) if scope.owner_person_id: owner = await session.get( IdentityPerson, uuid.UUID(str(scope.owner_person_id)), ) if ( owner is not None and owner.work_email and not _is_leaver(owner) ): out.append(owner) seen_ids.add(str(owner.id)) grant_rows = list((await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, ) )).scalars().all()) for g in grant_rows: if not is_billing_role(g.role_key, g.role_display_name): continue if g.person_id in seen_ids: continue person = await session.get(IdentityPerson, uuid.UUID(str(g.person_id))) if person is None or not person.work_email: continue if _is_leaver(person): continue out.append(person) seen_ids.add(str(person.id)) for pid in extra_person_ids or []: pid_str = str(pid).strip() if not pid_str or pid_str in seen_ids: continue try: person = await session.get(IdentityPerson, uuid.UUID(pid_str)) except (ValueError, TypeError): continue if person is None or not person.work_email: continue if _is_leaver(person): continue out.append(person) seen_ids.add(str(person.id)) if out and _is_prod_environment(scope.environment): admin_ids = { str(row[0]) for row in (await session.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where( AuthUserRole.user_id.in_([str(p.id) for p in out]), AuthRole.name.in_( ("it_admin", "global_admin", "cloud_admin"), ), ) )).all() } dropped = [p for p in out if str(p.id) not in admin_ids] if dropped: logger.info( "cloud.billing_alert prod gate: scope=%s env=%s dropped " "%d non-admin recipient(s): %s", scope.id, scope.environment, len(dropped), ", ".join(p.work_email or str(p.id) for p in dropped), ) out = [p for p in out if str(p.id) in admin_ids] return out async def _invoice_measure_recipient_ids( session: AsyncSession, people: list[IdentityPerson], ) -> set[str]: """Of ``people``, which sit inside the invoice-measure confidentiality circle (:func:`can_view_invoice_measure` — finance / cloud_admin / global_admin, legacy ``it_admin`` canonicalised). The red line (DECISIONS 2026-07-29 #1) is "no non-circle user receives any invoice-basis number on any surface", and a notification is a surface: a net-basis card renders ``CmdbCloudBillingDaily. total_cost_usd`` and names the basis, so a recipient who divides it by the list figure the same scope's ``/billing/daily`` serves them recovers the account's discount rate. The prod gate above already leaves only admin-tier holders on prod scopes; this is the check every OTHER scope needs. Stored roles only (same reasoning as :func:`_stored_role_names`): every circle role is an ``auth_user_roles`` row, and a cron-dispatched card has no request context whose ``X-View-Role`` narrowing could apply. """ if not people: return set() rows = (await session.execute( select(AuthUserRole.user_id, AuthRole.name) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthUserRole.user_id.in_([str(p.id) for p in people])) )).all() by_person: dict[str, list[str]] = {} for uid, role_name in rows: by_person.setdefault(str(uid), []).append(role_name) return { pid for pid, names in by_person.items() if can_view_invoice_measure(names) } async def _period_already_alerted( session: AsyncSession, *, scope_id: str, rule_type: str, period_key: str, ) -> bool: """Has *any* alert already fired for this (scope, type, period)? Pre-claim gate that is **threshold-agnostic** by design. Without this, an admin who lowers a monthly threshold from $500 → $100 (or bumps the daily-spike multiplier from 1.5× → 1.2×) mid-period lets the same scope re-page on the same data: the slot UNIQUE includes ``threshold_usd`` so a different threshold opens a NEW slot, and ``_try_claim_cloud_scope_alert_slot`` happily INSERTs it. From the operator's POV that's one scope, one period, two cards — pure spam, no new signal. Read-only — the actual idempotency anchor remains the UNIQUE constraint on the dedup table; this just lets the per-period "already told you about this scope" guarantee survive admin threshold-tuning within the period. NOTE: deliberately permissive — if the SELECT raises (transient DB issue), we return ``False`` so the downstream claim path can still attempt the INSERT. The UNIQUE constraint is the authoritative line of defence; this is just user-friendliness. """ # SAVEPOINT-wrap the read: if the SELECT raises, ``begin_nested`` # rolls back to the savepoint so the swallow below cannot leave the # shared session in rolled-back-pending state (which would surface # as a PendingRollbackError on the caller's NEXT statement — the # _try_claim INSERT). Without the savepoint, "swallow and continue" # silently poisons the session. try: async with session.begin_nested(): existing = await session.execute( select(CmdbCloudScopeAlertFired.id) .where( CmdbCloudScopeAlertFired.scope_id == scope_id, CmdbCloudScopeAlertFired.rule_type == rule_type, CmdbCloudScopeAlertFired.period_key == period_key, ) .limit(1), ) return existing.first() is not None except Exception: # noqa: BLE001 — fall back to claim-attempt on read error return False async def _audit_cloud_scope_alert_fired( *, scope_id: str, integration_id: str | None, rule_type: str, period_key: str, measure: str, threshold_usd: Decimal, current_usd: Decimal, baseline_usd: Decimal | None = None, multiplier: float | None = None, fx_rate_cny_per_usd: float | None = None, dispatched: bool, recipients_count: int | None = None, ) -> None: """Round-51 R3-P0-3: write an audit row every time a spike/threshold alert decision lands. The dedup table stores ``(scope, rule_type, period_key)`` only — when an operator months later asks "why did this card fire at 3am", the forensic trail is otherwise gone: no baseline, no recipient set, no FX rate at decision time, no multiplier value at decision time. The audit row carries all of that on an independent session so a downstream txn rollback can't erase the audit trail. ``measure`` (``"list"`` / ``"net"``, REQUIRED — same no-default rationale as :func:`_alert_spike`) names the basis the comparison ran on. It is BOTH forensic metadata (the old row didn't say which basis its numbers were in, so "was this figure comparable to the console?" was unanswerable) AND the switch that decides whether the absolute amounts may be persisted at all: **2026-07-29 保密面扫描 — 审计面补漏.** The platform invariant (DECISIONS 2026-07-29 #1) is *no non-circle user may receive any invoice-basis number on any surface*; the circle is ``cost_view.can_view_invoice_measure`` = finance / cloud_admin / global_admin. ``audit_logs`` is read through ``can_access_audit`` (global_admin / **auditor** / **management**) and, per resource, through the ``cmdb``-prefix rule in [audit/router.py](../../audit/router.py) — both strictly wider than the circle. So on the ``net`` basis (the default for unstamped rules, and the PERMANENT state for Azure / Tencent / UCloud / KSyun which have no list column) ``current_usd`` / ``baseline_usd`` / ``threshold_usd`` were invoice-basis figures handed to auditor and management. Cross-surface leak, not just visibility: read ``current_usd`` here, read the same scope+day on the list-basis ``GET /cmdb/billing/daily``, divide → that scope's discount rate. On ``net`` we therefore persist the **ratio, not the amounts**: ``actual_multiple`` = current / baseline is dimensionless, is the number the forensic question actually needs ("how far over baseline did it go"), and cannot be combined with any list-basis absolute to recover a discount. ``threshold_usd`` is withheld with the other two because on the spike path it is itself spend-derived (``baseline × multiplier``), not an admin-entered config value. Withheld, NOT masked-at-read. Three reasons this is a write-side fix: * ``_enrich_audit_logs`` is shared by five HTTP surfaces, the XLSX export and the ``query_audit_logs`` MCP tool. A redactor there would have to guess which ``*_usd`` keys across every domain are confidential cost facts versus asset prices / procurement totals — blunt instrument, platform-wide blast radius. * audit rows are immutable and retained indefinitely, so a confidential number written into one is exposed to every future reader of a table whose gate is *deliberately* wider than the circle and will not be narrowed (auditor governance needs the action ledger — see the predicate test in ``test_scope_alert_audit_invoice_measure_2026_07_29.py``). * nothing is actually lost: the absolute figures live in ``cmdb_cloud_billing_daily`` / ``cmdb_cost_facts``, which DO have correct measure gating, and this row records ``scope_id`` + ``period_key`` + ``measure`` — everything a circle member needs to go read them. ``list`` is the engineering basis everyone may read, so the amounts are kept verbatim there — the redaction must not cost forensic precision on the basis that isn't confidential. READ-SIDE TWIN (defence in depth, 2026-07-30): the detail also stamps :func:`audit_cost_basis_detail` — the per-row ``cost_basis`` marker that :func:`app.core.masking.redact_invoice_basis_amounts` keys on (this action is registered ``COST_BASIS_PER_ROW`` in ``AUDIT_ACTION_COST_BASIS``). On ``list`` rows the marker proves the amounts are releasable to non-circle readers; on ``net`` rows nothing amount-shaped is persisted, so the read-side redactor has nothing to do — the marker just keeps the row self-describing. Best-effort: an audit-write failure must not kill the cron tick. Mirror the ``billing_alerts._record_alert_audit`` pattern. """ # Dimensionless "how far over baseline" — safe on both bases. actual_multiple: float | None = None if baseline_usd is not None and abs(float(baseline_usd)) > 1e-9: actual_multiple = round(float(current_usd) / float(baseline_usd), 4) detail: dict[str, Any] = { # Read-side twin marker (``cost_basis``): see docstring. Must be # stamped on EVERY row so ``COST_BASIS_PER_ROW`` classification # never falls through to the absent-marker (= redact) branch on # a legitimate list-basis row. **audit_cost_basis_detail(measure), "rule_type": rule_type, "period_key": period_key, "measure": measure, "multiplier": multiplier, "actual_multiple": actual_multiple, "fx_rate_cny_per_usd": fx_rate_cny_per_usd, "dispatched": dispatched, "recipients_count": recipients_count, "integration_id": integration_id, } if measure == MEASURE_LIST: detail |= { "threshold_usd": str(threshold_usd), "current_usd": str(current_usd), "baseline_usd": ( str(baseline_usd) if baseline_usd is not None else None ), } else: # Anything that is not exactly ``list`` withholds — a typo or a # future unknown basis lands on the confidential side, matching # ``rule_measure``'s "NULL / unknown reads as net" convention. # # Explicit marker rather than silently-absent keys: a reader who # finds no amount must be able to tell "withheld by policy" from # "the cron failed to compute it". detail |= { "amounts_withheld": True, "amounts_withheld_reason": "invoice_basis_confidential", } try: async with async_session_factory() as audit_session: audit = AuditService(audit_session) await audit.log( trace_id=f"cmdb.scope_alert.{rule_type}.{scope_id}.{period_key}", actor_type="system", action="cloud.scope_alert.fired", status="success", resource_type="cmdb_cloud_scope", resource_id=scope_id, detail=detail, ) await audit_session.commit() except Exception: # noqa: BLE001 — best-effort audit, never raise logger.exception( "cloud.scope_alert.fired audit write failed for " "scope=%s rule=%s period=%s", scope_id, rule_type, period_key, ) async def _try_claim_cloud_scope_alert_slot( session: AsyncSession, *, scope_id: str, rule_type: str, period_key: str, threshold_usd: Decimal, current_usd: Decimal, ) -> bool: """Atomically claim the (scope, rule_type, period, threshold) dedup slot via ``INSERT`` inside ``begin_nested()``. Returns ``True`` when the INSERT succeeded — caller owns this slot and MUST dispatch the alert. Returns ``False`` on :class:`IntegrityError` (UNIQUE-key collision) — another run already fired for this slot; caller MUST skip dispatch. Inverting the old "compute → maybe dispatch → mutate JSON map → pray the outer commit lands" sequence eliminates the dispatch-then-rollback class: * old: alert sent, ``scope.alert_rules[*].last_fired`` mutated, then outer transaction rolled back (unregistered-reconciliation crash, FX-rate hiccup, etc.) → dedup state lost, next cron tick re-fires. * new: dedup row INSERTed first via savepoint; if the outer txn rolls back the dedup row goes with it AND the alert was never dispatched, so the next run will correctly retry. If the dedup row commits but dispatch fails downstream, the operator gets a log line and a missing card — never a *duplicate* card. Same pattern as :func:`apikey.budget_alerts._try_claim_dedup_slot` (PR #298). Mirrored here because the cmdb dedup key shape differs (per-scope vs per-assignment). The savepoint is essential — without it an ``IntegrityError`` from the INSERT would poison the caller's outer transaction. """ try: async with session.begin_nested(): session.add(CmdbCloudScopeAlertFired( id=uuid.uuid4(), scope_id=scope_id, rule_type=rule_type, period_key=period_key, threshold_usd=threshold_usd, current_usd=current_usd, fired_at=datetime.now(UTC), )) except IntegrityError: return False return True # ── Alert-rule cost basis (口径) ────────────────────────────────────── # # ``"net"`` (实付/发票口径) is the historical default every threshold and # budget was entered in; ``"list"`` (业务折前) is the basis the conversion # script re-denominates to and stamps explicitly. NULL / missing / any # unknown value reads as net — the conservative side (a net-denominated # threshold evaluated on net can only be right; the inverse false-fires). MEASURE_NET = "net" MEASURE_LIST = "list" def rule_measure(rule: dict[str, Any]) -> str: """The basis ONE threshold rule's ``threshold_usd`` is denominated in. Per-rule, deliberately NOT per-scope: the whole-group same-measure invariant (DECISIONS 2026-07-28) constrains the DATA ROWS feeding one comparison, not the rule set. A group-wide ``all(...)`` gate let a single unstamped rule — which a *scope owner* can create through ``PUT /cmdb/scopes/{id}/alerts`` — silently flip every still-list- denominated sibling onto the net basis, i.e. thresholds ~2.7× too high that can never be crossed. """ return MEASURE_LIST if rule.get("measure") == MEASURE_LIST else MEASURE_NET def _rule_threshold_usd(rule: dict[str, Any]) -> float | None: """One rule's absolute threshold as a float, or ``None`` if unusable.""" try: thr = float(rule.get("threshold_usd") or 0) except (TypeError, ValueError): return None return thr if thr > 0 else None def rule_tier_identity( rules: list[dict[str, Any]], rule: dict[str, Any], ) -> tuple[float | None, int, int]: """``(threshold_pct, tier_index, tier_total)`` — how to name WHICH of a scope's threshold rules fired **without disclosing an amount**. The redacted (non-circle) threshold card carries no USD figure, and its "触发规则" tile used to read only ``月度阈值`` / ``日阈值``. With 50/80/100 % tiers plus tier suppression and one card per ``(period, basis)``, the scope's own owner could not tell whether they had crossed 50 % or 100 % of the budget they themselves set — the tile named the rule FAMILY, not the rule. ``tier_index`` / ``tier_total`` rank this rule among the scope's ACTIVE rules of the SAME ``(type, basis)`` by threshold ascending. Ordering only — no magnitude, and cross-basis peers are excluded because their thresholds are not comparable. **This rank is the only identifier the redacted card may carry.** ``threshold_pct`` is returned for the CIRCLE card (which shows amounts anyway) and **must not** reach a redacted one. It looks dimensionless and shipped that way for one round; it is not. ``GET /cmdb/scopes`` (``lifecycle_router.py:1092-1112``) serves ``budget_usd_monthly`` + ``budget_measure`` to a scope's employee owner/member, and on a ``net`` scope — which ``NULL`` means, i.e. most of them — that budget is an invoice-basis figure, so ``pct × budget`` yields the exact net threshold. That converts the accepted ``spend >= threshold`` residual into an absolute net lower bound, which is the disclosure withholding ``threshold_usd`` exists to prevent. A rank multiplies into nothing. Neither restates an absolute figure, so the invoice-basis red line (DECISIONS 2026-07-29 #1: a non-circle reader gets no invoice-basis number from any surface) holds. The threshold amount itself is still withheld: it is a net-denominated USD number, and "the owner authored it" does not make handing it back on a card a non-disclosure — an admin may have authored it, and the card is also delivered to ``cc_person_ids`` third parties who never saw it. """ pct_raw = rule.get("threshold_pct") pct: float | None try: pct = float(pct_raw) if pct_raw is not None else None except (TypeError, ValueError): pct = None own = _rule_threshold_usd(rule) rule_type = rule.get("type") basis = rule_measure(rule) peers = sorted( t for t in ( _rule_threshold_usd(r) for r in rules if isinstance(r, dict) and r.get("active", True) and r.get("type") == rule_type and rule_measure(r) == basis ) if t is not None ) index = peers.index(own) + 1 if own is not None and own in peers else 0 return pct, index, len(peers) def alert_slot_rule_type(rule_type: str, basis: str) -> str: """The dedup-slot discriminator for one rule: its type, qualified by the cost basis it is denominated in. ``(scope, rule_type, period_key)`` used to be the slot identity, which made the cross-run dedup gate basis-BLIND while tier suppression was per ``(type, basis)``: a net monthly alert firing on the 3rd consumed the month's only monthly slot and permanently suppressed the list monthly alert — two different numbers answering two different questions, and list is the engineering-facing default. Net keeps the BARE ``rule_type`` so rows claimed before this split keep deduping the net question they were claimed for (no migration, no backfill). ``rule_type`` is ``String(32)``; the longest value is ``daily_spike:list`` (16). """ return rule_type if basis == MEASURE_NET else f"{rule_type}:{basis}" def scope_budget_measure(scope: CmdbCloudScope) -> str: """The basis ``scope.budget_usd_monthly`` is denominated in.""" return ( MEASURE_LIST if scope.budget_measure == MEASURE_LIST else MEASURE_NET ) async def _stored_role_names( session: AsyncSession, actor_user_id: uuid.UUID, ) -> list[str]: """The actor's STORED role names (``auth_user_roles``). For callers that don't carry ``user.role_names`` (MCP, internal, tests). Stored roles only — derived scope (manager subtree, office-admin-derived ``asset_admin``) is irrelevant here: the one gate that uses this is ``can_manage_cloud``, whose inputs (``cloud_admin`` / ``global_admin``) are both stored roles. """ from app.auth.models import AuthRole, AuthUserRole rows = (await session.execute( select(AuthRole.name) .join(AuthUserRole, AuthUserRole.role_id == AuthRole.id) .where(AuthUserRole.user_id == actor_user_id), )).scalars().all() return list(rows) def apply_scope_budget( scope: CmdbCloudScope, *, budget_usd_monthly: Decimal | None, budget_measure: str | None, can_change_measure: bool, ) -> dict[str, Any]: """Write the monthly budget and/or its cost basis, keeping the dependent percentage alert-rule stamps in lock-step. Returns an audit ``changes`` fragment (empty when nothing moved). Single entry point for every budget writer (admin PATCH, external-API PATCH) because the number and its basis must never be settable independently of the rules derived FROM them: * a percentage rule's ``threshold_usd`` is ``pct × budget``, so changing the budget re-denominates it — it is re-materialised here (which is also exactly what ``threshold_pct`` means: "always N% of whatever I budget") and re-stamped with the budget's basis; * changing the NUMBER on a scope whose recorded basis is ``list`` without saying which basis the new number is in is REFUSED. Assuming "same basis" turns an admin correcting the ~2.7×-inflated figure back to the invoice number into list-basis evaluation against a net budget (#2079 verbatim); assuming "net" silently breaks a legitimate list-basis bump. Only the caller knows which number they typed. ``can_change_measure`` is the caller's authority to MOVE the recorded basis (2026-07-29 review, F1). Naming the basis the scope already uses is always allowed — that is the disambiguation the bullet above demands, and it re-denominates nothing. CHANGING it is admin-only (``can_manage_cloud``): a percentage rule's ``threshold_usd`` is ``pct × budget``, so a basis flip re-materialises and re-stamps every pct rule from the SAME (possibly ~2.7×-inflated) number. A scope owner — who may be a plain employee — flipping a list-converted budget to ``net`` alone gets ``pct × inflated`` thresholds stamped ``net`` and evaluated against net spend: rules that can never fire, in a state that IS self-consistent and therefore invisible to the evaluator's ambiguity signal (which only catches stamp/budget DISAGREEMENT). Raising the numeric budget was always the owner's prerogative; silently disarming their own alerts through the basis field was not — see ``set_scope_alert_rules``' trust-boundary comment. """ if budget_measure is not None and budget_measure not in ( MEASURE_NET, MEASURE_LIST, ): raise AppException( code="CLOUD_BUDGET_BAD_MEASURE", message=( f"budget_measure must be 'net' or 'list', got " f"{budget_measure!r}" ), status_code=400, ) current_measure = scope_budget_measure(scope) prev_budget = scope.budget_usd_monthly budget_changes = ( budget_usd_monthly is not None and (prev_budget is None or Decimal(prev_budget) != budget_usd_monthly) ) if not budget_changes and budget_measure is None: return {} if budget_usd_monthly is not None and budget_measure is None: if current_measure == MEASURE_LIST: raise AppException( code="CLOUD_BUDGET_MEASURE_REQUIRED", message=( "This scope's monthly budget is recorded in the " "pre-discount list measure (业务折前口径). State which " "measure the new figure is denominated in " "(budget_measure='list' to keep the list basis, or " "'net' if you are entering the invoice/实付 figure) — " "guessing re-creates the 2026-07 false-alert incident." ), status_code=400, ) effective_measure = MEASURE_NET else: effective_measure = budget_measure or current_measure if effective_measure != current_measure and not can_change_measure: raise AppException( code="CLOUD_BUDGET_MEASURE_FORBIDDEN", message=( f"Changing this scope's budget cost basis " f"({current_measure} → {effective_measure}) requires cloud-" f"admin authority: it re-denominates every percentage alert " f"threshold derived from the budget. Ask a cloud/IT admin, " f"who will restate the budget FIGURE in the new basis at the " f"same time. Setting the monthly budget itself (in the " f"scope's current {current_measure} basis) does not need " f"this." ), status_code=403, ) # Admin-side residual: a basis flip that does NOT restate the number # leaves the OLD figure labelled with the NEW basis — arithmetically # silent (nothing disagrees) but semantically wrong, and the pct rules # below inherit it. Audited via ``changes``; logged here too because the # audit trail is not where an on-call SRE looks when a threshold stops # firing. if ( effective_measure != current_measure and budget_usd_monthly is None and scope.budget_usd_monthly is not None ): logger.event( "CLOUD_SCOPE_BUDGET_BASIS_MOVED_WITHOUT_RESTATEMENT", { "scope_id": scope.id, "from_measure": current_measure, "to_measure": effective_measure, "budget": scope.budget_usd_monthly, }, level=logging.WARNING, ) changes: dict[str, Any] = {} if budget_changes: changes["budget_usd_monthly"] = { "from": str(prev_budget) if prev_budget is not None else None, "to": str(budget_usd_monthly), } scope.budget_usd_monthly = budget_usd_monthly # Compare the STORED value so normalising a legacy NULL to the explicit # ``'net'`` it always meant is recorded as the (real) change it is, # while a no-op re-assert of the same basis stays audit-silent. if scope.budget_measure != effective_measure: changes["budget_measure"] = { "from": current_measure, "to": effective_measure, } scope.budget_measure = effective_measure # Re-materialise every percentage rule against the new (budget, basis) # pair so basis and stamps can never disagree. restamped: list[dict[str, Any]] = [] new_rules: list[Any] = [] budget_f = ( float(scope.budget_usd_monthly) if scope.budget_usd_monthly is not None else 0.0 ) for r in scope.alert_rules or []: if not isinstance(r, dict) or r.get("threshold_pct") is None: new_rules.append(r) continue try: pct = float(r["threshold_pct"]) except (TypeError, ValueError): new_rules.append(r) continue nr = dict(r) if budget_f > 0 and pct > 0: nr["threshold_usd"] = round(budget_f * pct / 100.0, 2) nr["measure"] = effective_measure else: # No budget to multiply against — the stored threshold is a # leftover in the OLD basis and cannot be re-derived. Drop the # stamp so it evaluates on net (conservative) instead of # claiming a basis it no longer has. nr.pop("measure", None) if nr != r: restamped.append({ "type": nr.get("type"), "threshold_pct": pct, "threshold_usd": nr.get("threshold_usd"), "measure": nr.get("measure"), }) new_rules.append(nr) if restamped: scope.alert_rules = new_rules changes["pct_rules_redenominated"] = restamped return changes @dataclass(frozen=True) class MeasureAggregate: """The three cost aggregates a threshold comparison needs, all computed wholly within ONE basis (never blended — that is #2079's root cause). ``None`` on a window means "this basis is not complete over THAT window", and every rule whose comparison needs it is skipped + logged. Per window, not per basis (2026-07-29 review F5): the two windows have different widths, so a single list-NULL day on the 3rd makes the MTD window incomplete for ~28 days while today's own row is fine. Discarding the whole basis unarmed every list-denominated DAILY threshold on the scope for the rest of the month. """ today_cost: float | None month_to_date_cost: float | None prev_month_total_usd: float | None = None async def evaluate_scope_alert_rules( session: AsyncSession, scope: CmdbCloudScope, *, today_cost: float | None, today_date: date, month_to_date_cost: float | None, measure: str, extra_measures: dict[str, MeasureAggregate] | None = None, integration: Integration | None = None, dispatch: bool = True, prev_month_total_usd: float | None = None, ) -> int: """Fire Feishu + email per matching alert rule. Idempotent end-to-end via claim-first INSERT into ``cmdb_cloud_scope_alert_fired`` (migration 171) — see :func:`_try_claim_cloud_scope_alert_slot`. ``measure`` (``"list"`` / ``"net"``, REQUIRED — a default here would let a caller silently mislabel the card's cost basis, the exact incident shape of 2026-07-27) names the basis ``today_cost`` / ``month_to_date_cost`` / ``prev_month_total_usd`` were computed on; it is threaded into the threshold card's 口径说明 so recipients know which console number to reconcile against. ``extra_measures`` supplies the SAME three aggregates for any OTHER basis the caller could compute this run, keyed by basis name. Each rule is compared against the aggregate matching ITS OWN ``measure`` stamp (:func:`rule_measure`) — one scope may legitimately hold a list-denominated rule the conversion script converted next to a net-denominated rule an admin added afterwards, and evaluating either against the other's basis is a systematic false-fire (net threshold vs list spend) or false-negative (list threshold vs net spend). A rule whose basis is available NEITHER as ``measure`` NOR in ``extra_measures`` — or whose OWN window is ``None`` on that basis' aggregate — is SKIPPED and logged, never silently compared on a different basis. The caller omits a basis (or a window within it) precisely when that data is incomplete, so "skip + surface" is the only correct answer: comparing a list threshold against net spend is #2079's false trigger inverted into a permanent false negative. Daily rules read ``today_cost``, monthly rules read ``month_to_date_cost`` — each gated on its own window only. Returns the number of rules that actually dispatched this run. Per-rule claim-first contract: 1. Compute (slot_rule_type, period_key, threshold_usd) — uniquely identifies a dedup slot. ``slot_rule_type`` carries the rule's cost basis (:func:`alert_slot_rule_type`), so a net alert and a list alert for the same type+period hold SEPARATE slots. 2. Atomically INSERT a row into ``cmdb_cloud_scope_alert_fired`` inside a SAVEPOINT. UNIQUE-key collision → another run already fired for this slot → skip. 3. Claim won → dispatch the notification. Why claim-first (was: SELECT-then-INSERT-the-JSON-map): The prior shape mutated ``scope.alert_rules[*].last_fired`` and relied on the outer cron's ``session.commit()`` to persist. Any exception in the long path between dispatch and commit (the unregistered-account reconciliation loop, an FX-rate hiccup, a transient driver failure, etc.) rolled back the mutation while the Feishu card was already in flight. Next cron tick re-fired. Prod incident 2026-05-16: the ``backfill_cost_facts.py`` script issued ~7 cards per scope across 60 scopes for this reason. Same TOCTOU class PR #298 fixed for the apikey budget alerts. ``dispatch=False`` is the **backfill mode** — the function evaluates every rule and *claims* every slot that would fire (so the next live cron run won't re-page them on the historical data) but does NOT actually send notifications. The :mod:`server.scripts.backfill_cost_facts` walker passes False so a 6-month replay doesn't fire 7 retro-month cards per scope. ``prev_month_total_usd`` carries the previous calendar month's USD total for this scope, used to render a "↑42% vs last month" trend chip on the monthly card. ``None`` (the default) means the caller couldn't cheaply resolve it; the card renders without the trend chip in that case. The caller in :func:`sync_billing_daily_for_integration` resolves it from the same daily-billing rows used to compute ``month_to_date_cost``. Tier suppression: when several rules of the same ``rule_type`` (daily / monthly) cross thresholds in the same run — e.g. month-to-date cost passes 50%/80%/100% of budget all on the same day — only the **highest-threshold** rule pages. Lower-threshold siblings still claim their slot so they stay quiet for the period. Recipient set per rule = ``scope.owner_person_id`` ∪ {billing-role active grant holders on this scope} ∪ ``rule.cc_person_ids``. See :func:`_resolve_billing_alert_recipients`. """ rules = scope.alert_rules or [] if not rules: return 0 # Per-basis aggregate table. ``measure`` is the primary (always # present); ``extra_measures`` adds any other basis the caller could # compute wholly within itself. aggregates: dict[str, MeasureAggregate] = { measure: MeasureAggregate( today_cost=today_cost, month_to_date_cost=month_to_date_cost, prev_month_total_usd=prev_month_total_usd, ), **(extra_measures or {}), } budget_measure = scope_budget_measure(scope) if integration is None: integration = await session.get( Integration, uuid.UUID(str(scope.integration_id)), ) provider_name = integration.provider if integration is not None else None fired_count = 0 notify = NotificationService(session) month_key = today_date.strftime("%Y-%m") day_key = today_date.isoformat() scope_id_str = str(scope.id) # Snapshot of (slot_rule_type, period_key) tuples that ALREADY have an # alert-fired row from a PREVIOUS run. The pre-claim guard below # uses this to suppress threshold-change re-fire across runs: # admin lowers monthly $500 → $100 mid-period, the snapshot # contains ``("monthly", "2026-05")`` because a $500-threshold # row was claimed earlier in the month, so the $100 attempt is # blocked. # # ``slot_rule_type`` is basis-qualified (``"monthly"`` for net, # ``"monthly:list"`` for list), so this gate suppresses only re-fires of # the SAME question. Basis-blind, it let whichever basis fired first # silence the other for the whole period. # # Snapshotted UP FRONT (not consulted live mid-loop) so the # WITHIN-run tier-suppression case — multiple thresholds at # 50%/80%/100% trip together and each must claim its own slot # for next-tick suppression — still gets every slot claimed. # Live consultation would only let the FIRST tier claim, then # treat the rest as "already alerted this period" and skip. pre_existing_period_alerts: set[tuple[str, str]] = set() # SAVEPOINT-wrap the read — see _period_already_alerted: a swallowed # SELECT failure must not leave the shared session rolled-back- # pending, or the per-rule claim INSERT below raises an opaque # PendingRollbackError that aborts the whole billing_daily run. try: async with session.begin_nested(): existing_rows = await session.execute( select( CmdbCloudScopeAlertFired.rule_type, CmdbCloudScopeAlertFired.period_key, ).where( CmdbCloudScopeAlertFired.scope_id == scope_id_str, CmdbCloudScopeAlertFired.period_key.in_( (month_key, day_key), ), ), ) pre_existing_period_alerts = { (r.rule_type, r.period_key) for r in existing_rows } except Exception: # noqa: BLE001 — same defensive shape as _period_already_alerted pass # Pre-compute the highest-threshold rule per ``rule_type`` that # would fire this run. Used inside the loop to suppress lower- # threshold siblings — see docstring. # Tier suppression is per (rule_type, basis): a list-basis winner must # not suppress a net-basis sibling, because the two numbers are not # comparable — "$1,200 > $900" across bases means nothing. winner_threshold_by_type: dict[tuple[str, str], float] = {} for r in rules: if not r.get("active", True): continue rt = r.get("type") try: thr = float(r.get("threshold_usd") or 0) except (TypeError, ValueError): continue if thr <= 0 or rt not in ("daily", "monthly"): continue agg = aggregates.get(rule_measure(r)) if agg is None: continue cur = agg.today_cost if rt == "daily" else agg.month_to_date_cost if cur is None or cur < thr: continue key = (rt, rule_measure(r)) if key not in winner_threshold_by_type or thr > winner_threshold_by_type[key]: winner_threshold_by_type[key] = thr # Ambiguity signal (2026-07-28 review, F2): a scope whose BUDGET was # re-denominated to the list basis but which still carries unstamped # active threshold rules is the state that used to fail silently — # those thresholds are most likely list-denominated leftovers of the # conversion (or an admin retune) and now evaluate on net, so they can # never be crossed. Migration 433 backfills every stamp the conversion # audit trail can prove; anything left is genuinely ambiguous and needs # a human. Logged here (the evaluator is the only place that knows both # halves) and surfaced per rule in the scope API + UI basis tag. if budget_measure == MEASURE_LIST: unstamped = [ r.get("type") for r in rules if isinstance(r, dict) and r.get("active", True) and r.get("type") in ("daily", "monthly") and rule_measure(r) == MEASURE_NET ] if unstamped: logger.event( "CLOUD_SCOPE_ALERT_MEASURE_AMBIGUITY", { "scope_id": scope_id_str, "rule_count": len(unstamped), "rule_types": unstamped, }, level=logging.WARNING, ) for rule in rules: if not rule.get("active", True): continue rule_type = rule.get("type") try: threshold = float(rule.get("threshold_usd") or 0) except (TypeError, ValueError): continue if threshold <= 0: continue # Each rule reads the aggregate computed in ITS OWN basis. Missing # basis = the caller could not compute that basis for this window # (incomplete list column, typically) → skip loudly. Never fall # back to another basis: that is exactly the cross-measure # comparison the whole-group invariant forbids. rule_basis = rule_measure(rule) agg = aggregates.get(rule_basis) if agg is None: logger.event( "CLOUD_SCOPE_ALERT_NO_AGGREGATE", { "scope_id": scope_id_str, "rule_type": rule_type, "threshold": threshold, "measure": rule_basis, }, level=logging.WARNING, ) continue if rule_type == "daily": current = agg.today_cost period_key = day_key slot_base_type = CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY elif rule_type == "monthly": current = agg.month_to_date_cost period_key = month_key slot_base_type = CLOUD_SCOPE_ALERT_RULE_TYPE_MONTHLY else: continue # The basis exists but THIS rule's window is not complete in it. Same # posture as the missing-basis branch above — skip, log, claim no # slot. A daily rule is NOT collateral of an incomplete MTD window # (and vice versa): the two windows are gated independently. if current is None: logger.warning( "cloud.scope_alert skipped: scope_id=%s rule_type=%s " "threshold=%s is denominated in the %r measure, which has no " "complete %s window this run (data incomplete) — NOT " "evaluated on another basis or another window.", scope_id_str, rule_type, threshold, rule_basis, "daily" if rule_type == "daily" else "month-to-date", ) continue # Slot identity carries the basis, matching tier suppression's # ``(type, basis)`` key — see ``alert_slot_rule_type``. slot_rule_type = alert_slot_rule_type(slot_base_type, rule_basis) if current < threshold: continue # Legacy JSON fast-path skip — deploy-day back-compat. Pre- # migration-171 environments tracked dedup in # ``rule.last_fired[fired_key]``. The 2026-05-16 prod hotfix # data-stamped this map for the current month + every # historical month-key that would have fired, so the very # first post-deploy cron tick MUST honour those marks or # every previously-fired card re-fires on a fresh # (empty-on-deploy) ``cmdb_cloud_scope_alert_fired`` table. # We read the JSON BEFORE the claim attempt so we don't # paper over the legacy state with a fresh ``fired_at=now()`` # row that misrepresents when the alert actually went out. legacy_fired_map = rule.get("last_fired") or {} legacy_fired_key = ( f"daily:{day_key}" if rule_type == "daily" else f"monthly:{month_key}" ) if legacy_fired_map.get(legacy_fired_key): continue # legacy JSON says already fired — skip claim. # Threshold-independent pre-claim gate. An admin who tunes the # threshold (e.g. $500 → $100) inside the same period would # otherwise let the new threshold open a NEW dedup slot (the # UNIQUE includes ``threshold_usd``) and re-page the same # scope on the same data — one scope, one period, two cards. # Refuse to dispatch a second alert in the same period # regardless of which threshold tripped first. Per BASIS, though: # "one alert per scope per period" is a promise about one number, and # net spend vs list spend are two. # # Uses the PRE-RUN snapshot (not a live SELECT) so the # within-run tier-suppression loop — every threshold of the # same type claims a slot, only the highest dispatches — is # NOT short-circuited by the first tier's claim. if (slot_rule_type, period_key) in pre_existing_period_alerts: continue # Claim-first atomic dedup. Claim BOTH winners (which will # dispatch) and tier-suppressed losers (which won't) so the # next cron tick doesn't re-evaluate either after a dip. claimed = await _try_claim_cloud_scope_alert_slot( session, scope_id=scope_id_str, rule_type=slot_rule_type, period_key=period_key, threshold_usd=Decimal(str(threshold)), current_usd=Decimal(str(current)), ) if not claimed: continue # already alerted this period — slot is taken. # Mirror the claim into the legacy ``last_fired`` JSON for # admin UIs that still render off it (read-side back-compat). # If this fails to persist (outer rollback), the new dedup # table is still authoritative — no double-fire risk. fired_map = rule.setdefault("last_fired", {}) fired_key = ( f"daily:{day_key}" if rule_type == "daily" else f"monthly:{month_key}" ) fired_map[fired_key] = True # Tier suppression: a higher-threshold sibling of the same # type is firing in this same run, so suppress this lower # one. Slot is already claimed; just don't dispatch. winner_thr = winner_threshold_by_type.get((rule_type, rule_basis)) if winner_thr is not None and threshold < winner_thr: continue # ``dispatch=False`` mode — backfill walker. Slot is claimed # (so the next live run won't re-page on historical data) but # we don't actually send the card. if not dispatch: continue # Per-rule recipient set: union of owner + billing-role grants # + this rule's cc_person_ids. recipients = await _resolve_billing_alert_recipients( session, scope, extra_person_ids=list(rule.get("cc_person_ids") or []), ) if not recipients: # Slot is already claimed above — intentional. A scope with # no owner, no billing-class grants, and no rule # ``cc_person_ids`` permanently has nobody to notify; if # we DIDN'T consume the slot, every cron tick would re- # query the (still empty) recipient set and re-attempt the # dispatch — log spam without action. The right fix on the # admin's side is to bind an owner or a billing grant; until # then the scope stays silent for this period and re-fires # NEXT period when the slot key rolls over (different # ``period_key``). # # Log loudly (once per (scope, period) — slot is single- # claim so this fires at most once even across cron retries) # so SRE / admin sees the gap in the log aggregator: a # threshold was crossed but nobody was notified. Without # this, the "your scope spiked but no card landed" path # is invisible until somebody manually correlates billing # reports against received alerts. logger.warning( "cloud.scope_alert dispatched to empty recipient set: " "scope_id=%s rule_type=%s period=%s threshold=%s " "current=%s — owner/billing-grant unassigned, alert " "silently consumed. Bind an owner or grant a billing-" "class role on this scope to restore notifications.", scope_id_str, slot_rule_type, period_key, threshold, current, ) continue # Which tier fired, in amount-free terms — the redacted card's only # way to distinguish 50 % from 100 % of the owner's own budget. rule_pct, tier_index, tier_total = rule_tier_identity(rules, rule) await _dispatch_scope_threshold_alert( notify, session, scope=scope, integration=integration, provider_name=provider_name, rule_type=rule_type, period_key=period_key, threshold=threshold, current=current, prev_month_total_usd=agg.prev_month_total_usd, recipients=recipients, measure=rule_basis, budget_measure=budget_measure, rule_pct=rule_pct, tier_index=tier_index, tier_total=tier_total, ) fired_count += 1 # Re-assign so SQLAlchemy picks up the ``last_fired`` JSON # mutations (legacy back-compat path — new dedup table is # authoritative). scope.alert_rules = list(rules) return fired_count async def _dispatch_scope_threshold_alert( notify: NotificationService, session: AsyncSession, *, scope: CmdbCloudScope, integration: Integration | None, provider_name: str | None, rule_type: str, period_key: str, threshold: float, current: float, prev_month_total_usd: float | None, recipients: list[IdentityPerson], measure: str, budget_measure: str = MEASURE_NET, rule_pct: float | None = None, tier_index: int = 0, tier_total: int = 0, ) -> None: """Render and send the cloud-bill threshold card to all recipients. ``measure`` (``"list"`` / ``"net"``, REQUIRED — no default, same rationale as :func:`_alert_spike`) names the cost basis the evaluator compared on FOR THIS RULE; the card carries the matching 口径说明 so a recipient cross-checking against the cloud console (which shows the discounted invoice figure) doesn't conclude the alert read wrong data — the 2026-07-27 incident follow-up that added the note to the spike card applies identically here. ``budget_measure`` is the basis ``scope.budget_usd_monthly`` is denominated in. The "X% 月度预算" progress chip is rendered ONLY when it matches ``measure`` — a list-basis spend over a net-denominated budget reads ~2.7× too high, which is the same cross-measure arithmetic the evaluator itself refuses to do. ``measure == MEASURE_NET`` makes the card PER-RECIPIENT (2026-07-29 confidentiality sweep, notification surface): every USD figure on it is invoice-basis, so non-circle recipients (:func:`_invoice_measure_recipient_ids`) get a variant that names the crossed rule and its period but carries NO amount — not the period spend, not the threshold, not the overage, not the budget or its percentage. The threshold and the budget are the owner's own inputs, but "spend ≥ $500" is still a lower bound on an invoice-basis number, and ``current == pct × budget`` reconstructs the figure exactly. Dropping the card for them instead would leave a non-prod scope's owner unaware their own budget line was crossed; "your scope crossed the threshold you configured, open the (list-basis) billing page" is true, actionable and leaks nothing. ``rule_pct`` / ``tier_index`` / ``tier_total`` (:func:`rule_tier_identity`) name WHICH rule fired on that redacted variant — the tile used to read only ``月度阈值`` / ``日阈值``, so an owner running 50/80/100 % tiers could not tell which one they had crossed. Dimensionless by construction (a declared percentage, or a rank among same-basis siblings); the amount stays withheld. ``tier_total <= 1`` renders no rank — a scope with one rule of that type has nothing to disambiguate. Card shape mirrors the daily-spike card (:func:`_alert_spike`): a ``card_subtitle`` (cloud account / environment / period), a grid of :class:`CardStat` tiles (period spend, threshold, overage, budget progress, last-month delta, projection), and a "建议下一步" :class:`CardSection`. The previous template-only rendering only carried four lines of plain text — recipients couldn't tell which month the alert was for, what the budget was, or how it compared to last month. Title carries the **explicit period_key** so a backfilled historical-month alert can't be mistaken for a current-month spike: ``"☁️ 云账单月度阈值 — 2026-04 — tikv-storage-dev"``. Best-effort per recipient — a dispatch failure for one recipient is logged but does NOT abort delivery to the others or undo the already-claimed dedup slot. """ from app.notifications.service import CardSection, CardStat scope_label = scope.display_name or scope.external_id integ_name = integration.name if integration is not None else "" env_label = scope.environment or "—" provider_zh, provider_en = _format_cloud_provider_label(provider_name) overage = current - threshold is_monthly = rule_type == "monthly" period_zh = "月度" if is_monthly else "日" period_en = "monthly" if is_monthly else "daily" period_ja = "月次" if is_monthly else "日次" period_label_zh = f"{period_key} 月累计花费" if is_monthly else f"{period_key} 花费" period_label_en = f"{period_key} cumulative spend" if is_monthly else f"{period_key} spend" period_label_ja = ( f"{period_key} の月間累計コスト" if is_monthly else f"{period_key} のコスト" ) # Last-month trend chip (monthly only) — resolved per recipient. trend_chip: LocalizedText | None = None if is_monthly and prev_month_total_usd is not None and prev_month_total_usd > 1e-2: delta_pct = (current - prev_month_total_usd) / prev_month_total_usd * 100.0 sign = "▲" if delta_pct >= 0 else "▼" trend_chip = LocalizedText( zh=f"{sign} {abs(delta_pct):.0f}% vs 上月", en=f"{sign} {abs(delta_pct):.0f}% vs last month", ja=f"{sign} {abs(delta_pct):.0f}% 前月比", ) # Budget progress — only when the scope has a positive budget. budget_usd: float | None = None budget_progress: LocalizedText | None = None if scope.budget_usd_monthly is not None: try: b = float(scope.budget_usd_monthly) if b > 0: budget_usd = b # Percentage only when the budget shares this rule's basis. # Across bases the ratio is meaningless (a list-basis spend # over a net budget reads ~2.7× high) — show the budget # figure with its own 口径 label instead of a false percent. if budget_measure == measure: pct = current / b * 100.0 budget_progress = LocalizedText( zh=f"{pct:.0f}% 月度预算", en=f"{pct:.0f}% of monthly budget", ja=f"月次予算の {pct:.0f}%", ) except (TypeError, ValueError): pass # Simple linear month-end projection (monthly only). projection_render_zh: str | None = None projection_render_en: str | None = None projection_render_ja: str | None = None if is_monthly: from calendar import monthrange try: yy, mm = period_key.split("-") year_i = int(yy) month_i = int(mm) today = datetime.now(UTC).date() anchor_last = monthrange(year_i, month_i)[1] if (today.year, today.month) == (year_i, month_i): dom = today.day elif (year_i, month_i) < (today.year, today.month): dom = anchor_last else: dom = anchor_last if dom > 0 and current > 0: projected = current * anchor_last / dom projection_render_zh = f"${projected:,.0f} (按当前节奏)" projection_render_en = f"${projected:,.0f} (at current pace)" projection_render_ja = f"${projected:,.0f}(現在のペース)" except (ValueError, ZeroDivisionError): pass # 双口径保密红线(2026-07-30 第二批复查 P1-2)。A LIST-basis rule fires # the FULL card to every recipient (``redacted`` below is False by # construction) — correct for the compared amounts, which are 折前 and # readable by everyone. But the budget tile is a SECOND basis: when the # scope declares ``budget_measure`` net, ``budget_usd`` is an # invoice-basis absolute, and the full card was handing it to scope # owners, billing-grant holders and rule ``cc`` third parties. Worse, # this batch's root-cause fix (a non-circle author's new absolute rule # is stamped ``list``) MANUFACTURES exactly the list-rule-on-net-budget # combination, so the leak's frequency rises with the fix. # # So the tile gets its own out-of-circle gate, independent of the # comparison basis: withheld unless the reader is in the circle. _budget_is_confidential = ( budget_usd is not None and budget_measure != MEASURE_LIST ) # Shared across the full and the redacted tile builders. ``None`` when # the scope has no integration name to name — same as before. account_hint = ( LocalizedText( zh=f"云账号: {integ_name}", en=f"Account: {integ_name}", ja=f"クラウドアカウント: {integ_name}", ) if integ_name else None ) # ``CardStat.trend`` is typed ``CardText`` too, so the last string this # builder had to pick a language for now resolves per recipient like # every other one — the ``en`` argument is gone. def _build_card_stats(hide_budget: bool = False) -> list[CardStat]: stats: list[CardStat] = [ CardStat( label=LocalizedText( zh=period_label_zh, en=period_label_en, ja=period_label_ja, ), value=f"${current:,.2f} USD", trend=trend_chip, trend_is_good=False, hint=account_hint, ), CardStat( label=LocalizedText( zh="阈值", en="Threshold", ja="しきい値", ), value=f"${threshold:,.2f} USD", ), CardStat( label=LocalizedText( zh="超出阈值", en="Over threshold", ja="しきい値超過分", ), value=f"${overage:,.2f} USD", trend=budget_progress, trend_is_good=False, ), ] if budget_usd is not None and not hide_budget: # Name the budget's own 口径 when it differs from the compared # basis, so a recipient doesn't read the two numbers as directly # comparable. budget_label = ( LocalizedText(zh="月度预算", en="Monthly budget", ja="月次予算") if budget_measure == measure else LocalizedText( zh=f"月度预算({'折前' if budget_measure == MEASURE_LIST else '实付'}口径)", en=f"Monthly budget ({budget_measure})", ja=( f"月次予算({'割引前' if budget_measure == MEASURE_LIST else '実支払'}基準)" ), ) ) stats.append(CardStat( label=budget_label, value=f"${budget_usd:,.2f} USD", )) if projection_render_zh: stats.append(CardStat( label=LocalizedText( zh="月末预估", en="Month-end projection", ja="月末見込み", ), value=LocalizedText( zh=projection_render_zh, en=projection_render_en or projection_render_zh, ja=projection_render_ja or projection_render_zh, ), trend_is_good=False, )) stats.append(CardStat( label=LocalizedText(zh="环境", en="Environment", ja="環境"), value=env_label, )) return stats # Which of the scope's rules fired, in amount-free terms: its RANK among # same-(type, basis) siblings by threshold ascending. Suppressed when the # scope holds a single rule of this type — nothing to disambiguate. # # Rank, NOT the declared percentage, and this is the whole subtlety: a # percentage looks dimensionless but is not, because `GET /cmdb/scopes` # serves `budget_usd_monthly` + `budget_measure` to a scope's employee # owner/member (`lifecycle_router.py:1092-1112`). On a `net`-basis scope # (which NULL means, i.e. most of them) that budget IS an invoice-basis # figure, so `pct × budget` handed the recipient the exact net threshold — # sharpening the accepted `spend ≥ threshold` residual into an absolute # net lower bound, which is precisely what withholding `threshold_usd` # was for. Rank multiplies with a budget into nothing. rule_qualifier_zh: str | None = None rule_qualifier_en: str | None = None rule_qualifier_ja: str | None = None if tier_total > 1 and tier_index > 0: rule_qualifier_zh = f"第 {tier_index}/{tier_total} 档" rule_qualifier_en = f"tier {tier_index} of {tier_total}" rule_qualifier_ja = f"{tier_total} 段階中 {tier_index} 段階目" def _build_redacted_card_stats() -> list[CardStat]: """Amount-free tiles: which rule, which period, which environment. The monthly trend chip rides along — it is a ratio of two net figures, neither of which the recipient holds. The rule tile carries the tier qualifier so an owner running several tiers can tell WHICH of their own lines fired — by RANK (「第 1/2 档」), never by the declared percentage: `GET /cmdb/scopes` serves that owner ``budget_usd_monthly`` + ``budget_measure``, so ``pct × budget`` reconstructs the exact net threshold. See ``rule_tier_identity``.""" rule_value_en = f"{period_en} threshold" rule_value_zh = f"{period_zh}阈值" rule_value_ja = f"{period_ja}しきい値" if rule_qualifier_en: rule_value_en = f"{rule_value_en} · {rule_qualifier_en}" rule_value_zh = f"{rule_value_zh} · {rule_qualifier_zh}" rule_value_ja = f"{rule_value_ja} · {rule_qualifier_ja}" return [ CardStat( label=LocalizedText( zh="超阈值周期", en="Period over threshold", ja="しきい値超過期間", ), value=period_key, trend=trend_chip, trend_is_good=False, hint=account_hint, ), CardStat( label=LocalizedText( zh="触发规则", en="Rule", ja="発動ルール", ), value=LocalizedText( zh=rule_value_zh, en=rule_value_en, ja=rule_value_ja, ), ), CardStat( label=LocalizedText(zh="环境", en="Environment", ja="環境"), value=env_label, ), ] # Same 口径说明 wording as the daily-spike card (``_alert_spike``) so # the two alert families explain their cost basis identically. measure_note_zh = ( "口径说明：金额为业务折前（list 牌价）口径，通常高于云控制台 / " "发票展示的折后金额，请勿与控制台绝对值直接对比。" if measure == "list" else "口径说明：金额为发票净额（折后）口径。" ) measure_note_en = ( "Basis note: amounts are on the pre-discount list-price basis, " "usually HIGHER than the discounted figure shown in the cloud " "console / invoice — do not compare absolute values against the " "console directly." if measure == "list" else "Basis note: amounts are on the invoice-net (post-discount) basis." ) measure_note_ja = ( "基準について：金額は割引前(リスト価格)基準です。クラウドコンソール / " "請求書に表示される割引後の金額より高くなるのが通常ですので、コンソール" "の絶対値と直接比較しないでください。" if measure == "list" else "基準について：金額は請求ネット(割引後)基準です。" ) # Why the figures are missing, in the recipient's own terms. Points at # the surface they CAN read (the list-basis billing page) so the card is # a next step rather than a dead end. redacted_note_zh = ( "口径说明：本次比较按发票净额（实付）口径评估。实付金额仅财务 / " "云管可见，本卡因此不展示绝对金额；请在「云账单」页按业务折前口径" "查看明细，或联系 IT 获取实付数字。" ) redacted_note_en = ( "Basis note: this comparison was evaluated on the invoice-net " "basis. Invoice-net amounts are restricted to finance / cloud " "admins, so this card omits absolute figures — open the cloud " "billing page (pre-discount list basis) for details, or ask IT for " "the invoice figure." ) redacted_note_ja = ( "基準について：今回の比較は請求ネット(実支払)基準で評価しました。" "実支払金額は経理 / クラウド管理者のみが参照できるため、このカードでは" "絶対金額を表示していません。詳細は「クラウド請求」ページで割引前" "(リスト価格)基準の内訳をご確認いただくか、IT までお問い合わせください。" ) # The ``en`` argument this used to take only picked which language the # section chrome was written in — ``LocalizedText`` now resolves that # per recipient, so the branch (and the parameter) are gone. def _build_card_sections(redacted: bool = False) -> list[CardSection]: return [CardSection( title=LocalizedText( zh="建议下一步", en="Recommended next steps", ja="推奨される次のステップ", ), body=LocalizedText( zh=( "1) 打开账单详情查看本月新增的服务 / 资源\n" "2) 如属正常迁移 / 灰度,在 scope 备注里记录\n" "3) 如不预期,及时止血或联系 IT 排查\n\n" f"{redacted_note_zh if redacted else measure_note_zh}\n" "如对帐单有任何疑问，请联系 IT 团队进行确认。" ), en=( "1) Open billing details to review new services / resources this month\n" "2) If planned (migration / canary), add a note in the scope remarks\n" "3) If unexpected, act quickly to reduce spend or escalate to IT\n\n" f"{redacted_note_en if redacted else measure_note_en}\n" "If you have any questions about the bill, please contact the IT team for confirmation." ), ja=( "1) 請求明細を開き、今月追加されたサービス / リソースを確認してください\n" "2) 想定内の移行 / カナリアであれば、スコープの備考に記録してください\n" "3) 想定外の場合は、速やかにコストを抑えるか IT にエスカレーションしてください\n\n" f"{redacted_note_ja if redacted else measure_note_ja}\n" "請求内容についてご不明な点があれば、IT チームまでご確認ください。" ), ), tone="warn", )] # Period-bearing title — recipient can tell at a glance which # month the alert is for. Backfill-driven historical alerts no # longer masquerade as current-month spikes. title_zh = f"☁️ 云账单{period_zh}阈值 — {period_key} — {scope_label}" title_en = f"☁️ Cloud bill {period_en} threshold — {period_key} — {scope_label}" body_text_zh = ( f"{provider_zh} · `{scope_label}` 在 {period_key} {period_zh}累计 " f"${current:,.2f} USD,已超出阈值 ${threshold:,.2f}。" ) body_text_en = ( f"{provider_en} · `{scope_label}` has accumulated ${current:,.2f} USD " f"in {period_key} ({period_en}), exceeding the threshold of ${threshold:,.2f}." ) # The qualifier is repeated in the body because email recipients read the # plain-text line, not the card tiles. _q_zh = f"（{rule_qualifier_zh}）" if rule_qualifier_zh else "" _q_en = f" ({rule_qualifier_en})" if rule_qualifier_en else "" redacted_body_zh = ( f"{provider_zh} · `{scope_label}` 在 {period_key} 的{period_zh}累计" f"花费已超过该 scope 配置的{period_zh}阈值{_q_zh}。本次比较按实付口径" f"评估，金额不在本卡展示（仅财务 / 云管可见）。" ) redacted_body_en = ( f"{provider_en} · `{scope_label}` crossed its configured " f"{period_en} spend threshold{_q_en} in {period_key}. The comparison " f"ran on the invoice-net basis, so the amounts are not shown on this " f"card (finance / cloud admins only)." ) card_subtitle = LocalizedText( zh=f"{integ_name or provider_zh} / {env_label} 环境 / {period_key}", en=f"{integ_name or provider_en} / {env_label} / {period_key}", ja=f"{integ_name or provider_en} / {env_label} 環境 / {period_key}", ) # Resolve circle membership when EITHER basis in play is confidential: # the compared amounts (net rule ⇒ whole card redacted) or the budget # tile alone (list rule on a net budget ⇒ full card minus that tile). circle_ids = ( await _invoice_measure_recipient_ids(session, recipients) if measure != MEASURE_LIST or _budget_is_confidential else set() ) for r in recipients: _rl = (r.preferred_language or "").lower() is_en = _rl.startswith("en") or _rl.startswith("ja") out_of_circle = str(r.id) not in circle_ids redacted = measure != MEASURE_LIST and out_of_circle # Only reachable on the list-rule path — the redacted card withholds # the budget wholesale already. hide_budget = _budget_is_confidential and out_of_circle try: await notify.dispatch( NotificationEnvelope( recipient=r, title=title_en if is_en else title_zh, body_text=( (redacted_body_en if is_en else redacted_body_zh) if redacted else (body_text_en if is_en else body_text_zh) ), channels=["feishu", "email"], url=f"/cloud/scopes/{scope.id}", card_subtitle=card_subtitle, card_stats=( _build_redacted_card_stats() if redacted else _build_card_stats(hide_budget=hide_budget) ), card_sections=_build_card_sections(redacted), related_resource_type="cmdb_cloud_scope", related_resource_id=scope.id, ), trace_id="cloud.scope_alert", ) except Exception: # noqa: BLE001 logger.exception( "scope alert dispatch failed scope=%s recipient=%s " "period=%s", scope.id, r.id, period_key, ) # ── Sweep implementations (used by scheduled handlers) ───────────── async def sweep_expiring_grants( session: AsyncSession, now: datetime | None = None, ) -> dict[str, Any]: """Run the T-14 / T-7 / T-1 reminder + T+0 revoke cycle. Emits one notification per grant per stage (idempotent via ``last_notified_stage``). Grants with ``expires_at <= now`` are auto-revoked via the driver; driver failures are logged but the grant row still flips to ``expired`` (provider-side cleanup may lag). Scope-owner exemption (2026-05-12): a grant whose ``person_id`` == its scope's ``owner_person_id`` is treated as perpetual — no T-14/ T-7/T-1 reminder, no T+0 revoke. The owner is responsible for approving every other member's grant on the scope; auto-revoking their own access creates a chicken-and-egg deadlock (they'd lose the ability to approve their own renewal, and re-requesting access to a scope you own is busywork). Treated the same way as the ``it_admin`` / ``cloud_admin`` / ``global_admin`` privileged-role exemption in :func:`sweep_inactive_grants`, but checked at the (scope, person) pair level rather than globally — a person who is owner of scope A and a regular member on scope B still has their scope-B grant subject to the normal expiry cycle. """ now_dt = now or datetime.now(UTC) service = CloudLifecycleService(session) notify_days = await _setting_json(session, "cloud.grant_notify_days", [14, 7, 1]) # NB: ``cloud.grant_renewal_days`` is deliberately NOT read here any # more. The reminder card used to promise 「一键续期 N 天」 with N from # that setting; renewal is now a request whose duration the requester # chooses (bounded by GRANT_TTL_HARD_CAP_DAYS) and the approver signs # off on, so quoting a fixed N on the card would state a number nobody # is bound by. stages = [ (14, GRANT_NOTIFY_T14), (7, GRANT_NOTIFY_T7), (1, GRANT_NOTIFY_T1), ] stages = [(d, s) for d, s in stages if d in notify_days] notified = 0 revoked = 0 owner_skipped = 0 prod_skipped = 0 # prod grants held past expiry (revoke pass) prod_reminder_skipped = 0 # prod grants skipped in the reminder pass perpetual_skipped = 0 # cloud/global-admin perpetual grants (both passes) perpetual_normalized = 0 # stale TTL cleared off perpetual rows revoke_notified = 0 # people told their access was recycled scanned = 0 # union of grants visited across reminder + expiry passes errors: list[str] = [] # {person_id: [(scope_id, role_label)]} — accumulated during the # revoke pass so the "your access was recycled" notice is one card per # person instead of one per grant. revoked_by_person: dict[str, list[tuple[str, str]]] = {} # Cloud/global admins hold PERPETUAL grants — see # ``load_perpetual_person_ids`` # for the incident this closes. Resolved once per run so the per-grant # check is a constant-time set lookup, exactly like the inactivity # sweep's ``exempt_person_ids``. perpetual_person_ids = await load_perpetual_person_ids(session) # Prefetch every scope's owner AND environment once, for the # owner-exemption and prod-exemption checks. The expiry sweep touches # O(grants) rows per tick; without this map we'd do a # ``session.get(CmdbCloudScope, …)`` per grant just to read two # columns, which the inactivity sweep refactor explicitly called out # as N+1 cost. scope_owner_map: dict[str, str | None] = {} scope_env_map: dict[str, str] = {} for sid, oid, env in (await session.execute( select( CmdbCloudScope.id, CmdbCloudScope.owner_person_id, CmdbCloudScope.environment, ) )).all(): scope_owner_map[str(sid)] = str(oid) if oid else None # ``environment`` is NOT NULL with default ``"dev"`` but historic # rows may carry NULL — coerce so the prod check never trips on # ``None.lower()``. scope_env_map[str(sid)] = (env or "dev") def _is_owner_grant(grant: CmdbCloudAccessGrant) -> bool: owner_id = scope_owner_map.get(str(grant.scope_id)) return owner_id is not None and owner_id == str(grant.person_id) def _is_prod_grant(grant: CmdbCloudAccessGrant) -> bool: # Production cloud access is NOT managed by this lifecycle sweep: # it is never auto-recycled (the revoke pass has always skipped # env=prod), and the prod cloud-write lock is default-LOCKED so # the "一键续期" magic link can never actually renew it. Sending a # "expires in N days — renew or it's auto-recycled" card for a # prod grant is therefore both false (no auto-recycle) and a # dead-end (the renew click 403s on the lock). Exclude prod from # every pass — the R&D security team owns prod cloud access. return _is_prod_environment(scope_env_map.get(str(grant.scope_id))) def _is_perpetual_grant(grant: CmdbCloudAccessGrant) -> bool: # Cloud admins are perpetual. Belt-and-braces alongside pass 0's # normalization: pass 0 clears ``expires_at`` so these rows drop # out of both pass queries, but a row written BETWEEN pass 0 and # a later pass (a concurrent grants_sync insert, an approve that # raced the role grant) would still carry a TTL. Checking the # prefetched set per grant means such a row is skipped rather # than revoked, and the next run's pass 0 cleans it up. return str(grant.person_id) in perpetual_person_ids # 0. Perpetual normalization pass — reconcile rows that carry a TTL # they should never have had. # # ``expires_at`` is the ONLY thing the two passes below key on, so # a perpetual holder is protected properly only when their rows # actually hold NULL. Skipping them at read time (the # ``_is_perpetual_grant`` check) protects access but leaves the # row lying about itself: the admin grant detail, the CSV export, # and every future reader still see "expires 2026-07-28", and the # moment someone reads ``expires_at`` without also re-deriving the # perpetual predicate the bug is back. # # So: heal the data, don't just guard the read. This is a bounded, # idempotent UPDATE (the perpetual set is a handful of people) and # it converges — once a row is NULL it never matches again. # ``last_notified_stage`` is cleared alongside so a holder who was # already nagged at T-14 does not keep a stale marker that would # suppress a legitimate future reminder if they ever lose the role. if perpetual_person_ids: norm_result = await session.execute( update(CmdbCloudAccessGrant) .where( CmdbCloudAccessGrant.person_id.in_(perpetual_person_ids), CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, CmdbCloudAccessGrant.expires_at.isnot(None), ) .values(expires_at=None, last_notified_stage=None) ) perpetual_normalized = int(norm_result.rowcount or 0) if perpetual_normalized: await session.commit() logger.info( "cloud.access_expiry_sweep: normalized %d perpetual " "(cloud_admin) grant(s) that were carrying a TTL — these " "would otherwise have been auto-revoked despite the UI " "showing 永久授权", perpetual_normalized, ) # 0b. The MIRROR of pass 0 — rows holding NULL that have no reason to. # # ``expires_at IS NULL`` is legitimate for exactly two populations: # a perpetual-ROLE holder, and the scope's CURRENT owner. A row that # is neither is invisible to every pass below (both key on # ``expires_at IS NOT NULL``) and to ``_disarm_owner_handover_trap`` # as it was originally written — so it is a permanent grant nobody # will ever revisit. # # How they get made: the auto-owner backfill writes the then-owner's # grant with ``force_perpetual`` (correct — the owner always has # access), ownership later transfers, and the handover re-stamp # skipped NULL rows on the grounds that "NULL means perpetual". # Production 2026-08-18: 7 rows, all from the 2026-06-02 backfill # batch, on scopes whose ownership had since moved. # # The cause is fixed in ``_disarm_owner_handover_trap``; this pass # heals the rows already in that state, in the same spirit as pass 0 # — heal the data, don't just guard the read. Stamping a fresh TTL # (not revoking) is the gentle correct action: it puts the holder # back into the normal cycle with a FULL T-14 / T-7 / T-1 runway, # which is exactly what the handover re-stamp would have given them. # ``last_notified_stage`` is cleared so no stale marker suppresses # that runway. Idempotent and converging: once stamped, the row no # longer matches. # # PROD IS EXCLUDED. Helix does not manage production cloud access at # all (both passes below skip ``PROD_ENVIRONMENTS``), so stamping a # TTL there would write a date nothing ever acts on while inflating # the read-only prod-overdue counter. Those rows stay NULL and stay # visible through the dimension's ``prod_grant_count`` fact. unreasoned_default_ttl = await _setting_int( session, "cloud.grant_default_ttl_days", 90, ) unreasoned_rows = list((await session.execute( select(CmdbCloudAccessGrant, CmdbCloudScope) .join(CmdbCloudScope, CmdbCloudScope.id == CmdbCloudAccessGrant.scope_id) .where( CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, CmdbCloudAccessGrant.expires_at.is_(None), CmdbCloudScope.deleted_at.is_(None), ) )).all()) unreasoned_stamped = 0 for grant_row, scope_row in unreasoned_rows: if grant_row.person_id is None: continue # orphan-cleanup history row; no holder to re-stamp if str(grant_row.person_id) in perpetual_person_ids: continue # perpetual by ROLE if scope_row.owner_person_id is not None and str( scope_row.owner_person_id, ) == str(grant_row.person_id): continue # the CURRENT owner always has access if _is_prod_environment(scope_row.environment): continue # Helix does not manage prod cloud access ttl = int(scope_row.default_grant_ttl_days or unreasoned_default_ttl) grant_row.expires_at = now_dt.replace(tzinfo=None) + timedelta(days=ttl) grant_row.last_notified_stage = None unreasoned_stamped += 1 if unreasoned_stamped: await session.commit() logger.info( "cloud.access_expiry_sweep: stamped a fresh TTL on %d grant(s) " "holding NULL expiry with no reason to (not a perpetual-role " "holder, not the scope's current owner) — left behind by an " "ownership transfer; they now re-enter the normal reminder cycle", unreasoned_stamped, ) # NOTE on scope-owner grants: their ``expires_at`` legitimately goes # stale (140 active rows were months past their stamped expiry on # 2026-07-29) because the owner exemption below skips them in both # passes. The stale date is a latent trap — the exemption is evaluated # against the scope's CURRENT owner, so on the day ownership is handed # over, the outgoing owner's rows stop being exempt while already long # past due, and the next sweep revokes them instantly with no T-14 / # T-7 / T-1 card (every reminder window is in the past too). # # That trap is disarmed at its cause — ``rebind_scope_owner`` re-stamps # the outgoing owner's grants with a fresh TTL — rather than by having # this sweep roll every owner grant forward nightly. Rolling here was # tried and rejected: it invents a meaningful-looking expiry for rows # that are in fact exempt, writes to the table every day for no # behavioural gain, and empties the ``owner_skipped`` counter that # on-call reads (owner rows would no longer reach either pass at all). # 1. Reminder pass — select grants whose expires_at falls in one # of the (now + N-day, now + (N-1)-day) buckets and haven't been # notified for that stage yet, then send ONE card PER PERSON # covering everything they have expiring in that bucket. # # The per-grant fan-out this replaces was measured on 2026-07-15: # 861 cards to 198 people in a single cron run, feishu AND email, # with one person (xiaopeng.zhu) receiving 63 cards in one day — # then 54 more on 07-17, 54 on 07-23 and 47 on 07-28. At that # volume the card stops being a notification and becomes noise # people filter, which defeats the entire point of warning before # revoking. # # ``sweep_inactive_grants`` learned exactly this lesson and moved # to (scope, person) grouping; its docstring even records "a person # with N grants received N notifications. Spam." The expiry sweep # never got the same fix. It does now, grouped one level wider — # per PERSON, not per (scope, person) — because a holder with roles # on eight accounts expiring the same day wants one message, not # eight. for days, stage in stages: window_start = now_dt + timedelta(days=days - 1) window_end = now_dt + timedelta(days=days) rows = (await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, CmdbCloudAccessGrant.expires_at.isnot(None), CmdbCloudAccessGrant.expires_at > window_start, CmdbCloudAccessGrant.expires_at <= window_end, or_( CmdbCloudAccessGrant.last_notified_stage.is_(None), CmdbCloudAccessGrant.last_notified_stage != stage, ), ) )).scalars().all() scanned += len(rows) # Bucket the eligible grants by holder before sending anything. by_person: dict[str, list[CmdbCloudAccessGrant]] = {} for grant in rows: if _is_owner_grant(grant): # Owner's own grant on their scope — no reminder spam. # The revoke pass below will also skip it. owner_skipped += 1 continue if _is_prod_grant(grant): # Production scope — not managed by Helix. No expiry # reminder (it would be false + un-actionable, see # ``_is_prod_grant``). The revoke pass skips it too. prod_reminder_skipped += 1 continue if _is_perpetual_grant(grant): # Cloud admin — perpetual. Nagging them to renew a grant # that is never revoked is the same false + un-actionable # card the prod exemption exists to suppress. perpetual_skipped += 1 continue if grant.person_id is None: continue by_person.setdefault(str(grant.person_id), []).append(grant) for person_id, person_grants in by_person.items(): # No-autoflush: ``last_notified_stage`` from a previous # iteration is dirty; a SELECT triggers autoflush → UPDATE # cmdb_cloud_access_grants, which can deadlock against a # concurrent renewal or admin-revoke on the same row. with session.no_autoflush: person = await session.get( IdentityPerson, uuid.UUID(person_id), ) scope_by_id: dict[str, CmdbCloudScope | None] = {} for g in person_grants: sid = str(g.scope_id) if sid not in scope_by_id: scope_by_id[sid] = await session.get( CmdbCloudScope, uuid.UUID(sid), ) if person is None or not person.work_email: continue renderable = [ g for g in person_grants if scope_by_id.get(str(g.scope_id)) is not None ] if not renderable: continue try: # SAVEPOINT (2026-08-06 review): scope this iteration's writes so a # DB-level failure rolls back only its own work. ``savepoint`` # re-raises everything, so the handler chain below keeps its exact # semantics — it only stops a failure from poisoning the session # for the remaining items and the run's trailing commit. async with savepoint(session, label="grant expiry scan item"): # Resolved ONCE per person, before any copy is built: this # card fans out to a single recipient, so unlike the card # chrome (which ``LocalizedText`` resolves per reader) the # title/body/line-items can all be composed directly in # the reader's language. # # Until 2026-08-25 they were not composed in any language # — they were hardcoded Chinese string literals, and only # the button below carried ``LocalizedText``. Production # 2026-07: a US-based engineer received all three # escalating reminders in Chinese, did not act on any of # them, and lost AWS access to an account holding his # team's BYOC configuration. 330 of 443 active employees # read English (see ``core.locale``), so a Chinese-only # expiry warning is the majority case, not the edge one. lang = resolve_lang(person) expires_suffix = { "zh": " 到期", "en": " expires", "ja": " 期限", }[lang] lines: list[str] = [] # Sorted by urgency, then by account, so the card reads # like a to-do list rather than DB order. for g in sorted( renderable, key=lambda x: ( x.expires_at or now_dt.replace(tzinfo=None), str(x.scope_id), ), ): sc = scope_by_id[str(g.scope_id)] assert sc is not None # filtered above # The date suffix is the only localizable token on the # line — account and role are provider-side identifiers # and stay verbatim in every language. lines.append( f" • {sc.display_name or sc.external_id} · " f"{g.role_display_name or g.role_key} · " f"{g.expires_at:%Y-%m-%d}{expires_suffix}" ) scope_ids = {str(g.scope_id) for g in renderable} # Deep-link: one account → straight to it; several → the # subscription list, because there is no single right # destination and guessing one hides the rest. url = ( f"/my/cloud/scopes/{next(iter(scope_ids))}" if len(scope_ids) == 1 else "/my/cloud/grants" ) # Per-person SAVEPOINT: a dispatch/outbox write that # aborts the transaction would otherwise leave the shared # session rollback-pending, silently no-op'ing every later # person + the caller's final commit. async with session.begin_nested(): await service.notify.dispatch( NotificationEnvelope( recipient=person, title=pick_variant({ "zh": ( f"[云访问即将到期] {len(renderable)} " f"项权限将在 {days} 天内到期" ), "en": ( f"[Cloud access expiring] " f"{len(renderable)} permission(s) " f"expire in {days} day(s)" ), "ja": ( f"[クラウドアクセス期限切れ間近] " f"{len(renderable)} 件の権限が " f"{days} 日以内に期限切れになります" ), }, lang) or "", body_text=pick_variant({ "zh": ( f"你有 {len(renderable)} 项云权限将在 " f"{days} 天内到期,涉及 " f"{len(scope_ids)} 个云账号:\n\n" + "\n".join(lines) + "\n\n如仍需使用,请提交续期申请 —— " "需要订阅 Owner 审批后才会延长。\n" "到期当天未获批准的权限会被自动回收。" ), "en": ( f"{len(renderable)} of your cloud " f"permissions expire within " f"{days} day(s), across " f"{len(scope_ids)} cloud account(s):" "\n\n" + "\n".join(lines) + "\n\nIf you still need this access, " "submit a renewal request — it is " "extended only after the account " "owner approves it.\nPermissions " "without an approved renewal are " "released automatically on the " "expiry date." ), "ja": ( f"あなたのクラウド権限 " f"{len(renderable)} 件が {days} " f"日以内に期限切れになります" f"(対象クラウドアカウント " f"{len(scope_ids)} 件):\n\n" + "\n".join(lines) + "\n\n継続して利用する場合は更新を" "申請してください —— サブスクリプ" "ションの Owner の承認後に延長され" "ます。\n承認されなかった権限は期限" "日に自動的に回収されます。" ), }, lang) or "", url=url, channels=["feishu", "email"], card_actions=[ CardAction( label=LocalizedText( zh="去申请续期", en="Request renewal", ja="更新を申請する", ), url=url, style="primary", ), ], # Anchored on the PERSON, not a single grant: # the card is about all of them, and picking # one arbitrarily would make the in-app # notification deep-link lie about scope. related_resource_type="identity_person", related_resource_id=person.id, ), trace_id="cloud.access_expiry_sweep", ) for g in renderable: g.last_notified_stage = stage # Commit per person, not once at the end of the sweep. # Every outbox INSERT above FK-references the recipient's # ``identity_persons`` row, and TiDB's FK check takes a # lock on that parent row that is held until COMMIT. A # sweep-wide transaction turned a mass reminder day # (2026-07-10: 821 notifications, 20-minute run) into a # login outage — each notified employee's # ``UPDATE identity_persons SET last_login_at`` blocked # until 1205 lock-wait timeout, so Google/Feishu login # 500'd platform-wide for the whole run. The sweep is # idempotent (``last_notified_stage`` guards re-sends), # so per-item durability is strictly better than batch # atomicity here. await session.commit() notified += 1 except Exception as e: # noqa: BLE001 logger.warning( "expiry notify failed for person %s: %s", person_id, e, ) errors.append( f"notify person {person_id}: " f"{format_exception_for_log(e)}" ) # 2. T+0 revoke pass — but skip prod scopes via the SAME # ``_is_prod_grant`` predicate as the reminder pass (both derive # from ``PROD_ENVIRONMENTS``; a hardcoded ``env == "prod"`` here # would silently diverge if the constant ever grows). Production # access expiring silently is too disruptive (page services in # the middle of an incident, lock SREs out of the box they need), # so prod grants are held active past expiry with no notification # to anyone. Dev/staging keep the existing revoke-on-expiry # default. Track the prod-skipped count separately so the daily # digest can surface "N prod grants are overdue, need attention". rows = (await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, CmdbCloudAccessGrant.expires_at.isnot(None), CmdbCloudAccessGrant.expires_at <= now_dt, ) )).scalars().all() scanned += len(rows) for grant in rows: if _is_owner_grant(grant): # Scope owner's own grant — never auto-revoked. Notifying the # owner that THEIR OWN grant has expired is the same spam loop # the owner exemption exists to prevent. owner_skipped += 1 continue if _is_prod_grant(grant): # ``_is_prod_grant`` reads the prefetched ``scope_env_map`` # (NULL environment coerced to "dev") and derives "is prod" # from PROD_ENVIRONMENTS via ``_is_prod_environment`` — the # SAME predicate as the reminder pass, so the two passes can # never diverge (a hardcoded "prod" literal here would # auto-REVOKE an environment the reminder pass exempts). # # Production access is NOT managed by Helix's lifecycle sweep. # We never auto-recycle it, AND — per the standing "don't # touch prod" rule — we send NO notification to anyone (not # the member, not the owner). The R&D security team owns prod # cloud access end-to-end. Keep only an internal count + a log # line (below) for oncall visibility; no user-facing push. prod_skipped += 1 continue if _is_perpetual_grant(grant): # Cloud admin — perpetual, never auto-revoked. Pass 0 should # already have cleared this row's ``expires_at`` (so it would # not even be selected here); reaching this branch means the # row appeared after pass 0 ran. Skip, and let the next run's # pass 0 normalize it. perpetual_skipped += 1 continue try: # SAVEPOINT (2026-08-06 review): scope this iteration's writes so a # DB-level failure rolls back only its own work. ``savepoint`` # re-raises everything, so the handler chain below keeps its exact # semantics — it only stops a failure from poisoning the session # for the remaining items and the run's trailing commit. async with savepoint(session, label="grant expiry revoke"): # Per-item SAVEPOINT: revoke_grant flushes to the shared # session (only its audit write commits separately); without # isolation a failure mid-revoke would poison every later # grant in the sweep. async with session.begin_nested(): await service.revoke_grant( str(grant.id), reason=GRANT_REVOKE_REASON_EXPIRED, actor_user_id=None, # type: ignore[arg-type] trace_id="cloud.access_expiry_sweep", suppress_driver_error=True, ) # Then flip the status to expired (distinguish from manual # revoke for reporting). grant.status = GRANT_STATUS_EXPIRED # Per-item commit — revoke_grant's audit write already # commits separately; holding the grant-row + notification # locks across the remaining loop only recreates the # 2026-07-10 lock pile-up (see reminder-pass comment). await session.commit() revoked += 1 if grant.person_id is not None: revoked_by_person.setdefault( str(grant.person_id), [], ).append(( str(grant.scope_id), grant.role_display_name or grant.role_key, )) except Exception as e: # noqa: BLE001 logger.exception("expiry revoke failed for grant %s", grant.id) errors.append( f"revoke {grant.id}: {format_exception_for_log(e)}" ) # 3. Tell the people whose access was just recycled. # # Until 2026-07-29 this pass sent NOTHING. The flow was: three # escalating reminder cards, then silence — access vanished on the # expiry date and the holder found out by getting denied in the AWS # console. ``notification_outbox`` confirmed it: every row with # ``related_resource_type='cmdb_cloud_access_grant'`` was a # 「云访问即将到期」 reminder; not one was a recycling notice. # # Aggregated per person for the same reason as the reminders, and # sent AFTER the revoke loop so a notification failure can never # leave access live while the holder was told it was gone. for person_id, items in revoked_by_person.items(): try: # SAVEPOINT (2026-08-06 review): scope this iteration's writes so a # DB-level failure rolls back only its own work. ``savepoint`` # re-raises everything, so the handler chain below keeps its exact # semantics — it only stops a failure from poisoning the session # for the remaining items and the run's trailing commit. async with savepoint(session, label="grant expiry notify"): person = await session.get(IdentityPerson, uuid.UUID(person_id)) if person is None or not person.work_email: continue lines = [] for scope_id, role_label in items: sc = await session.get(CmdbCloudScope, uuid.UUID(scope_id)) sc_label = ( (sc.display_name or sc.external_id) if sc else scope_id ) lines.append(f" • {sc_label} · {role_label}") # Same per-recipient resolution as the reminder pass above — # a revocation notice the reader cannot read is the reminder # bug one step later in the lifecycle, and this is the card # that tells them the access is ALREADY gone. lang = resolve_lang(person) async with session.begin_nested(): await service.notify.dispatch( NotificationEnvelope( recipient=person, title=pick_variant({ "zh": ( f"[云访问已回收] {len(items)} " "项权限到期已回收" ), "en": ( f"[Cloud access released] {len(items)} " "expired permission(s) were released" ), "ja": ( f"[クラウドアクセス回収済み] {len(items)} " "件の期限切れ権限を回収しました" ), }, lang) or "", body_text=pick_variant({ "zh": ( f"以下 {len(items)} 项云权限已到期," "已按策略回收:" f"\n\n" + "\n".join(lines) + "\n\n如仍需使用,可在订阅详情页提交续期" f"申请(回收后 " f"{RENEWAL_REVIVAL_MAX_AGE_DAYS} 天内" "仍可申请恢复,需订阅 Owner 审批);" "超过该期限请重新提交权限申请。" ), "en": ( f"The following {len(items)} cloud " "permission(s) reached their expiry date " "and were released by policy:" f"\n\n" + "\n".join(lines) + "\n\nIf you still need this access, you " "can request a restore from the account " "page — restores stay available for " f"{RENEWAL_REVIVAL_MAX_AGE_DAYS} days " "after release and need the account " "owner's approval. After that window, " "please submit a new access request." ), "ja": ( f"以下の {len(items)} 件のクラウド権限が" "期限切れとなり、ポリシーに従って回収され" "ました:" f"\n\n" + "\n".join(lines) + "\n\n継続して利用する場合は、サブスク" "リプション詳細ページから復旧を申請でき" f"ます(回収後 " f"{RENEWAL_REVIVAL_MAX_AGE_DAYS} 日以内、" "Owner の承認が必要)。この期間を過ぎた" "場合は、権限申請をあらためて提出して" "ください。" ), }, lang) or "", url="/my/cloud/grants", channels=["feishu", "email"], card_actions=[ CardAction( label=LocalizedText( zh="申请恢复访问", en="Request access restore", ja="アクセス復旧を申請する", ), url="/my/cloud/grants", style="primary", ), ], related_resource_type="identity_person", related_resource_id=person.id, ), trace_id="cloud.access_expiry_sweep", ) await session.commit() revoke_notified += 1 except Exception as e: # noqa: BLE001 # Never fail the sweep over a notification — the revoke is # already durable and correct. Surface in ``errors`` so the # adaptive threshold can page if this becomes systemic. logger.warning( "expiry revoke notice failed for person %s: %s", person_id, e, ) errors.append( f"revoke-notice {person_id}: {format_exception_for_log(e)}" ) if prod_skipped or prod_reminder_skipped: # Surface the prod counts so an oncall scanning logs sees "we're # sitting on N production grants past expiry, go look" — but note # prod is deliberately unmanaged (no auto-revoke, no push). The # same counts are in the return dict for any caller (digest job, # dashboard) that wants to render them; logging here guarantees # they land in stdout regardless. logger.info( "cloud.access_expiry_sweep: prod is unmanaged — skipped %d " "reminder(s) and held %d grant(s) past expires_at " "(env=prod: no auto-revoke, no notification)", prod_reminder_skipped, prod_skipped, ) return { # ``scanned`` is the union of grants visited across the # reminder windows (T-14 / T-7 / T-1) and the T+0 expiry # pass. The cron handler reads this to compute the # ``err_threshold = max(5, scanned * 0.10)`` adaptive paging # floor; without it the threshold collapses to a constant 5. "scanned": scanned, "notified": notified, "expired_revoked": revoked, "prod_skipped": prod_skipped, "prod_reminder_skipped": prod_reminder_skipped, "owner_skipped": owner_skipped, # Cloud-admin perpetual holders skipped across both passes, and # rows whose stale TTL pass 0 cleared. ``perpetual_normalized`` # should trend to 0 once the backfill migration has run — a # non-zero value on a steady-state run means some writer is still # stamping a TTL on a perpetual holder's row. "perpetual_skipped": perpetual_skipped, "perpetual_normalized": perpetual_normalized, # Rows healed by pass 0b: NULL expiry with no reason to hold it # (left behind by an ownership transfer). Steady state is 0. "unreasoned_null_expiry_stamped": unreasoned_stamped, # People told their access was recycled. Before 2026-07-29 this # was structurally 0 — the revoke pass sent nothing at all. "revoke_notified": revoke_notified, "errors": errors, } async def sweep_inactive_grants( session: AsyncSession, now: datetime | None = None, ) -> dict[str, Any]: """Three-stage inactivity sweep, group-by (scope, person): * T+0: a (scope, person) GROUP whose freshest active grant has ``last_login_at`` older than ``cloud.inactivity_threshold_days`` (default 60) — or all NULL on grants ALL granted before cutoff — → notify scope owner ONCE for the group, mark every grant in the group ``inactive_owner``. * T+14: group still ``inactive_owner`` and still no newer login → notify IT admins ONCE for the group, mark every grant ``inactive_it``. * T+30: group still ``inactive_it`` and still no newer login → disable every grant in the group (driver revoke, status='disabled'). Why group-by, not per-grant (FIXED 2026-05-05): Cloud IAM tracks login at the IAM USER level (AWS PasswordLastUsed / AccessKeyLastUsed; Aliyun RAM equivalent). There is no per-policy / per-role login data on any provider we ship. Operating per-grant produced two real false-alert classes: 1. The pre-fix refresh wrote ``last_login_at`` to ONE specific grant per (scope, person) — if it failed for one role of the same person, that role kept ``last_login_at = NULL`` even though the IAM user had logged in 7 days ago. The sweep then fired ``60+ 天无登录`` on, e.g., ``IAMUserChangePassword`` for a user who was actively using the account daily. Reported 2026-05-05 via the AWS SG li.da / 达丽 case. 2. Even when refresh succeeded uniformly, a person with N grants on a scope received N notifications (one per role) on the same inactivity event. Spam. Both go away when the unit of evaluation is the (scope, person) group with ``MAX(last_login_at)`` — one decision, one notification, consistent stage transitions across the whole group. Privileged-role exemption: people holding ``it_admin`` / ``global_admin`` / ``cloud_admin`` are skipped at every stage. Their grants represent platform-management access (the cloud admin legitimately holds a Reader-equivalent role on every Azure subscription / AWS account so they can investigate billing / incidents). The inactivity sweep used to flag those — a cloud admin with ``last_login_at IS NULL`` (Helix doesn't track Azure portal SSO logins) on dozens of scopes generated dozens of misleading "revoke this admin's access" notifications. See PR #170 follow-up where Kidd (cloud_admin) received 30+ false alerts. Scope-owner exemption (2026-05-12): the (scope, person) group is also skipped when ``person_id`` == that scope's ``owner_person_id``. The scope owner is the human responsible for approving every other member's grant on the scope; auto-disabling their own access on the 60+14+30 schedule strands them outside their own subscription (re-requesting a grant for a scope you own is busywork; the owner often has no one else to approve it). Symmetric to the expiry-sweep owner exemption added in the same PR. Production exemption (2026-07-16): (scope, person) groups on a production scope (``PROD_ENVIRONMENTS``, same predicate as the expiry sweep's ``_is_prod_grant``) are skipped at EVERY stage — no owner reminder, no IT escalation, no T+30 auto-disable. Prod cloud access is not managed by Helix lifecycle automation; the R&D security team owns it end-to-end (PR #1961 invariant, which the expiry sweep already honored). Counted as ``skipped_prod``. """ now_dt = now or datetime.now(UTC) threshold = await _setting_int(session, "cloud.inactivity_threshold_days", 60) escalation = await _setting_int(session, "cloud.inactivity_escalation_days", 14) disable_after = await _setting_int(session, "cloud.inactivity_disable_days", 30) cutoff_t0 = now_dt - timedelta(days=threshold) service = CloudLifecycleService(session) # Privileged roles that legitimately hold cloud grants as part of # their job. Any grant assigned to a person in this set is skipped # at every stage of the sweep. Resolved once per run so the per- # grant filter is a constant-time set lookup. exempt_role_names = ("it_admin", "global_admin", "cloud_admin") exempt_person_ids: set[str] = { str(pid) for pid in (await session.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthRole.name.in_(exempt_role_names)) )).scalars().all() } # Scope-owner exemption (per-(scope, person) pair, NOT global). # ``it_admin``/``cloud_admin`` exemption is keyed on the person # alone — once you're a cloud admin, every grant you hold is # skipped. Owner exemption is narrower: only the grant that lives # ON THE SCOPE YOU OWN is skipped. The same person, if granted on # someone else's scope, still flows through the normal inactivity # policy. scope_owner_map: dict[str, str | None] = {} scope_env_map: dict[str, str] = {} for sid, oid, env in (await session.execute( select( CmdbCloudScope.id, CmdbCloudScope.owner_person_id, CmdbCloudScope.environment, ) )).all(): scope_owner_map[str(sid)] = str(oid) if oid else None # NULL coercion rationale lives in ``_is_prod_environment``. scope_env_map[str(sid)] = env or "dev" def _is_owner_pair(scope_id: str, person_id: str) -> bool: owner_id = scope_owner_map.get(scope_id) return owner_id is not None and owner_id == person_id def _is_prod_pair(scope_id: str) -> bool: # Production cloud access is NOT managed by Helix's lifecycle # sweeps — same ``PROD_ENVIRONMENTS``-derived predicate as the # expiry sweep's ``_is_prod_grant`` (see ``_is_prod_environment`` # / PR #1961). Before 2026-07-16 this sweep had NO prod # exclusion: it sent owner/IT "60+ 天无登录" cards about prod # grants and at T+30 called ``driver.revoke_access`` against a # production account, bypassing the prod write lock (which is # router/MCP-scoped). Prod groups are skipped at EVERY stage — # no reminder, no IT escalation, no auto-disable. The R&D # security team owns prod cloud access end-to-end. return _is_prod_environment(scope_env_map.get(scope_id)) summary: dict[str, Any] = { "notified_owner": 0, "notified_it": 0, "disabled": 0, "skipped_privileged": 0, "skipped_owner": 0, # (scope, person) groups on a production scope, skipped at any # stage — prod is unmanaged by the sweep (see ``_is_prod_pair``). "skipped_prod": 0, # ``groups_*`` count distinct (scope, person) decisions, where # ``notified_*`` count individual notifications dispatched # (currently 1:1 with groups for owner-stage and equal to # group_count*admin_count for IT-stage). Surface both so the # cron history reflects both noise level and decision count. "groups_notified_owner": 0, "groups_notified_it": 0, "groups_disabled": 0, "errors": [], } # ── Pull all active grants once, group in Python ── # # Grouping in SQL via window functions is cleaner but our async # SQLAlchemy + MySQL/TiDB stack makes the per-grant FK loads # easier when we already have the rows materialized. The grant # table is bounded (a person has O(10) scopes, a scope has O(100) # people; total grants ~10k at our scale) — fitting in a single # ``SELECT`` is fine. all_active_grants = list((await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, ) )).scalars().all()) # Bucket by (scope_id, person_id). groups: dict[tuple[str, str], list[CmdbCloudAccessGrant]] = {} for g in all_active_grants: groups.setdefault((str(g.scope_id), str(g.person_id)), []).append(g) def _aware(t: datetime | None) -> datetime | None: """Coerce to UTC-aware. MySQL/TiDB returns naive datetimes even on TIMESTAMP columns; cutoffs are aware. Comparing them directly raises TypeError. Naive values are interpreted as UTC (the column convention). """ if t is None: return None if t.tzinfo is None: return t.replace(tzinfo=UTC) return t def _group_last_login(grants: list[CmdbCloudAccessGrant]) -> datetime | None: """MAX(last_login_at) across the group; None iff ALL are NULL.""" candidates = [ _aware(g.last_login_at) for g in grants if g.last_login_at is not None ] return max(candidates) if candidates else None def _group_freshest_granted_at( grants: list[CmdbCloudAccessGrant], ) -> datetime | None: """MAX(granted_at, created_at fallback) across the group. Used for the "all NULL last_login_at" branch — a group is inactive only if EVERY grant was issued before the cutoff. That's equivalent to "the FRESHEST grant in the group is older than cutoff" (max < cutoff implies all < cutoff). A brand-new grant added five minutes ago should keep the group safe, even if the older roles in the group were granted long enough ago to look stale on their own. Original implementation returned ``min`` and checked ``oldest < cutoff`` — that fires whenever ANY single grant is old, regardless of newer additions, which is the opposite of the intent. Fixed 2026-05-05 review-pass HIGH. """ ts: list[datetime] = [] for g in grants: t = _aware(g.granted_at) or _aware(g.created_at) if t is not None: ts.append(t) return max(ts) if ts else None def _group_is_inactive( grants: list[CmdbCloudAccessGrant], ) -> bool: """Group inactive criterion. Symmetric with the per-grant criterion this function replaces, but evaluated on the group aggregate so a single fresh login by the IAM user keeps every grant in the group safe. """ gll = _group_last_login(grants) if gll is not None: return gll < cutoff_t0 # All NULL last_login_at — only inactive if EVERY grant was # issued before the cutoff. That's equivalent to "the # freshest granted_at is < cutoff" (max < cutoff ⇒ all < # cutoff). A brand-new grant added today must keep the # whole group safe even if the older roles in the bundle # are long-aged on their own. freshest = _group_freshest_granted_at(grants) return freshest is not None and freshest < cutoff_t0 def _group_stage( grants: list[CmdbCloudAccessGrant], ) -> str | None: """Aggregate stage: ``inactive_it`` if ANY grant in the group carries that flag, else ``inactive_owner`` if ANY does, else None. Stages advance monotonically per group, so this rolls the group's max-progress stage up correctly. """ stages = {g.last_notified_stage for g in grants} if GRANT_NOTIFY_INACTIVE_IT in stages: return GRANT_NOTIFY_INACTIVE_IT if GRANT_NOTIFY_INACTIVE_OWNER in stages: return GRANT_NOTIFY_INACTIVE_OWNER return None # Snapshot ``updated_at`` per grant BEFORE any mutation in this # run. Two reasons: # # 1. Stage 0's pending mutation triggers an autoflush at the next # SELECT/get; the ``onupdate=func.now()`` server-side default # bumps updated_at, SQLAlchemy expires the column from cache, # and the next ``g.updated_at`` access becomes a sync IO # lazy-load that fails inside the async greenlet # (``MissingGreenlet``). Snapshotting upfront sidesteps the # cache invalidation entirely. # 2. Even without the lazy-load issue, Stage 1 needs to know # "when did the owner stamp land?" — the answer must be the # PRE-run timestamp, not "five microseconds ago because Stage 0 # just bumped it". Same for Stage 2 vs Stage 1. pre_run_updated_at: dict[str, datetime | None] = { str(g.id): _aware(g.updated_at) for g in all_active_grants } def _group_max_pre_run_updated_at( grants: list[CmdbCloudAccessGrant], ) -> datetime | None: ts = [ pre_run_updated_at.get(str(g.id)) for g in grants ] ts = [t for t in ts if t is not None] return max(ts) if ts else None # Groups whose stage was advanced in THIS run — exclude from # later-stage processing on the same tick. Without this guard a # group could go None → inactive_owner → inactive_it → disabled # in a single cron run, collapsing the policy's 60+14+30 days # into one tick. advanced_this_run: set[tuple[str, str]] = set() # ── Stage 0 → inactive_owner ── for (scope_id, person_id), gs in groups.items(): if _is_prod_pair(scope_id): # Prod scope — unmanaged, no inactivity reminder. A person # with BOTH prod and non-prod grants still gets normal # handling on their non-prod groups (grouping is per-scope). summary["skipped_prod"] += 1 continue if person_id in exempt_person_ids: summary["skipped_privileged"] += 1 continue if _is_owner_pair(scope_id, person_id): # Owner of this scope — exempt. Don't advance stage, don't # spam the owner with "you're inactive on your own scope". summary["skipped_owner"] += 1 continue if _group_stage(gs) is not None: # Already past Stage 0 — handled by Stage 1/2 branches. continue if not _group_is_inactive(gs): continue scope = await session.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or not scope.owner_person_id: continue owner = await session.get( IdentityPerson, uuid.UUID(str(scope.owner_person_id)), ) if ( owner is None or owner.work_email is None or owner.employment_status in ("left", "leaver_pending") ): continue person = await session.get(IdentityPerson, uuid.UUID(person_id)) # Pick a "representative" grant for the notification's # related_resource_id — the email body lists all roles in the # group, but the link needs to point somewhere. Use the # most-recently-touched row for relevance. rep = max(gs, key=lambda g: g.updated_at or g.created_at) role_lines = "\n".join( f" • {g.role_display_name or g.role_key}" for g in gs ) gll = _group_last_login(gs) gll_text = gll.isoformat() if gll else "(从未登录)" async with best_effort( session, label="cloud: inactive-grant owner notification", log=logger, ) as be_8256_step: await service.notify.dispatch( NotificationEnvelope( recipient=owner, title=( f"[云账号不活跃] {person.display_name if person else person_id} " f"在 {scope.display_name or scope.external_id} " f"已 {threshold}+ 天无登录" ), body_text=( f"Scope: {scope.display_name or scope.external_id}\n" f"成员: {person.display_name if person else person_id}\n" f"涉及 {len(gs)} 个角色:\n{role_lines}\n" f"上次登录: {gll_text}\n\n" f"如仍需保留,请确认;{escalation} 天后若仍无活动,我们会" f"通知 IT,最终 {disable_after} 天后将自动 disable " f"这 {len(gs)} 个 grant。" ), url=f"/cloud/scopes/{scope.id}", channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=rep.id, ), trace_id="cloud.scope_inactivity_sweep", ) for g in gs: g.last_notified_stage = GRANT_NOTIFY_INACTIVE_OWNER advanced_this_run.add((scope_id, person_id)) summary["groups_notified_owner"] += 1 summary["notified_owner"] += 1 # Commit per group: the outbox INSERT above FK-locks the # owner's identity_persons row until COMMIT (TiDB FK # check). One sweep-wide transaction across hundreds of # dispatches blocked those owners' logins for the whole # run — same mechanism as the 2026-07-10 # ``sweep_expiring_grants`` login outage. Stage # transitions are idempotent (``last_notified_stage`` # guards), so per-group durability is safe. await session.commit() if not be_8256_step.ok: summary["errors"].append( f"notify owner scope={scope_id} person={person_id}: {be_8256_step.reason()}" ) # ── Stage inactive_owner → inactive_it ── admin_ids = list((await session.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthRole.name.in_(("global_admin", "it_admin"))) )).scalars().all()) admins: list[IdentityPerson] = [] for aid in admin_ids: p = await session.get(IdentityPerson, aid) if p and p.work_email and p.deleted_at is None and p.employment_status == "active": admins.append(p) # Empty admin set → every Stage 1 group would write a per-group # error "no admin notification succeeded — group held". Surface # the underlying cause once and short-circuit Stage 1 entirely. # Stage 0 already ran (groups got their owner-stage notification); # Stage 2 can't fire this tick because nothing reaches # ``inactive_it``. Operator sees one warning, not N copies. skip_stage1 = len(admins) == 0 if skip_stage1: summary["errors"].append( "no IT admins (global_admin/it_admin) with valid work_email " "are active — Stage 1 (inactive_owner → inactive_it) skipped " "this run; restore at least one admin and the next tick will " "advance any stuck groups" ) # Refresh (last_login_at, updated_at, status) for every active # grant from DB before Stage 1 and Stage 2 decisions. Covers two # races during a long sweep run: # • An admin renewing a grant (bumps updated_at) between # Stage 0 and Stage 1 — the policy clock should reset, not # run on the snapshot value. # • ``refresh_last_login_for_grants`` writing a fresher # last_login_at concurrently — Stage 1's "still inactive?" # recheck should see it. # Note on autoflush: this SELECT triggers SQLAlchemy autoflush of # any pending mutations, including Stage 0's ``last_notified_stage`` # writes on advanced groups. The MySQL/TiDB ``onupdate=func.now()`` # default then bumps ``updated_at`` on those same rows, so the # fresh-state read for Stage-0-advanced groups returns a freshly- # bumped ``updated_at``. That's harmless because every Stage-0- # advanced group is gated by ``advanced_this_run`` below and # skipped at Stage 1 / Stage 2 regardless of what its updated_at # says — but a future maintainer who removes the # ``advanced_this_run`` guard MUST also re-evaluate this comment. async def _fetch_fresh_grant_state( ids: list[str], ) -> dict[str, dict[str, Any]]: if not ids: return {} rows = (await session.execute( select( CmdbCloudAccessGrant.id, CmdbCloudAccessGrant.last_login_at, CmdbCloudAccessGrant.updated_at, CmdbCloudAccessGrant.status, ).where( CmdbCloudAccessGrant.id.in_( [uuid.UUID(gid) for gid in ids] ), ) )).all() return { str(r.id): { "last_login_at": _aware(r.last_login_at), "updated_at": _aware(r.updated_at), "status": r.status, } for r in rows } def _fresh_group_last_login( grants: list[CmdbCloudAccessGrant], state: dict[str, dict[str, Any]], ) -> datetime | None: ts = [ state[str(g.id)]["last_login_at"] for g in grants if str(g.id) in state and state[str(g.id)]["last_login_at"] is not None ] return max(ts) if ts else None def _fresh_group_max_updated_at( grants: list[CmdbCloudAccessGrant], state: dict[str, dict[str, Any]], ) -> datetime | None: ts = [ state[str(g.id)]["updated_at"] for g in grants if str(g.id) in state and state[str(g.id)]["updated_at"] is not None ] return max(ts) if ts else None def _fresh_group_active_grants( grants: list[CmdbCloudAccessGrant], state: dict[str, dict[str, Any]], ) -> list[CmdbCloudAccessGrant]: return [ g for g in grants if state.get(str(g.id), {}).get("status") == GRANT_STATUS_ACTIVE ] all_grant_ids = [ str(g.id) for gs in groups.values() for g in gs ] stage1_state: dict[str, dict[str, Any]] = ( {} if skip_stage1 else await _fetch_fresh_grant_state(all_grant_ids) ) escalation_cutoff = now_dt - timedelta(days=escalation) for (scope_id, person_id), gs in groups.items(): if skip_stage1: break if _is_prod_pair(scope_id): # Prod scope — unmanaged. Stage 0 already refuses to stamp # prod groups, so this only fires on a stale # ``inactive_owner`` flag from before the prod exclusion # landed (or a scope re-classified to prod mid-policy) — # either way the group must not escalate to IT. summary["skipped_prod"] += 1 continue if person_id in exempt_person_ids: # Stage 0 already skips privileged groups, so a group # whose person is currently in ``exempt_person_ids`` # cannot have advanced this tick. The carve-out still # matters for groups carrying a stale ``inactive_owner`` # flag from a prior run (e.g. assignment added after # migration 118 cleared the prior backlog, or the # person was promoted to ``cloud_admin`` between # ``sweep_inactive_grants`` runs). summary["skipped_privileged"] += 1 continue if _is_owner_pair(scope_id, person_id): # Owner of this scope — same late-rebind case as the # privileged-role branch above. A group carrying a stale # ``inactive_owner`` flag from before the owner was # transferred to this person must not escalate to IT. summary["skipped_owner"] += 1 continue if (scope_id, person_id) in advanced_this_run: # Stage 0 advanced this group THIS tick — don't double- # process it through Stage 1 in the same run. Otherwise # a brand-new inactivity event would skip the # ``escalation`` waiting period entirely. continue if _group_stage(gs) != GRANT_NOTIFY_INACTIVE_OWNER: continue # Re-evaluate "still inactive?" against fresh DB state; an # admin renewing or last_login refresh during the sweep # should defer the group out of Stage 1 here. active_gs = _fresh_group_active_grants(gs, stage1_state) if not active_gs: continue gll = _fresh_group_last_login(active_gs, stage1_state) if gll is None or gll >= cutoff_t0: continue max_upd = _fresh_group_max_updated_at(active_gs, stage1_state) if max_upd is None or max_upd >= escalation_cutoff: # Fresh ``updated_at`` past the escalation cutoff means # the prior owner-stage stamp was bumped by a recent # write (Stage 0 of THIS run is excluded above by # ``advanced_this_run``; in practice this is an admin # renewal landing during the run). Defer. continue gs = active_gs scope = await session.get(CmdbCloudScope, uuid.UUID(scope_id)) person = await session.get(IdentityPerson, uuid.UUID(person_id)) if scope is None: continue rep = max(gs, key=lambda g: g.updated_at or g.created_at) role_lines = "\n".join( f" • {g.role_display_name or g.role_key}" for g in gs ) # Track at-least-one-success across the per-admin loop. Without # this guard, a group whose IT-stage notifications ALL failed # (every admin's Feishu+email dispatch raised) would still # advance to ``inactive_it`` on the strength of zero human # eyes seeing the alert, and Stage 2 would then auto-disable # the grants on the next tick. That's a worst-case silent # outage. Guard ensures at least one admin actually got the # message before we charge the policy clock forward. dispatched_at_least_one = False for admin in admins: try: await service.notify.dispatch( NotificationEnvelope( recipient=admin, title=( f"[IT 升级] {person.display_name if person else person_id} " f"在 {scope.display_name or scope.external_id} 长期不活跃" ), body_text=( f"Owner 已被通知 {escalation}+ 天仍无处理。\n" f"涉及 {len(gs)} 个角色:\n{role_lines}\n" f"再 {disable_after - escalation} 天后该 person 在该 " f"scope 的全部 grant 将自动 disable。" ), url=f"/admin/cloud/scopes/{scope.id}?tab=members", channels=["feishu", "email"], related_resource_type="cmdb_cloud_access_grant", related_resource_id=rep.id, ), trace_id="cloud.scope_inactivity_sweep", ) summary["notified_it"] += 1 dispatched_at_least_one = True except Exception as e: # noqa: BLE001 summary["errors"].append( f"notify IT scope={scope_id} person={person_id} " f"admin={admin.id}: {e}" ) if not dispatched_at_least_one: # No human got the IT-stage alert. Leave the group at # ``inactive_owner`` for the next tick — surface the # decision to skip in the error list so the operator # notices the dispatch outage. summary["errors"].append( f"notify IT scope={scope_id} person={person_id}: " f"no admin notification succeeded — group held at " f"inactive_owner for next tick" ) continue for g in gs: g.last_notified_stage = GRANT_NOTIFY_INACTIVE_IT advanced_this_run.add((scope_id, person_id)) summary["groups_notified_it"] += 1 # Per-group commit — releases the admins' identity_persons # FK locks taken by the outbox INSERTs above (see Stage 0 # comment / 2026-07-10 login outage). await session.commit() # ── Stage inactive_it → disable ── # Re-fetch fresh state again — Stage 1 mutated last_notified_stage # on its advanced groups (those are guarded by ``advanced_this_run`` # below) but other admin actions (renewal, manual revoke) may have # landed since the Stage 1 fetch. stage2_state = await _fetch_fresh_grant_state(all_grant_ids) disable_cutoff = now_dt - timedelta(days=disable_after) for (scope_id, person_id), gs in groups.items(): if _is_prod_pair(scope_id): # Prod scope — NEVER auto-disable. A stale ``inactive_it`` # flag (pre-exclusion backlog / scope re-classified to prod) # must not translate into a ``driver.revoke_access`` call # against a production account. Belt-and-braces: even if # this guard regressed, ``disable_grant_due_to_inactivity`` # itself refuses prod grants. summary["skipped_prod"] += 1 continue if person_id in exempt_person_ids: # Same late-promotion case as Stage 1: a person promoted # to ``cloud_admin`` between ``sweep_inactive_grants`` # runs can carry a stale ``inactive_it`` flag. Disabling # a privileged user's legitimate scope grant is a real # outage (loses Reader access on every Azure subscription # / AWS account they need to investigate billing); refuse # it always. summary["skipped_privileged"] += 1 continue if _is_owner_pair(scope_id, person_id): # Owner exemption — never auto-disable a scope owner's # access to their own scope, even if a stale ``inactive_it`` # flag persisted from before the owner rebind. summary["skipped_owner"] += 1 continue if (scope_id, person_id) in advanced_this_run: # Stage 0 or Stage 1 already advanced this group this # tick — don't disable on the same run. continue if _group_stage(gs) != GRANT_NOTIFY_INACTIVE_IT: continue active_gs = _fresh_group_active_grants(gs, stage2_state) if not active_gs: continue gll = _fresh_group_last_login(active_gs, stage2_state) if gll is None or gll >= cutoff_t0: continue max_upd = _fresh_group_max_updated_at(active_gs, stage2_state) if max_upd is None or max_upd >= disable_cutoff: continue gs = active_gs disabled_at_least_one = False for grant in gs: try: _, did_disable = await service.disable_grant_due_to_inactivity( str(grant.id), trace_id="cloud.scope_inactivity_sweep", expected_updated_at_le=stage2_state.get( str(grant.id), {}, ).get("updated_at"), ) if did_disable: summary["disabled"] += 1 disabled_at_least_one = True except Exception as e: # noqa: BLE001 summary["errors"].append( f"disable {grant.id}: {format_exception_for_log(e)}" ) if disabled_at_least_one: # Group counter only ticks on at-least-one real disable # this tick. Mirrors the at-least-one-success gate that # Stage 1 applies for IT-admin notifications: counters # must reflect what actually happened, not what we tried. summary["groups_disabled"] += 1 # Per-group commit — disable_grant_due_to_inactivity calls # the cloud driver over HTTP; don't hold this group's row # locks while later groups make their own driver calls # (see Stage 0 comment / 2026-07-10 login outage). await session.commit() return summary # ── Leaver grant sweep — safety net for the offboarding pipeline ── async def sweep_active_grants_on_left_persons() -> dict[str, Any]: """Find every active / pending cloud_access_grant whose holder is ``employment_status='left'`` and revoke it. This is the daily safety net behind the T+0 force-transfer in ``hr_lifecycle.advance_service``. The primary path runs every 15 min and revokes per-blocker through the normal RBAC flow; this sweep catches the residuals it could not handle: * Historical leavers who departed before the offboarding pipeline learned how to revoke cloud_access_grants (P0546-class rows). * Grants that materialised on the cloud side AFTER the person transitioned to ``left`` (drift surfaced by ``cmdb.grants_sync``). * Blockers that the primary path marked ``skipped`` after a manual decision — the local row stays active and the cloud-side role assignment lingers until the next sweep. No ``recent_days`` cutoff (cf. ``post_leaver_verify``). The sweep is cheap (one indexed query + per-grant cloud call) and correctness requires it to cover ALL left persons, not just recent ones. Each grant revoke runs in its own DB session + commit so one driver hang doesn't hold the whole sweep transaction open. The discovery query opens a session and closes it before the per-grant loop starts. ``suppress_driver_error=True`` is intentional: the Helix-side invariant (a leaver MUST NOT own an active grant in our records) is more important than the provider-side one. Cloud-side failures flip ``requires_manual_cloud_revoke=True`` on the row so the drift / orphan-principal sweeps re-try the provider call. Without the suppress flag, one Azure 429 / GCP propagation-delay 403 would leave the local row active and the leaver visible as an orphan in the daily T+1 verify run, generating noise that no admin can clear until the cloud cooperates. """ # ``select`` + ``IdentityPerson`` are already imported at module # level (lines 34, 77) — no in-function shim needed. # Step 1: discover. Read-only join across persons + grants; the # result is a flat list of grant_ids so the per-grant loop below # doesn't depend on the discovery session's lifetime. async with async_session_factory() as discovery_session: rows = (await discovery_session.execute( select(CmdbCloudAccessGrant.id) .join( IdentityPerson, IdentityPerson.id == CmdbCloudAccessGrant.person_id, ) .where( IdentityPerson.employment_status == "left", IdentityPerson.deleted_at.is_(None), CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ) )).scalars().all() grant_ids = [str(gid) for gid in rows] summary: dict[str, Any] = { "scanned": len(grant_ids), "revoked": 0, "cloud_side_failed": 0, "errors": [], } if not grant_ids: return summary # Round-27 pre-launch (2026-05-25): cap mass-revoke blast radius. # A single bad HRIS sync that mass-flips actives to "left", or an # ad-hoc SQL fix that touches too many rows, would otherwise cascade # into a fleet-wide cloud-grant revoke on the next cron tick — and # ``suppress_driver_error=True`` below flips Helix-side # ``status=revoked`` even when the cloud rejects, making rollback # expensive. Mirror the OrphanCleanupRequest cap pattern # (``lifecycle_router.py:3975``: max 500). Refuse + page on-call so # IT can verify before greenlighting the storm. _LEAVER_SWEEP_MAX_PER_RUN = 50 if len(grant_ids) > _LEAVER_SWEEP_MAX_PER_RUN: # Surface the blast-radius via RuntimeError so the cron's # status flips to ``failed`` and the failure_notifier pages # IT. Manual investigation + force-flag (or admin batch # processing via ``revoke_grant`` calls) is required. raise RuntimeError( f"cmdb.leaver_grant_sweep: {len(grant_ids)} grants pending " "revoke for leaver persons exceeds the per-run cap " f"({_LEAVER_SWEEP_MAX_PER_RUN}). This usually indicates a " "bad HRIS sync or ad-hoc DB write that mass-flipped persons " "to ``employment_status=left``. Manually verify the leaver " "list (SELECT id, work_email, employment_status FROM " "identity_persons WHERE employment_status='left' AND " "deleted_at IS NULL) before proceeding; once confirmed, " "process via the per-grant admin revoke API or raise the " "cap in source for a controlled run.", ) # Step 2: revoke each. Per-grant session so a slow cloud call on # one grant doesn't serialize the whole sweep behind a fat # transaction. for grant_id in grant_ids: try: async with async_session_factory() as inner_session: inner_service = CloudLifecycleService(inner_session) grant = await inner_service.revoke_grant( grant_id, reason=GRANT_REVOKE_REASON_OFFBOARDING, actor_user_id=None, # type: ignore[arg-type] trace_id="cmdb.leaver_grant_sweep", suppress_driver_error=True, ) await inner_session.commit() summary["revoked"] += 1 # ``requires_manual_cloud_revoke`` lives in the grant.detail # JSON sidecar (see ``_revoke_grant_locked`` ~line 2260) — # it is NOT an ORM column. A naive ``getattr(grant, "...", # False)`` would always return False and silently dead-code # the counter. Defending against the same family of bug # noted in feedback_no_getattr_default_for_model_columns. if (grant.detail or {}).get("requires_manual_cloud_revoke"): summary["cloud_side_failed"] += 1 except Exception as e: # noqa: BLE001 logger.exception( "leaver_grant_sweep: revoke failed for grant %s", grant_id, ) summary["errors"].append( f"grant {grant_id}: {format_exception_for_log(e)}" ) return summary # ── Manual-revoke retry sweep ─────────────────────────────── async def sweep_pending_manual_cloud_revokes( *, min_age_seconds: int = 3600, ) -> dict[str, Any]: """Retry cloud-side revoke for grants where the local row is ``status=revoked`` but the driver call previously failed. Closes the leak surfaced by round-N pre-launch audit (CR4 P1): ``_revoke_grant_locked(suppress_driver_error=True)`` stamps the Helix row REVOKED + ``detail.requires_manual_cloud_revoke=True`` whenever the cloud-side call failed (5xx / throttle / IAM lock / propagation delay). Without this sweep the flag is purely advisory — no cron ever re-attempts, so a leaver's cloud-side permission persists indefinitely. Guard: ``min_age_seconds`` (default 1 h) gives transient cloud issues time to clear before we hammer the API again, and avoids racing the original revoke transaction's commit. Each retry runs in its own DB session; provider failures are swallowed per-grant (still left as ``requires_manual_cloud_revoke`` for the next pass). """ cutoff = datetime.now(UTC) - timedelta(seconds=min_age_seconds) async with async_session_factory() as discovery_session: # JSON_EXTRACT path: MySQL 8 / TiDB syntax matches the writer # at ``_revoke_grant_locked`` ~line 2540 where we store the # flag under ``detail.requires_manual_cloud_revoke``. rows = (await discovery_session.execute( select(CmdbCloudAccessGrant.id) .where( CmdbCloudAccessGrant.status == GRANT_STATUS_REVOKED, CmdbCloudAccessGrant.revoked_at < cutoff, func.json_extract( CmdbCloudAccessGrant.detail, "$.requires_manual_cloud_revoke", ) == sa_true(), # Storm cap (2026-06-10): escalated grants are OFF the # automated retry treadmill — a human was paged and owns # the next step; the admin "重试云侧回收" button resets # the counter + stamp to resume sweeping. JSON_EXTRACT # returns SQL NULL for a missing key, so IS NULL matches # both legacy rows (no key) and post-reset rows. func.json_extract( CmdbCloudAccessGrant.detail, "$.manual_revoke_escalated_at", ).is_(None), ) )).scalars().all() escalated_pending = (await discovery_session.execute( select(func.count(CmdbCloudAccessGrant.id)) .where( CmdbCloudAccessGrant.status == GRANT_STATUS_REVOKED, func.json_extract( CmdbCloudAccessGrant.detail, "$.requires_manual_cloud_revoke", ) == sa_true(), func.json_extract( CmdbCloudAccessGrant.detail, "$.manual_revoke_escalated_at", ).isnot(None), ) )).scalar() or 0 grant_ids = [str(gid) for gid in rows] summary: dict[str, Any] = { "scanned": len(grant_ids), "cleared": 0, "still_failing": 0, # Grants paged-and-parked, awaiting a human in the cloud # console + the admin reset button. Surfaced in the job's # result_summary so the backlog is visible per tick. "escalated_pending": int(escalated_pending), "errors": [], } if not grant_ids: return summary # Round-1 review follow-up (Review-B P1): wrap each per-grant retry # in the SAME ``distributed_lock("grant_revoke:{grant_id}", ttl=120)`` # that ``revoke_grant`` holds. Without it, an admin's manual revoke # click concurrent with this sweep would call ``driver.revoke_access`` # twice on the same grant → double audit rows for one logical action. # LockBusy → skip silently (the lock-holder is doing the same work). for grant_id in grant_ids: try: async with distributed_lock( f"grant_revoke:{grant_id}", ttl=120, ): async with async_session_factory() as inner_session: try: inner_service = CloudLifecycleService(inner_session) cleared = await inner_service.retry_manual_cloud_revoke( grant_id, trace_id="cmdb.manual_revoke_retry_sweep", ) await inner_session.commit() if cleared: summary["cleared"] += 1 else: summary["still_failing"] += 1 except Exception as e: # noqa: BLE001 logger.exception( "manual_revoke_retry_sweep: grant %s failed", grant_id, ) # Round-2 review follow-up (SS2 P2-2): cap the # str(e) length to keep ``scheduled_job_runs. # summary`` bounded — DB exception messages can # carry bind-param values + stack hints. Type # name + 200 chars is enough forensic signal. summary["errors"].append( f"grant {grant_id}: " f"{type(e).__name__}: {str(e)[:200]}" ) try: await inner_session.rollback() except Exception: # noqa: BLE001 pass except LockBusy: # Concurrent admin revoke or another sweep tick is handling # this exact grant — skip silently; the lock holder's outcome # is the source of truth, our next tick will re-evaluate. logger.info( "manual_revoke_retry_sweep: grant %s locked by another " "actor, skipping this tick", grant_id, ) summary["still_failing"] += 1 # Round-3 review follow-up (LS3 P2-B): the hourly raise condition # ``still_failing == scanned AND cleared == 0`` doesn't fire when # ``len(errors) < scanned`` (the typical "driver returns ok=False # but never raises" plateau). Long-stuck grants (>7 days) would # otherwise silently never page. Raise a SEVERITY_HIGH compliance # incident per stuck grant so operator UI surfaces the backlog # even when the cron-handler dedup rules don't trigger. Dedup is # per-grant via uuid5 — same grant stuck for 30 days = 1 row, # detail refreshed. # Round-4 review follow-up (P2-1): per-row independent session. # Pre-fix shared session would enter PendingRollback on first # IntegrityError and silently drop every subsequent row + the # final commit. Now each row has its own session (mirroring # orphan_cleanup's pattern) — one failure is contained to that # row, audit summary stays honest. Also split counters so # ``aging_seen`` (rows discovered) vs ``aging_incidents_written`` # (incidents actually persisted) don't lie when writes fail. aging_cutoff = datetime.now(UTC) - timedelta(days=7) aging_seen = 0 aging_incidents_written = 0 async with async_session_factory() as discovery_aging_session: aging_rows = (await discovery_aging_session.execute( select(CmdbCloudAccessGrant.id, CmdbCloudAccessGrant.revoked_at) .where( CmdbCloudAccessGrant.status == GRANT_STATUS_REVOKED, CmdbCloudAccessGrant.revoked_at < aging_cutoff, func.json_extract( CmdbCloudAccessGrant.detail, "$.requires_manual_cloud_revoke", ) == sa_true(), ) )).all() if aging_rows: import uuid as _uuid from app.compliance.models import SEVERITY_CRITICAL from app.compliance.service import ComplianceIncidentService for row in aging_rows: aging_seen += 1 dedup_id = _uuid.uuid5( _uuid.NAMESPACE_OID, f"manual_revoke_retry_aging:{row.id}", ) try: async with async_session_factory() as inc_session: inc_svc = ComplianceIncidentService(inc_session) await inc_svc.create_or_upsert_incident( incident_type="cloud_grant_manual_revoke_stuck", severity=SEVERITY_CRITICAL, person_id=None, resource_type="cmdb_cloud_access_grant", resource_id=dedup_id, resource_label=f"grant={row.id}", title=( f"Cloud-side revoke stuck >7d on grant " f"{row.id} (revoked_at={row.revoked_at})" )[:255], detail={ "grant_id": str(row.id), "revoked_at": ( row.revoked_at.isoformat() if row.revoked_at else None ), }, ) await inc_session.commit() aging_incidents_written += 1 except Exception: # noqa: BLE001 logger.exception( "manual_revoke_retry_sweep: aging " "compliance_incident write failed for grant=%s", row.id, ) summary["aging_seen"] = aging_seen summary["aging_incidents_written"] = aging_incidents_written # Back-compat: keep the original field name so dashboards / queries # built on top of the Round-3 commit keep working. Reflects what # the OLD code intended (rows discovered), now with accurate count. summary["aging_unrevoked"] = aging_seen return summary # ── Scope sync (called by cmdb.scopes_sync handler) ───────── async def sync_scopes_for_integration( integration_id: str, ) -> dict[str, Any]: """Pull the tenant's scope list from the driver and upsert. Orphan scopes (present in DB but not in driver response) are marked status=closed (not deleted — the billing history is worth keeping). New scopes with no owner get a compliance_incident and reject grants until IT assigns one. """ async with async_session_factory() as session: integration = await session.get(Integration, uuid.UUID(integration_id)) if integration is None or integration.kind != "cloud": return {"status": "skipped", "reason": "not a cloud integration"} # Mirror the guard already enforced in # ``sync_grants_for_integration`` (line ~5739). Disabling a # cloud integration must keep the WHOLE sync triad # (scopes / roles / grants) quiet, not just the grants sweep. # Pre-fix: an admin disabled a misbehaving integration # (revoked credentials, sub-account decommission) but the # scope-sync cron kept loading it; the driver call returned # empty / errors, the drop-tolerance guard caught total wipes # but partial drops within the tolerance band slipped through, # mass-closing legitimate scopes / soft-deleting roles # dropdown-wide. The drop-tolerance floor (configurable via # ``platform_settings['cmdb_sync.sweep_drop_tolerance_percent']``, # default 20) is a backstop for driver bugs, NOT a bypass # for "this integration shouldn't be syncing at all." if not integration.enabled: return {"status": "skipped", "reason": "integration disabled"} driver = await _load_cloud_driver(session, integration) try: scope_infos = await driver.list_scopes() partition_reader = getattr( driver, "scope_snapshot_partitions", None, ) scope_partitions = ( partition_reader() if callable(partition_reader) else {} ) except NotImplementedError: return {"status": "skipped", "reason": "driver has no list_scopes"} except Exception as e: # noqa: BLE001 logger.exception("list_scopes failed for %s", integration.name) return {"status": "failed", "error": format_exception_for_log(e, max_len=400)} finally: try: await driver.close() except Exception: # noqa: BLE001 pass partition_result = { scope_type: { "complete": state.complete, "error": state.error, } for scope_type, state in scope_partitions.items() } incomplete_scope_types = { scope_type for scope_type, state in scope_partitions.items() if not state.complete } ignored_incomplete_scope_partitions: list[str] = [] if incomplete_scope_types: # #2282: the GCP driver merges project and billing-account # enumerations from independent APIs. A successful project leg # must not turn a failed billing leg into authoritative absence. # #2446 removes implicit credential-wide / first-run billing # authority. Every integration must explicitly choose # ``billing_scope_mode=projects_only`` (which produces a complete # empty partition) or ``allowlist`` (which verifies every account's # parent chain). A missing/invalid policy is incomplete even on the # first run and must block before writes; silently treating it as # projects-only would recreate the unverified tenant boundary. blocking_partition_types = sorted(incomplete_scope_types) if blocking_partition_types: reasons = "; ".join( f"{scope_type}: " f"{scope_partitions[scope_type].error or 'incomplete'}" for scope_type in blocking_partition_types ) return { "status": "failed", "error": ( "scope sync: incomplete authoritative partition(s) " f"{', '.join(blocking_partition_types)} — refusing " f"all upserts and close-sweep ({reasons})" ), "scope_partitions": partition_result, } sync_job_id = str(uuid.uuid4()) upserted = 0 # Default monthly budget + alert rules applied to every NEW # scope. Existing scopes are NOT touched (the on_duplicate_key # update list deliberately excludes ``budget_usd_monthly`` and # ``alert_rules`` so admin-customised values stick across syncs). # Knobs: # * ``cloud.default_monthly_budget_usd`` — per-tenant initial # budget. Default 1000 USD; admins can drop a 0 to disable # auto-budgeting tenant-wide. # * ``cloud.default_alert_rules`` — JSON array of rule # templates. ``threshold_pct`` (0-100) is interpreted as a # percentage of the scope's monthly budget at evaluation # time, so the template is budget-relative not absolute. default_budget = await _setting_float( session, "cloud.default_monthly_budget_usd", 1000.0, ) default_alert_rules = await _build_default_alert_rules( session, default_budget, ) # A provider account can be represented by more than one connector # (for example Tencent inventory and Tencent DNS). Preserve the # existing accountable owner when a second connector discovers the # same provider-native scope. Copy only a single unambiguous owner; # conflicting historical assignments remain a governance finding. discovered_external_ids = { str(info.external_id).strip() for info in scope_infos if str(info.external_id).strip() } inherited_owner_by_external_id: dict[str, str] = {} if discovered_external_ids: owner_rows = (await session.execute( select( CmdbCloudScope.external_id, CmdbCloudScope.owner_person_id, ) .join( Integration, Integration.id == CmdbCloudScope.integration_id, ) .where( Integration.provider == integration.provider, CmdbCloudScope.external_id.in_(discovered_external_ids), CmdbCloudScope.owner_person_id.isnot(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, CmdbCloudScope.deleted_at.is_(None), ) )).all() owners_by_external_id: dict[str, set[str]] = {} for external_id, owner_person_id in owner_rows: owners_by_external_id.setdefault(str(external_id), set()).add( str(owner_person_id), ) inherited_owner_by_external_id = { external_id: next(iter(owner_ids)) for external_id, owner_ids in owners_by_external_id.items() if len(owner_ids) == 1 } BATCH_SIZE = 50 for info in scope_infos: # Promote the provider's per-account contact email out of the # ``raw`` blob into a first-class column. AWS Organizations # puts it at ``raw["email"]`` (see AwsCloudDriver.list_scopes); # other drivers don't report one yet, so this is None for them. # CLOUD-AUTHORITATIVE — refreshed on every upsert (INSERT and # ON DUPLICATE KEY) like ``display_name``; the owner override # lives in a separate column and is never touched here. cloud_email = _extract_scope_email(info.raw) insert_values: dict[str, Any] = { "id": str(uuid.uuid4()), "integration_id": str(integration.id), "scope_type": info.scope_type, "external_id": info.external_id, "display_name": info.display_name, "status": info.status, "raw_data": info.raw or {}, "cloud_account_email": cloud_email, # Name-derived environment for FIRST discovery only — it # is deliberately absent from the ON DUPLICATE KEY UPDATE # list below, so a human's later reclassification always # wins and is never reverted by a subsequent sync. The # column default was a flat ``dev`` for every provider, # which meant a production account looked like 开发 in the # UI (and skipped the prod write lock + expiry-sweep # exemption) until somebody noticed by hand. See # ``classify_scope_environment`` for the nonprod caveat. "environment": classify_scope_environment(info.display_name), "last_seen_sync_job_id": sync_job_id, "last_synced_at": datetime.now(UTC), } inherited_owner = inherited_owner_by_external_id.get( str(info.external_id).strip(), ) if inherited_owner: insert_values["owner_person_id"] = inherited_owner if default_budget > 0: insert_values["budget_usd_monthly"] = default_budget insert_values["alert_rules"] = default_alert_rules stmt = mysql_insert(CmdbCloudScope).values( **insert_values, ).on_duplicate_key_update( display_name=info.display_name, # ``scope_type`` IS refreshed from the driver (added # 2026-07-30). It is not admin-tunable configuration — # it is the driver's classification of what the cloud # side actually is, and it is load-bearing: every # per-scope fan-out (grants / roles / billing_daily / # cost_facts) admits only ``TENANT_LEVEL_SCOPE_TYPES``. # Leaving it out meant a row that first landed with a # wrong value kept it FOREVER while every sync happily # refreshed display_name / raw_data / last_synced_at # around it — invisible drift between the driver's # truth and the stored value. # # Real incident: prod scope 97a6bfd7 (Azure sub # 7f070020-…, "data-integration-team-internal-azure") # was created 2026-07-08 by ``manual_provision`` with a # hand-typed ``scope_type='azure'`` (free-form input # until this same PR made it a 422), then re-discovered # by every scopes_sync run without ever being # corrected. When the IAM fan-out gained the # tenant-level filter, that one stale string silently # dropped a REAL subscription (921 roles, 7 live # grants) out of grants/roles sync — and its 921 role # rows were only 3% of the integration's catalog, i.e. # UNDER the 20% drop-tolerance floor, so the roles # mark-and-sweep would have soft-deleted every one of # them without tripping any guard. Refreshing here is # what makes such a row self-heal on the next tick # instead of needing a manual UPDATE. scope_type=info.scope_type, # Cloud contact email IS refreshed from the driver each # sync (cloud-authoritative, same policy as # ``display_name`` / ``scope_type``). The owner override # (``account_email_override`` + its modifier stamp) is # NOT in this list, so a human edit survives every sync # while "云上值" stays current. cloud_account_email=cloud_email, # Deliberately NOT overwriting ``status`` on update: a # scope the admin has manually set to ``suspended`` / # ``closed`` in Helix should stay that way even if the # provider still reports it active. Cloud-side status # transitions (rare) need an explicit admin sync step # via the UI drawer. # # Also NOT overwriting ``budget_usd_monthly`` / # ``alert_rules`` — these are admin-tunable per scope; # blowing them away on every sync would erase customer # configuration. Initial values land via the INSERT # branch only. raw_data=info.raw or {}, last_seen_sync_job_id=sync_job_id, last_synced_at=datetime.now(UTC), # ``deleted_at`` un-soft-delete is gated on # ``decommissioned_at IS NULL``. Pre-fix: ANY driver # re-listing of a decommissioned scope cleared the # soft-delete tombstone (``deleted_at=None``) while # ``status`` and ``decommissioned_at`` stayed put. The # half-revived row then bypassed both list filters # (default ``status != 'closed'`` + ``deleted_at IS # NULL``) and reappeared in the 订阅列表 tab. Real # incident 2026-05-21: 10 GCP org-root projects # decommissioned in the 2026-05-18 bulk takedown # resurrected on 2026-05-19 20:24 sync. Behaviour # parity with ``status`` (admin-set tombstone wins # over re-discovery) — if the admin closed it, the # sync mustn't quietly re-open it. Bare column refs # in ON DUPLICATE KEY UPDATE resolve to the existing # row's values (not VALUES()). deleted_at=case( (CmdbCloudScope.decommissioned_at.isnot(None), CmdbCloudScope.deleted_at), else_=null(), ), ) await session.execute(stmt) upserted += 1 if upserted % BATCH_SIZE == 0: await session.commit() # Flush any tail rows before the sweep so lock contention on # the orphan-close UPDATE is minimal. await session.commit() # Pre-launch hardening 2026-05-16 (LS2 / CR4 P0): refuse the # close-sweep when the driver returned zero scopes or fewer # than the configured-tolerance floor below the Helix-active # baseline. Both anomalies silently flip every active scope to # ``closed`` — billing UI / self-service grant flow goes blank, # every existing IAM grant is orphaned in the UI. Fail loud # and let the next sync retry rather than masquerading as "ok, # closed N". Tolerance from platform_settings (see below). scope_baseline_q = select(func.count(CmdbCloudScope.id)).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, # Exclude soft-deleted rows from the baseline. A legacy # ``deleted_at IS NOT NULL`` + ``status='active'`` row # would inflate the baseline and mask a real provider # regression behind the configured-tolerance floor. CmdbCloudScope.deleted_at.is_(None), # Narrow to ever-synced rows so the baseline matches the # sweep scope (added 2026-05-28 alongside the sweep # predicate change). Admin-seeded scopes with NULL # ``last_seen_sync_job_id`` (scope-request approval flow, # future migration import) are structurally outside the # cloud-sync's authority — they inflate the baseline but # can never be matched by an upsert, so leaving them in # the count would make the drop-ratio guard misfire on # every otherwise-healthy sync (the same flaw the resource # ``_safe_sweep`` had pre-2026-05-28). CmdbCloudScope.last_seen_sync_job_id.is_not(None), ) scope_baseline = int( (await session.execute(scope_baseline_q)).scalar() or 0, ) scope_observed_q = select(func.count(CmdbCloudScope.id)).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.last_seen_sync_job_id == sync_job_id, ) scope_observed = int( (await session.execute(scope_observed_q)).scalar() or 0, ) # Mirrors sync.py::_safe_sweep — tolerance resolves via # platform_settings['cmdb_sync.sweep_drop_tolerance_percent'] # (admin-editable, no restart) with the .env fallback # settings.CMDB_SYNC_SWEEP_DROP_TOLERANCE_PERCENT. Cheap pre- # check ``upserted < scope_baseline`` short-circuits the # healthy-sync path so we skip the DB roundtrip when no drop # could possibly trip the guard. Tolerance read exactly once # so the floor and error-message percentage agree. if scope_observed == 0 and scope_baseline > 0: raise RuntimeError( f"scope sync: provider returned 0 scopes but baseline is " f"{scope_baseline} active scopes for integration " f"{integration.name!r} — refusing close-sweep (likely " "partial driver failure)." ) if scope_baseline > 0 and scope_observed < scope_baseline: scope_tolerance_pct = await get_sweep_drop_tolerance_percent(session) if drop_exceeds_tolerance( baseline=scope_baseline, observed=scope_observed, tolerance_percent=scope_tolerance_pct, ): raise RuntimeError( f"scope sync: provider persisted {scope_observed} unique " f"scopes ({upserted} raw upserts) vs " f"{scope_baseline} active baseline (>{scope_tolerance_pct}% drop) for " f"integration {integration.name!r} — refusing close-sweep " "(likely partial driver failure)." ) # Mark-and-close rows not seen in this sync — do NOT soft-delete # (billing history needs the FK to resolve). Separate transaction # from the upsert loop above so a partial upsert failure doesn't # trigger a mass-close of unseen rows. # # ``last_seen_sync_job_id IS NOT NULL`` predicate (added 2026-05-28 # to mirror the equivalent ``_safe_sweep`` change in sync.py): # only close scopes that WERE once observed by some prior sync. # Scope rows with NULL job_id are either manual-API inserts that # haven't been touched by sync yet, or fresh imports from a # future migration — neither should be auto-closed. Same incident # class as the legacy-CMDB → sync collision; closing those scopes # would cascade-disable every grant under them. # Capture scope ids BEFORE the sweep so we can cascade-disable # their grants in the same transaction. Without this cascade the # scopes flip to ``status='closed'`` but their active / # pending grants stay live; the drift sweep in # ``sync_grants_for_integration`` is restricted to # ``synced_scope_ids`` (the scopes the driver returned grants # for), so closed scopes are NEVER eligible for drift cleanup # and accumulate ``status='active'`` zombie grants forever. # Production incident 2026-05-28: UCloud closed scope # ``1ccf3d0a-…`` flipped on 2026-05-05 but kept 29 active # discovered grants stranded at ``last_seen=2026-05-04``. # See feedback_on_duplicate_key_preserve_consistency.md and # the matching explicit-decommission cascade at # ``decommission_scope`` above. # # SELECT-then-UPDATE race window assumption: the cron + admin # "sync now" paths both hold ``cloud_sync:`` # for the whole call (see scheduled_jobs/handlers.py + # lifecycle_router._run_scope_sync_background), so no other # scope-sync run can interleave. Other write paths # (scope_request_service approvals, admin grant edits) do NOT # insert grants on a scope whose row is about to flip to # ``closed`` — they only mutate existing scopes or insert # brand-new scope rows, never grants on existing-soon-doomed # ones. If a future write path violates that assumption it # MUST acquire ``cloud_sync:`` first or risk # leaving a freshly-inserted grant stranded on a closed scope. scope_ids_to_close = (await session.execute( select(CmdbCloudScope.id).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.last_seen_sync_job_id.is_not(None), CmdbCloudScope.last_seen_sync_job_id != sync_job_id, CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ) )).scalars().all() scope_ids_to_close = [str(sid) for sid in scope_ids_to_close] sweep = update(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.last_seen_sync_job_id.is_not(None), CmdbCloudScope.last_seen_sync_job_id != sync_job_id, CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ).values(status="closed") res = await session.execute(sweep) closed = res.rowcount # Cascade-disable grants on the closed scopes. Mirror the shape # the explicit ``decommission_scope`` path uses: ``ACTIVE`` and # ``PENDING`` → ``DISABLED`` with a distinct ``revoke_reason`` # so compliance audits can tell sweep-close from explicit # decommission. ``revoked_at`` is stamped for symmetry with the # explicit revoke path (so the "已撤销" tab can sort by recency). grants_disabled = 0 now_close = datetime.now(UTC) if scope_ids_to_close: grant_sweep = update(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id.in_(scope_ids_to_close), CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ).values( status=GRANT_STATUS_DISABLED, revoke_reason=GRANT_REVOKE_REASON_SCOPE_CLOSED, revoked_at=now_close, updated_at=now_close, # Clear the inactivity-stage so a future scan can't # re-fire a stale notification on a row that's now # closed by scope cascade. last_notified_stage=None, ) grant_res = await session.execute(grant_sweep) grants_disabled = grant_res.rowcount or 0 # Two audit rows — `cloud.scope.sweep_close` always fires # when the sweep closed at least one scope (independent of # whether any grants needed cascading), so a security # auditor reviewing "which scopes closed and when" has a # single canonical surface. `cloud.scope.sweep_close_cascade` # is the additional grant-cascade trail and only fires when # grants actually moved — keeps the cascade audit's # ``grants_disabled`` and ``closed_grant_*`` semantically # meaningful (no zero-count rows that an auditor has to # filter out). audit_close = AuditService(session) await audit_close.log( trace_id=f"scope-sync-{sync_job_id}", actor_type="system", actor_user_id=None, action="cloud.scope.sweep_close", status="success", resource_type="integration", resource_id=str(integration.id), detail={ "integration_id": str(integration.id), "provider": integration.provider, "closed_scope_ids": scope_ids_to_close, "closed_scope_count": len(scope_ids_to_close), "reason": "no_longer_returned_by_provider", }, ) if grants_disabled: await audit_close.log( trace_id=f"scope-sync-{sync_job_id}", actor_type="system", actor_user_id=None, action="cloud.scope.sweep_close_cascade", status="success", resource_type="integration", resource_id=str(integration.id), detail={ "integration_id": str(integration.id), "provider": integration.provider, "closed_scope_ids": scope_ids_to_close, "grants_disabled": grants_disabled, "reason": "scope_closed_by_sweep", }, ) await session.commit() # Orphan-owner detection — compliance_incident for new # unowned scopes. async with best_effort( session, label="lifecycle_service best-effort step", log=logger, ) as be_9405_step: from app.compliance.models import SEVERITY_WARNING from app.compliance.service import ComplianceIncidentService inc_service = ComplianceIncidentService(session) unowned_count = 0 unowned_rows = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.owner_person_id.is_(None), CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, # Billing-account scopes are infra, not an ownership-gap # risk like an unowned project — an unassigned one just # isn't open for requests yet. Don't raise an incident. CmdbCloudScope.scope_type != SCOPE_TYPE_GCP_BILLING_ACCOUNT, ) )).scalars().all() for scope in unowned_rows: # Aliyun RD master (management/payer) account is infra — # it's auto-registered as a scope only so master-allocated # (empty-OwnerID) billing lines have a home (see # AliyunCloudDriver.list_scopes). It is never "open for # requests" and an unassigned owner is not an ownership-gap # risk — same rationale as GCP billing accounts above. Skip # the incident. (Filtered in Python rather than SQL to stay # portable across TiDB JSON-boolean extraction quirks.) _raw = scope.raw_data or {} if isinstance(_raw, dict) and _raw.get("rd_master"): continue await inc_service.create_or_upsert_incident( incident_type="cloud_scope_unowned", severity=SEVERITY_WARNING, person_id=None, title=( f"云 scope 未指定 owner: {scope.display_name or scope.external_id} " f"(tenant={integration.name})" ), detail={ "scope_id": str(scope.id), "integration_id": str(integration.id), "external_id": scope.external_id, }, ) unowned_count += 1 if not be_9405_step.ok: logger.warning("unowned scope incident create failed: %s", be_9405_step.reason()) unowned_count = 0 await session.commit() return { "integration_id": integration_id, "alias": integration.name, "upserted": upserted, "closed": closed, "grants_disabled_by_sweep": grants_disabled, "unowned_incidents": unowned_count, "scope_partitions": partition_result, "ignored_incomplete_scope_partitions": ( ignored_incomplete_scope_partitions ), } # ── Role catalog sync ────────────────────────────────────────── async def _admin_promotion_overrides( session: AsyncSession, integration_id: str, ) -> dict[str, bool]: """The admin's 「默认下拉?」 decisions for one integration, by role_key. ``is_promoted`` is stored per (integration, scope, role_key) row, but the decision the admin makes in the catalog UI is tenant-wide ("this role should be primary in Aliyun", not "…in *this* sub-account" — which is why promote-by-key flips every same-keyed row at once). Without this lookup the decision only ever reached the rows that existed when it was made: every sub-account discovered afterwards got a fresh row seeded from the driver's classification, and a demoted role reappeared in the employee dropdown there. Prod 2026-06-11 → 2026-09-07: ten ``DBaaS-Prod-*`` CloudSSO roles demoted on 21 sub-accounts, promoted again on the 8 onboarded since, with the catalog table still showing 否 because it reports the cross-scope consensus. Migration 623 added the flag and backfilled the provable half of the history. Mixed values under one key (possible only via the row-id promote endpoint, which no UI calls) resolve to ``False`` — the cautious direction is keeping a role OUT of the employee dropdown. """ rows = (await session.execute( select(CmdbCloudRole.role_key, CmdbCloudRole.is_promoted) .where( CmdbCloudRole.integration_id == integration_id, CmdbCloudRole.promotion_overridden_by_admin.is_(True), ) )).all() out: dict[str, bool] = {} for role_key, is_promoted in rows: if role_key in out and out[role_key] != bool(is_promoted): out[role_key] = False continue out[role_key] = bool(is_promoted) return out async def _reconcile_admin_promotions( session: AsyncSession, integration_id: str, overrides: dict[str, bool], ) -> int: """Re-stamp rows that drifted away from the admin's decision. Seeding new rows (``_upsert_synced_cloud_role``) fixes the forward direction only. This heals what already drifted: rows inserted before the flag existed, rows revived from a mark-and-sweep soft delete, and anything a per-row flip left inconsistent. Two statements per sync run at most — the flag is rare, and the WHERE narrows on the ``(integration_id, scope_id, role_key)`` unique index prefix. Also stamps the flag onto the corrected rows so the whole (integration, role_key) group carries the decision from then on. """ fixed = 0 for target in (True, False): keys = [k for k, v in overrides.items() if v is target] if not keys: continue result = await session.execute( update(CmdbCloudRole) .where( CmdbCloudRole.integration_id == integration_id, CmdbCloudRole.role_key.in_(keys), CmdbCloudRole.is_promoted.is_(not target), ) .values( is_promoted=target, promotion_overridden_by_admin=True, ) .execution_options(synchronize_session=False) ) # ``getattr`` not ``result.rowcount``: the Core-UPDATE result is # typed ``Result[Any]``, which mypy says has no such attribute. fixed += int(getattr(result, "rowcount", 0) or 0) return fixed async def _upsert_synced_cloud_role( session: AsyncSession, *, integration_id: str, target_scope_id: str | None, role: Any, governance: RoleGovernance, sync_job_id: str, promotion_override: bool | None = None, ) -> None: """Upsert one provider role while preserving admin-owned fields. MySQL treats NULLs as distinct in composite UNIQUE keys. Tenant-wide roles therefore need an explicit canonical lookup; scope-local roles can retain the atomic native upsert. Both full sync and post-provision prewarm call this helper so their identity semantics cannot drift. ``promotion_override`` — the admin's tenant-wide 「默认下拉?」 decision for this ``role_key`` (from :func:`_admin_promotion_overrides`), or ``None`` when they never made one. It wins over the driver's classification **on INSERT**, which is the whole point: a role the admin removed from the employee dropdown must not come back promoted on the next sub-account Helix discovers. On UPDATE ``is_promoted`` is still never written here (the catalog UI owns the column). """ now = datetime.now(UTC) insert_promoted = ( role.is_promoted if promotion_override is None else promotion_override ) if target_scope_id is None: tenant_roles = list((await session.execute( select(CmdbCloudRole) .where( CmdbCloudRole.integration_id == integration_id, CmdbCloudRole.scope_id.is_(None), CmdbCloudRole.role_key == role.role_key, ) .order_by( CmdbCloudRole.deleted_at.is_(None).desc(), CmdbCloudRole.first_seen_at.asc(), CmdbCloudRole.id.asc(), ) .with_for_update() )).scalars()) if tenant_roles: active_roles = [r for r in tenant_roles if r.deleted_at is None] candidates = active_roles or tenant_roles def _curation_score(row: CmdbCloudRole) -> tuple[int, str]: score = sum(( bool(row.governance_overridden_by_admin), bool(row.promotion_overridden_by_admin), row.description_override is not None, bool(row.sort_order), row.is_promoted != bool(role.is_promoted), )) return (-score, str(row.id)) # Historical NULL-key duplicates could each receive a different # admin edit. Pick the most-curated active row as the canonical # identity, then merge every independently-owned field below so # normal sweep can retire the rest without losing a decision. canonical = min(candidates, key=_curation_score) ordered_candidates = sorted( candidates, key=lambda item: str(item.id), ) governance_override = next(( r for r in ordered_candidates if r.governance_overridden_by_admin ), None) description_override = next(( r.description_override for r in ordered_candidates if r.description_override is not None ), None) ranked = next(( r.sort_order for r in ordered_candidates if r.sort_order ), canonical.sort_order) # Promotion: the admin's tenant-wide decision wins when there # is one (``promotion_override``). Otherwise fall back to the # historical heuristic — any duplicate whose ``is_promoted`` # disagrees with the driver was flipped by hand, so that value # is the decision worth carrying onto the canonical row. flagged = next(( r.is_promoted for r in ordered_candidates if r.promotion_overridden_by_admin ), None) merged_promoted = next(( r.is_promoted for r in ordered_candidates if r.is_promoted != bool(role.is_promoted) ), canonical.is_promoted) if promotion_override is not None: merged_promoted = promotion_override elif flagged is not None: merged_promoted = flagged canonical.display_name = role.display_name canonical.description = role.description canonical.is_custom = role.is_custom canonical.source = role.source canonical.raw_data = role.raw or {} canonical.description_override = description_override canonical.sort_order = ranked canonical.is_promoted = merged_promoted if promotion_override is not None or flagged is not None: canonical.promotion_overridden_by_admin = True if governance_override is not None: canonical.governance_overridden_by_admin = True canonical.governance_tier = governance_override.governance_tier canonical.governance_reason = governance_override.governance_reason elif not canonical.governance_overridden_by_admin: canonical.governance_tier = governance.tier canonical.governance_reason = governance.reason canonical.last_seen_sync_job_id = sync_job_id canonical.last_synced_at = now canonical.deleted_at = None await session.flush() return ins = mysql_insert(CmdbCloudRole).values( id=str(uuid.uuid4()), integration_id=integration_id, scope_id=target_scope_id, role_key=role.role_key, display_name=role.display_name, description=role.description, is_custom=role.is_custom, source=role.source, raw_data=role.raw or {}, # Admin decision (if any) beats the driver's classification on # INSERT — otherwise a demoted role reappears in the employee # dropdown on every sub-account discovered after the demote. is_promoted=insert_promoted, promotion_overridden_by_admin=promotion_override is not None, governance_tier=governance.tier, governance_reason=governance.reason, last_seen_sync_job_id=sync_job_id, last_synced_at=now, ) stmt = ins.on_duplicate_key_update( display_name=role.display_name, description=role.description, is_custom=role.is_custom, source=role.source, raw_data=role.raw or {}, governance_tier=case( ( CmdbCloudRole.governance_overridden_by_admin, CmdbCloudRole.governance_tier, ), else_=ins.inserted.governance_tier, ), governance_reason=case( ( CmdbCloudRole.governance_overridden_by_admin, CmdbCloudRole.governance_reason, ), else_=ins.inserted.governance_reason, ), last_seen_sync_job_id=sync_job_id, last_synced_at=now, deleted_at=None, ) await session.execute(stmt) async def sync_roles_for_integration( integration_id: str, *, force_sweep: bool = False, ) -> dict[str, Any]: """Sync cloud role catalog for an integration, mark-and-sweep stale rows. ``force_sweep`` (admin override, default False): when True, bypass BOTH the every-scope-failed guard and the drop-tolerance guard below. The endpoint that sets it (``POST /cmdb/integrations/{id}/roles-sync/force-sweep``) writes its own ``cmdb.roles_sync.force_sweep_override`` audit row. Designed for the one-shot case where a driver-side filter change (e.g. PR #565 dropping ``AliyunReserved*`` mirror policies — 10 674 rows → 288 real CloudSSO permission sets) makes the legitimate new catalog look like a >20% partial-driver-failure to the safety floor. Without this escape valve the cron stays red forever and only ack silences the alarm without fixing state. """ # When the integration is brand-new, admins tend to hit "立即执行" on the # roles_sync job before the scopes_sync cron has run — and roles need # scopes to anchor to. Instead of silently returning ``skipped`` (which # looks like a failure to the admin), self-heal by running scopes_sync # first whenever the scope table is empty. auto_scoped = False async with async_session_factory() as session: integration = await session.get(Integration, uuid.UUID(integration_id)) if integration is None or integration.kind != "cloud": return {"status": "skipped"} # Disabled integrations must not run sync (paired with the # same guard in sync_scopes / sync_grants). Without this, a # disabled integration's stale driver call still feeds into # the role mark-and-sweep and can soft-delete legitimate # roles when the driver returns a partial enumeration. if not integration.enabled: return {"status": "skipped", "reason": "integration disabled"} # Tenant-level only — see the identical partition (and the # ``InvalidSubscriptionId`` post-mortem) in # ``sync_grants_for_integration``. ``list_scope_roles`` is # tenant-addressed for every provider: the Azure driver builds # ``/subscriptions/{external_id}/providers/Microsoft. # Authorization/roleDefinitions``, so a sub-tenant RG row fed # in here 400s and lands in ``errors`` on every run. It ALSO # inflated the every-scope-failed / drop-tolerance guards' # denominator with a scope that can never contribute roles. scopes = [ s for s in (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ) )).scalars().all() if s.scope_type in TENANT_LEVEL_SCOPE_TYPES ] if not scopes: scope_result = await sync_scopes_for_integration(integration_id) if scope_result.get("status") == "failed": return { "status": "failed", "reason": "auto-scope-sync failed", "scope_sync_error": scope_result.get("error"), } if scope_result.get("status") == "skipped": # scopes_sync itself bailed out (integration disabled / not # cloud / driver missing list_scopes) — there's nothing we # can do at this layer. Propagate the original reason. return scope_result auto_scoped = True async with async_session_factory() as session: integration = await session.get(Integration, uuid.UUID(integration_id)) if integration is None or integration.kind != "cloud": return {"status": "skipped"} # Same tenant-level partition as the pre-check query above — # both must agree or the "did scopes_sync help?" branch and the # actual work loop disagree about what a scope is. all_active = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ) )).scalars().all() scopes = [ s for s in all_active if s.scope_type in TENANT_LEVEL_SCOPE_TYPES ] skipped_non_tenant = [ f"{s.scope_type}:{s.external_id}" for s in all_active if s.scope_type not in TENANT_LEVEL_SCOPE_TYPES ] if skipped_non_tenant: # Named, not silent — same reasoning as grants_sync. A role # catalog that is empty for a scope nobody knows was skipped # renders as an empty request dropdown with no explanation. logger.info( "roles_sync %s: skipping %d non-tenant-level scope " "row(s) — not IAM-addressable: %s", integration.name, len(skipped_non_tenant), ", ".join(skipped_non_tenant[:20]), ) if not scopes: return { "status": "skipped", "reason": ( "no active tenant-level scopes " f"({len(skipped_non_tenant)} non-tenant-level row(s) " "skipped — not IAM-addressable)" if skipped_non_tenant else "no active scopes (even after scopes_sync)" ), "auto_scoped": True, "scopes_skipped_non_tenant": skipped_non_tenant, } driver = await _load_cloud_driver(session, integration) sync_job_id = str(uuid.uuid4()) upserted = 0 errors: list[str] = [] failed_scope_ids: set[str] = set() # The admin's 「默认下拉?」 decisions, read ONCE per run and applied # two ways: re-stamped onto rows that already drifted (below), and # used to seed rows this run inserts for newly-discovered # sub-accounts. Runs before the loop so a row revived from a # soft delete inside the loop is already carrying the decision. promotions = await _admin_promotion_overrides( session, str(integration.id), ) promotions_repaired = await _reconcile_admin_promotions( session, str(integration.id), promotions, ) if promotions_repaired: await session.commit() logger.info( "roles_sync re-stamped %d role row(s) back onto the admin's " "promote/demote decision (integration=%s)", promotions_repaired, integration.id, ) try: for scope in scopes: try: roles = await driver.list_scope_roles(CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, )) except NotImplementedError: continue except Exception as e: # noqa: BLE001 errors.append( f"{scope.external_id}: {format_exception_for_log(e)}" ) # Remember WHICH scope failed (#2135). Its role rows # will not carry this run's ``last_seen_sync_job_id`` # and the mark-and-sweep below deletes exactly that, # so without an exemption one scope's persistent 403 # / throttle wipes its entire role catalog — the # cloud-access dropdown goes empty — while the cron # stays green. The drop-ratio floor is computed over # the WHOLE integration, so a small scope never trips # it, and "revive on the next success" does not help # when the error is a permanent misconfiguration. failed_scope_ids.add(str(scope.id)) continue for role in roles: # Role scope_id — attach to this scope unless the # driver explicitly marked it tenant-wide (role.scope # is None — e.g. AWS permission sets). target_scope_id = ( None if role.scope is None else str(scope.id) ) # ``is_promoted`` write rule: the driver's # classification is authoritative on insert UNLESS # the admin already decided this role_key in the # catalog UI (``promotions``) — a demote must hold # for sub-accounts onboarded after it. On update we # DON'T overwrite the column at all, because the # catalog UI writes to it and we mustn't clobber # that on the next sync. The platform's source of # truth for "is this role on the dropdown" is the DB # row, which the next request render reads back. gov = driver.classify_role( role.role_key, role.display_name, scope.scope_type, ) await _upsert_synced_cloud_role( session, integration_id=str(integration.id), target_scope_id=target_scope_id, role=role, governance=gov, sync_job_id=sync_job_id, promotion_override=promotions.get(role.role_key), ) upserted += 1 # Per-scope commit — same fix class as # ``sync_billing_daily_for_integration``. This loop calls # the external ``driver.list_scope_roles`` per scope, so a # single end-of-loop commit held the transaction (and its # ``cmdb_cloud_roles`` row locks) open across minutes of # cloud API calls on large tenants — prod hit # ``(1205) Lock wait timeout exceeded`` on the role INSERT # (2026-05-13 / 2026-05-19). Committing per scope releases # those locks promptly. The mark-and-sweep + drop- # tolerance guard below run afterwards and read the # now-committed upserts; refusing the sweep (raise) after # partial commits is safe — re-discovered roles just get # their ``last_synced_at`` / un-delete refreshed # idempotently. Safe mid-loop: expire_on_commit=False. await session.commit() finally: try: await driver.close() except Exception: # noqa: BLE001 pass # Pre-launch hardening 2026-05-16 (LS2 / CR4 P0): refuse the # sweep when every scope's ``list_scope_roles`` failed (errors # full, upserted=0) OR upserted dropped more than the # configured tolerance below the Helix-active baseline. A # driver outage that 5xx's on every scope would otherwise # soft-delete every role on the tenant → grant request # dropdown empty across the org. Refusing here lets the next # cron retry while preserving the catalog. Tolerance from # platform_settings (resolver call below). # Count logical identities, not physical rows. Historical MySQL rows # may contain duplicate tenant-wide (scope_id=NULL, role_key) values # because NULL bypasses composite UNIQUE enforcement. A physical-row # baseline would reject the very sweep that retires those duplicates. role_baseline_keys = ( select(CmdbCloudRole.scope_id, CmdbCloudRole.role_key) .where( CmdbCloudRole.integration_id == str(integration.id), CmdbCloudRole.deleted_at.is_(None), # Mirror the sweep predicate (2026-05-28): admin-seeded # NULL-job rows are outside cloud-sync's authority. CmdbCloudRole.last_seen_sync_job_id.is_not(None), ) .group_by(CmdbCloudRole.scope_id, CmdbCloudRole.role_key) .subquery() ) role_baseline_q = select(func.count()).select_from(role_baseline_keys) role_baseline = int( (await session.execute(role_baseline_q)).scalar() or 0, ) role_observed_keys = ( select(CmdbCloudRole.scope_id, CmdbCloudRole.role_key) .where( CmdbCloudRole.integration_id == str(integration.id), CmdbCloudRole.deleted_at.is_(None), CmdbCloudRole.last_seen_sync_job_id == sync_job_id, ) .group_by(CmdbCloudRole.scope_id, CmdbCloudRole.role_key) .subquery() ) role_observed_q = select(func.count()).select_from(role_observed_keys) role_observed = int( (await session.execute(role_observed_q)).scalar() or 0, ) # Mirrors sync.py::_safe_sweep — tolerance resolves via # platform_settings['cmdb_sync.sweep_drop_tolerance_percent'] # (admin-editable, no restart) with the .env fallback # settings.CMDB_SYNC_SWEEP_DROP_TOLERANCE_PERCENT. Cheap pre- # check ``upserted < role_baseline`` short-circuits the # healthy-sync path so we skip the DB roundtrip when no drop # could possibly trip the guard. Tolerance read exactly once # so the floor and error-message percentage agree. # ``force_sweep`` is the admin-confirmed override. When True we # bypass BOTH guards but still record exactly what would have # tripped them in the result dict, so the audit trail captures # the magnitude of the bypassed delta. Mirror of the # ``force_sweep`` semantics in ``cmdb/sync.py::_safe_sweep``. forced_floor_bypass: dict[str, Any] | None = None if role_observed == 0 and role_baseline > 0: if not force_sweep: raise RuntimeError( f"role sync: every scope's list_scope_roles failed " f"(errors={len(errors)}) but baseline is {role_baseline} " f"roles for integration {integration.name!r} — refusing " "soft-delete sweep (likely transient driver failure)." ) forced_floor_bypass = { "trip": "every_scope_failed", "errors": len(errors), "baseline": role_baseline, "upserted": role_observed, "raw_upserted": upserted, } if role_baseline > 0 and role_observed < role_baseline: role_tolerance_pct = await get_sweep_drop_tolerance_percent(session) role_floor = minimum_observed_within_tolerance( baseline=role_baseline, tolerance_percent=role_tolerance_pct, ) if drop_exceeds_tolerance( baseline=role_baseline, observed=role_observed, tolerance_percent=role_tolerance_pct, ): if not force_sweep: raise RuntimeError( f"role sync: provider persisted {role_observed} unique " f"roles ({upserted} raw upserts) vs " f"{role_baseline} active baseline (>{role_tolerance_pct}% drop) for " f"integration {integration.name!r} — refusing soft-delete " "sweep (likely partial driver failure)." ) # Don't overwrite the every_scope_failed bypass record # (rarer / stricter trip) if both would fire on the # same run — keep the most-specific reason. if forced_floor_bypass is None: forced_floor_bypass = { "trip": "drop_ratio", "tolerance_pct": role_tolerance_pct, "baseline": role_baseline, "upserted": role_observed, "raw_upserted": upserted, "floor": role_floor, } # ``last_seen_sync_job_id IS NOT NULL`` (2026-05-28, mirror of the # ``_safe_sweep`` fix): admin-curated / future-migration role rows # with NULL job_id are structurally outside any sync's scope and # must not be auto-soft-deleted. Same incident class as the # legacy-CMDB → server-sync collision. sweep_where = [ CmdbCloudRole.integration_id == str(integration.id), CmdbCloudRole.last_seen_sync_job_id.is_not(None), CmdbCloudRole.last_seen_sync_job_id != sync_job_id, CmdbCloudRole.deleted_at.is_(None), ] if failed_scope_ids and not force_sweep: # ``force_sweep`` is the admin's explicit "I know the catalog # is stale, clear it anyway" override (it already bypasses # the drop-ratio floor and the every-scope-failed guard, and # exists for decommissioning an integration). The exemption # must not quietly defeat it — an operator who forced the # sweep has said the enumeration failures are expected. # # A scope we could not enumerate this round tells us NOTHING # about whether its roles still exist — "not seen" here means # "not looked at" (#2135). Exempt its rows. Tenant-wide rows # (``scope_id IS NULL``) stay sweepable: they belong to no # single scope, and at least one scope succeeded or we would # have bailed out before reaching the sweep. sweep_where.append( or_( CmdbCloudRole.scope_id.is_(None), CmdbCloudRole.scope_id.not_in(sorted(failed_scope_ids)), ) ) logger.warning( "role sync: exempting %d scope(s) from the sweep — they " "failed enumeration this round: %s", len(failed_scope_ids), sorted(failed_scope_ids), ) sweep = update(CmdbCloudRole).where( *sweep_where, ).values(deleted_at=datetime.now(UTC)) r = await session.execute(sweep) deleted = r.rowcount await session.commit() result: dict[str, Any] = { "status": "success" if not errors else "partial_failure", "upserted": upserted, "deleted": deleted, "errors": errors, # Rows the role fan-out cannot address — see the partition # above. Never an error; surfaced so an unexplained empty # role dropdown is traceable to one line of the summary. "scopes_skipped_non_tenant": skipped_non_tenant, } if auto_scoped: # Tell the admin (via the scheduled-jobs history UI) that we # auto-ran scopes_sync on their behalf — no missed signal. result["auto_scoped"] = True if forced_floor_bypass is not None: # Surface the bypassed-guard details so the # force-sweep endpoint can mirror them into the audit # row and the UI ``done`` modal can show what was forced # (e.g. "10 674 → 288, ratio guard bypassed"). result["forced_floor_bypass"] = forced_floor_bypass return result async def sync_roles_for_scope(scope_id: str) -> dict[str, Any]: """Sync the role catalog for a SINGLE cloud scope. Used by the post-provision path so the new scope's "添加成员" dropdown is populated immediately after ``driver.create_scope`` (or ``manual_provision``) lands a scope row, instead of waiting up to 24h for the next ``cmdb.roles_sync`` cron. Differs from :func:`sync_roles_for_integration` in two ways: * **No mark-and-sweep** — only inserts/updates rows for THIS scope. The cron-driven full sync remains responsible for retiring stale roles. Doing per-scope mark-and-sweep here would close every OTHER scope's roles on this integration (they wouldn't carry this run's ``sync_job_id``), breaking the catalog for every existing scope on the same integration. * **Best-effort** — failures (driver crash, network) come back in the result dict, not as raised exceptions. Callers (typically a FastAPI BackgroundTask) log on non-empty ``errors`` but do NOT propagate to the user-facing approve response: the scope already exists; the cron will recover. Uses a fresh ``async_session_factory()`` so a sqlalchemy error in here can't poison the caller's transaction. """ async with async_session_factory() as session: scope = await session.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None or scope.deleted_at is not None: return {"status": "skipped", "reason": "scope not found"} if scope.status != SCOPE_STATUS_ACTIVE: return { "status": "skipped", "reason": f"scope status is {scope.status!r}, expected active", } integration = await session.get( Integration, uuid.UUID(scope.integration_id), ) if integration is None or integration.kind != "cloud": return {"status": "skipped", "reason": "integration missing"} driver = await _load_cloud_driver(session, integration) sync_job_id = str(uuid.uuid4()) upserted = 0 errors: list[str] = [] # This is THE path a brand-new sub-account takes (post-provision # prewarm), i.e. exactly where a demoted role used to come back # promoted. Same tenant-wide decisions the batch sync reads. promotions = await _admin_promotion_overrides( session, str(integration.id), ) try: try: roles = await driver.list_scope_roles(CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, )) except NotImplementedError: # Say WHY, in the same words the batch path uses # (#2189-10). "driver does not implement" reads as a # driver capability gap and sends the admin to check the # integration; the real reason is almost always that this # scope is not tenant-level and therefore not # IAM-addressable — which is a property of the SCOPE. return { "status": "skipped", "reason": ( "scope is not tenant-level / not IAM-addressable " "(driver raised NotImplementedError for " "list_scope_roles)" ), } except Exception as e: # noqa: BLE001 # ``str(e)`` alone was previously here — drops AppException # message AND lands the empty string for httpx transport # errors. Canonical helper handles both. errors.append(format_exception_for_log(e)) roles = [] for role in roles: target_scope_id = ( None if role.scope is None else str(scope.id) ) # ``is_promoted`` update rule mirrors # :func:`sync_roles_for_integration`: driver # classification is authoritative on INSERT unless the # admin already decided this role_key (``promotions``); # on UPDATE we deliberately DON'T overwrite admin # promote/demote flips. Per-scope re-runs are idempotent. gov = driver.classify_role( role.role_key, role.display_name, scope.scope_type, ) await _upsert_synced_cloud_role( session, integration_id=str(integration.id), target_scope_id=target_scope_id, role=role, governance=gov, sync_job_id=sync_job_id, promotion_override=promotions.get(role.role_key), ) upserted += 1 finally: try: await driver.close() except Exception: # noqa: BLE001 pass await session.commit() return { "status": "success" if not errors else "partial_failure", "upserted": upserted, "errors": errors, } # ── Grants sync (cloud → Helix reverse discovery) ───────────── async def sync_grants_for_integration( integration_id: str, *, force_drift_sweep: bool = False, ) -> dict[str, Any]: """Discover role/policy assignments already live on the cloud and back-fill them into ``cmdb_cloud_access_grants``. ``force_drift_sweep`` (admin override): when True, bypass the 50% safety floor on the discovered-grant mark-and-sweep. Use ONLY when an admin has confirmed the drift is real (e.g. they intentionally cleaned up a large batch of users in the cloud console). Without this escape valve a single bulk cleanup deadlocks future syncs: the floor refuses every run because matched/existing stays below 50% until the stale rows close, but the stale rows can't close because the floor refuses. Each forced run still writes a ``cloud.grant.drift_removed_sweep`` audit row with ``force_override: True``; the calling endpoint adds a separate ``cloud.grant.drift_sweep_force_override`` row attributing the decision to the admin user. Semantics: * Runs per-scope when the driver's ``grants_scope_wide=True`` (Azure / GCP / AWS SSO — each scope has its own assignment set). * Runs once per integration, on the canonical (first active) scope, when ``grants_scope_wide=False`` (Aliyun / Tencent — grants are tenant-wide). For each DiscoveredGrant returned: * Filter to ``principal_type='user'``. * Resolve ``principal_email`` to an active ``IdentityPerson`` by case-insensitive ``work_email`` OR by an email alias on ``identity_person_email_aliases`` (via ``source`` JOIN — but for v1 we do a simple work_email match and count skips). * Upsert into ``cmdb_cloud_access_grants``: - If a row with ``(scope_id, person_id, role_key)`` exists, touch ``last_seen_grants_sync_job_id`` / ``last_seen_grants_sync_at`` and refresh ``external_ref`` + ``detail``. Do NOT change ``source`` (keep helix-issued rows badged as helix). - Else insert with ``source='discovered'``, ``status='active'``, ``granted_at=grant.granted_at or now``. After all grants are upserted, for each scope with no ``owner_person_id``, check whether exactly one person holds an owner-equivalent role (``driver.owner_role_keys``). If so, set ``scope.owner_person_id`` and emit a ``cloud.scope.auto_owner`` audit row. Ambiguous cases (0 or 2+ owner-role holders) are left alone for manual assignment. Returns a summary: ``{"scopes_scanned": int, "upserted": int, "touched": int, "skipped_unresolved": int, "auto_owners_assigned": int, "drift_closed": int, "errors": [str, ...]}`` Single-flight is enforced by the **caller** via the ``cloud_sync:`` distributed lock (see ``lifecycle_router._run_scope_sync_background`` for admin sync and ``scheduled_jobs/handlers.run_grants_sync`` for the cron path). Without that lock two overlapping runs each stamp grant rows with their own ``sync_job_id`` and the first run's drift sweep then closes every grant the second run touched (because ``last_seen_grants_sync_job_id`` stops matching the first run's id) — silent mass-revoke of legitimate access. The lock is on the caller side rather than here so the existing admin path (which also runs scopes_sync + roles_sync inside the same lock block) doesn't deadlock against an inner lock acquire. """ async with async_session_factory() as session: integration = await session.get(Integration, uuid.UUID(integration_id)) if integration is None or integration.kind != "cloud": return {"status": "skipped", "reason": "not a cloud integration"} if not integration.enabled: return {"status": "skipped", "reason": "integration disabled"} # Tenant-level scope types only — same filter the cost crons # apply (``sync_billing_daily_for_integration`` / # ``cost_service.sync_cost_facts_for_integration``). Sub-tenant # rows (today: ``azure_resource_group``, minted per approved AI # API key request by the Azure OpenAI auto-grant flow) carry an # RG NAME in ``external_id``, and every provider's IAM read is # tenant-addressed: the Azure driver builds # ``/subscriptions/{external_id}/providers/Microsoft. # Authorization/roleAssignments``, so the RG name produced a # hard ``InvalidSubscriptionId`` 400 → ``errors`` non-empty → # ``partial_failure`` → cmdb.grants_sync red on EVERY run from # the moment the first AOAI RG row was minted (prod 2026-07-29, # RG ``helix-aoai-20260729-5b25-pingcap-csp``). Sub-tenant rows # are also never swept by scopes_sync (``last_seen_sync_job_id`` # IS NULL is excluded from its sweep), so the failure was # permanent, not self-healing. # # Excluding them here (rather than only guarding in the driver) # also keeps them out of the ``grants_scope_wide=False`` # tenant-wide branch below — which tags the single enumeration # onto ``scopes[0]`` and sprays it across ``scopes[1:]`` — where # an RG row would have collected grants it does not represent. # # NB: partitioned in Python, NOT filtered in SQL like the cost # crons do. Silence is the hazard on the access-control plane: # ``scope_type`` was free-form admin input on the # manual-provision rescue path until 2026-07-30, so a typo'd or # legacy value would otherwise drop a REAL tenant scope out of # grants sync with no signal anywhere (its discovered grants # would quietly stop being refreshed — the drift sweep is gated # on ``synced_scope_ids``, so nothing gets revoked, it just goes # stale forever). Naming what we skipped in the summary + log is # what makes that case findable. all_active = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ).order_by(CmdbCloudScope.first_seen_at) )).scalars().all() scopes = [ s for s in all_active if s.scope_type in TENANT_LEVEL_SCOPE_TYPES ] skipped_non_tenant = [ f"{s.scope_type}:{s.external_id}" for s in all_active if s.scope_type not in TENANT_LEVEL_SCOPE_TYPES ] if skipped_non_tenant: logger.info( "grants_sync %s: skipping %d non-tenant-level scope " "row(s) — not IAM-addressable: %s", integration.name, len(skipped_non_tenant), ", ".join(skipped_non_tenant[:20]), ) if not scopes: return { "status": "skipped", "reason": ( "no active tenant-level scopes " f"({len(skipped_non_tenant)} non-tenant-level row(s) " "skipped — not IAM-addressable)" if skipped_non_tenant else "no active scopes (run scopes_sync first)" ), "scopes_skipped_non_tenant": skipped_non_tenant, } driver = await _load_cloud_driver(session, integration) sync_job_id = str(uuid.uuid4()) # Naive UTC — MySQL/TiDB DATETIME strips tz on round-trip and # expire_on_commit=False keeps these in-memory grant objects live # after commit; an aware value here would feed a naive/aware # subtraction into any in-session serializer, the same invariant the # approve-dispatch (~2735) and renew (~3003) paths enforce on # expires_at. This single coercion covers expires_at + the # last_seen_grants_sync_at markers that all derive from ``now``. now = datetime.now(UTC).replace(tzinfo=None) upserted = 0 touched = 0 skipped_unresolved = 0 # Sample of cloud-side principals that map to no Helix person. # Surfaced in the sync summary so the scope-detail banner can # name them instead of only counting them. unresolved_sample: list[str] = [] skipped_per_scope: dict[str, int] = {} errors: list[str] = [] candidates_by_scope: dict[str, _AutoOwnerCandidates] = {} # Build the scope-iteration plan. if getattr(driver, "grants_scope_wide", True): scope_plan = list(scopes) else: # Tenant-wide: one call, tagged with the first scope. scope_plan = scopes[:1] # Track which scope IDs were successfully synced (driver # returned a list, no exception). Only these scopes are eligible # for the stale-discovered-grant sweep at the end of the run — # closing grants on a scope whose driver call raised would # incorrectly punish a transient outage with permission # revocation. synced_scope_ids: set[str] = set() try: for scope in scope_plan: try: grants = await driver.list_scope_grants(CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, )) except NotImplementedError: logger.info( "grants_sync: %s driver does not implement " "list_scope_grants — skip", integration.provider, ) continue except Exception as exc: # noqa: BLE001 errors.append( f"{scope.external_id}: " f"{format_exception_for_log(exc)}" ) continue u, t, s, cands, unres = await _apply_discovered_grants( session, scope, grants, driver, sync_job_id, now, ) upserted += u touched += t skipped_unresolved += s unresolved_sample.extend(unres) if s > 0: skipped_per_scope[str(scope.id)] = s candidates_by_scope[str(scope.id)] = cands synced_scope_ids.add(str(scope.id)) # For tenant-wide drivers, spread the same discovered # grants across every remaining scope in the # integration — RAM users apply everywhere, so that's # the truthful representation. if not getattr(driver, "grants_scope_wide", True): for other in scopes[1:]: u2, t2, _s2, o2, _unres2 = await _apply_discovered_grants( session, other, grants, driver, sync_job_id, now, ) upserted += u2 touched += t2 # s2 duplicates the first scope's unresolved # count — don't double-count. candidates_by_scope[str(other.id)] = o2 synced_scope_ids.add(str(other.id)) # Per-scope commit — same fix class as # ``sync_billing_daily_for_integration`` / # ``sync_roles_for_integration``. ``list_scope_grants`` is # an external per-scope cloud call, so a single end-of-loop # commit held the transaction (and ``cmdb_cloud_access_ # grants`` row locks) open across minutes of cloud API # calls — the same long-transaction lock-contention shape # that 1205'd roles_sync / billing_daily. Committing the # per-scope upserts here releases those locks promptly. # IMPORTANT: this commits ONLY the discovered-grant upserts. # The stale-grant drift sweep + 50% safety floor + auto- # owner adoption below remain a SINGLE atomic block (their # own commit at the end), and the sweep is still gated on # ``synced_scope_ids`` + reads the now-committed upserts — # so per-scope commit cannot widen the mass-revoke blast # radius. Safe mid-loop: expire_on_commit=False. await session.commit() finally: try: await driver.close() except Exception: # noqa: BLE001 pass # Fallback: if the driver returned 0 grants for every scope but # the IAM-user table is populated, synthesise grants from # ``cmdb_iam_users.attached_policies``. This is the truthful # representation when: # * AWS ``account_mode='sso'`` is set but Identity Center # fields are blank — list_permission_sets returns [] # * Tencent / AliCloud RAM ListAttachedUserAllPolicies fails # silently (CAM auth scope mismatch, RAM master-only roles) # * Any provider that didn't ship a working # ``list_scope_grants`` yet but DID populate IAM users # The data is already in ``cmdb_iam_users`` so this costs zero # cloud API calls. Mark-and-sweep on ``last_seen_grants_sync_*`` # still keeps the rows fresh per run. if upserted == 0 and touched == 0 and not errors: derived = await _derive_grants_from_iam_users( session, str(integration.id), scopes, ) if derived: logger.info( "grants_sync %s: driver returned 0; deriving %d " "grants from cmdb_iam_users", integration.name, len(derived), ) # IAM users are tenant-wide for these providers; spray # across every active scope so the user-facing # /my-grants page surfaces the access correctly. first = True for scope in scopes: u, t, s, cands, unres = await _apply_discovered_grants( session, scope, derived, driver, sync_job_id, now, ) upserted += u touched += t # ``skipped_unresolved`` is per-scope but the same # emails resolve identically across tenant-wide # scopes — only count once. if first: skipped_unresolved += s unresolved_sample.extend(unres) first = False # Overwrite candidates pool: the IAM-derived # candidates are now the only source of truth for # owner adoption on these scopes. candidates_by_scope[str(scope.id)] = cands synced_scope_ids.add(str(scope.id)) # ── Stale-discovered-grant mark-and-sweep ───────────────── # # For every scope we just successfully synced, mark any # ``source='discovered'`` grant whose ``last_seen_grants_sync_*`` # markers were NOT touched in this run as ``revoked`` with # reason ``drift_removed``. Reasoning: # # * The driver returned the authoritative cloud-side grant # list for this scope. If a previously-discovered grant is # missing, the cloud side has already revoked it (admin # deleted the IAM user, detached the policy, removed the # role assignment, …). # # * Without this sweep, the row stays ``active`` forever, # auto-renewal keeps pushing ``expires_at`` out, and the # inactivity sweep eventually fires false "60 days no # login" notifications on a permission that no longer # exists. PR #170 follow-up: Lingtao Li (already cleaned # up in AWS) kept getting AWS SG IAMUserChangePassword # stale-grant alerts because nothing closed the row. # # * Helix-issued rows (``source='helix'``) are NEVER touched. # Helix is the system of record for those: cloud-side # drift is a compliance signal, handled by drift detection # elsewhere — auto-revoking would erase Helix's record of # "we approved this access". # # * ``pending`` rows are spared too — the approval flow can # legitimately leave them un-synced for short windows. # # Restricted to ``synced_scope_ids`` so a single scope's # transient driver failure cannot snowball into mass revoke # across the integration. # # Additional safety floor: borrowed from the seat-sync helper # ``safe_to_drift_revoke`` (50% match-rate floor). A driver # whose paginator silently truncates page 2+, or a cloud API # that returns a small subset on transient-overload, would # otherwise see Helix close the legitimate grants the # truncated response failed to enumerate. Floor refuses the # sweep when (existing_active >= 5) AND (matched < 50%). # Below 5 the sample is too small for the ratio to be useful # — we let those through (typical fresh-tenant case where the # only "missing" grant was an admin manually detaching one # policy). drift_closed = 0 closed_grant_ids: list[str] = [] if synced_scope_ids: stale_rows = (await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id.in_(synced_scope_ids), CmdbCloudAccessGrant.source == "discovered", CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, or_( CmdbCloudAccessGrant.last_seen_grants_sync_job_id.is_(None), CmdbCloudAccessGrant.last_seen_grants_sync_job_id != sync_job_id, ), ) )).scalars().all() existing_active = int((await session.scalar( select(func.count()) .select_from(CmdbCloudAccessGrant) .where( CmdbCloudAccessGrant.scope_id.in_(synced_scope_ids), CmdbCloudAccessGrant.source == "discovered", CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, ) )) or 0) matched_active = max(existing_active - len(stale_rows), 0) min_drift_ratio_floor = 5 min_drift_ratio = 0.5 floor_would_trip = ( existing_active >= min_drift_ratio_floor and existing_active > 0 and (matched_active / existing_active) < min_drift_ratio ) if floor_would_trip and not force_drift_sweep: errors.append( f"drift_sweep refused: matched {matched_active} " f"of {existing_active} existing discovered grants " f"({matched_active / existing_active:.0%}) is below " f"the {min_drift_ratio:.0%} safety floor — likely " "driver pagination truncation or transient outage. " "Investigate before re-running." ) else: # Per-row advisory lock so a concurrent revoke / # renew / resurrect on the same grant row serialises # against the sweep instead of producing a # last-write-wins overwrite. Same key the writers # already use (``grant_revoke:``). If the lock is # busy, we skip this stale row and let the next sync # tick re-evaluate it — the row is genuinely "stale" # only when no human writer is currently touching it, # and a single skipped tick is harmless (the row is # still classified as discovered-stale for the next # run if it remains so on the cloud side). for stale in stale_rows: try: async with distributed_lock( f"grant_revoke:{stale.id}", ttl=120, ): # Re-fetch the stale row's current state # under the lock — a concurrent writer # may have already revoked / renewed / # resurrected it between the outer SELECT # and the lock acquisition. Only proceed # if it's still ``active`` AND still # ``source='discovered'`` (the resurrect # path flips ``source='helix'`` so a # resurrected row is no longer the drift # sweep's responsibility). await session.refresh( stale, attribute_names=["status", "source"], ) if stale.status != GRANT_STATUS_ACTIVE: continue if stale.source != "discovered": continue stale.status = GRANT_STATUS_REVOKED stale.revoke_reason = ( GRANT_REVOKE_REASON_DRIFT_REMOVED ) stale.revoked_at = now # Notification stage carries no meaning # for a closed grant; clear so any future # inactivity sweep cannot pick this row # back up via a left-over # ``inactive_owner`` mark. stale.last_notified_stage = None closed_grant_ids.append(str(stale.id)) drift_closed += 1 except LockBusy: errors.append( f"drift_sweep skipped stale grant {stale.id}: " "concurrent writer holds grant_revoke lock; " "next sync tick will re-evaluate." ) if drift_closed: audit = AuditService(session) await audit.log( trace_id=f"grants-sync-{sync_job_id}", actor_type="system", actor_user_id=None, action="cloud.grant.drift_removed_sweep", status="success", resource_type="cmdb_cloud_access_grant", resource_id=str(integration.id), detail={ "integration_id": str(integration.id), "provider": integration.provider, "closed_count": drift_closed, "scopes_scanned": len(synced_scope_ids), # Track whether the safety floor was # bypassed by an explicit admin override. # The endpoint that set ``force_drift_sweep`` # writes its own ``drift_sweep_force_override`` # audit row with the admin's user_id; this # flag lets compliance correlate the two # without a JOIN on trace_id. "force_override": bool( force_drift_sweep and floor_would_trip, ), "matched_active": matched_active, "existing_active": existing_active, # Bounded list — stale_rows is at most the # discovered-active row count for the # synced scopes, which is a few hundred at # the tail in production. Stored as JSON # in ``audit_logs.detail`` so a compliance # auditor can answer "which exact grants # closed in this run" without joining. "closed_grant_ids": closed_grant_ids, }, ) await session.flush() # Auto-assign owner. Priority is owner-role holders first, then # fall back to any active-grant holder. Ties inside a pool go # to the earliest real granted_at, then to the lowest # person_id. See ``_pick_auto_owner_person`` for the full # rule — it honours the user request that every scope with # any grant at all should end up with a default owner (they # can hand it over later). auto_owners = 0 for scope in scopes: if scope.owner_person_id: continue cands = candidates_by_scope.get(str(scope.id)) if cands is None: continue person_id, reason = _pick_auto_owner_person(cands) if person_id is None: continue # Re-fetch under FOR UPDATE before writing the owner column # so a concurrent admin "set owner" PATCH doesn't get stomped # by this auto-owner adoption (or vice versa). The cron sync # ran the candidate computation outside any lock; the lock # is only held for the read-then-write of this single column. locked_scope = await session.scalar( select(CmdbCloudScope) .where(CmdbCloudScope.id == scope.id) .with_for_update() ) if locked_scope is None or locked_scope.owner_person_id: # Admin won the race — leave their assignment intact. continue locked_scope.owner_person_id = person_id locked_scope.updated_at = now scope.owner_person_id = person_id # keep in-memory copy in sync scope.updated_at = now auto_owners += 1 audit = AuditService(session) await audit.log( trace_id=f"grants-sync-{sync_job_id}", actor_type="system", actor_user_id=None, action="cloud.scope.auto_owner", status="success", resource_type="cmdb_cloud_scope", resource_id=str(scope.id), detail={ "scope_external_id": scope.external_id, "integration_id": str(integration.id), "provider": integration.provider, "new_owner_person_id": person_id, "reason": reason, "owner_pool_size": len(cands.owner), "active_pool_size": len(cands.active), }, ) await session.commit() # Continuous orphan-detection-in-sync removed 2026-05-07 (same-day # follow-up). The first cut tried to surface the orphan count by # calling ``scan_orphan_assignments(integration_id)`` after the # main commit — but ``scan_orphan_assignments`` re-iterates every # active scope and re-issues every ``list_permission_sets`` + # ``list_account_assignments`` + ``describe_user`` call the main # sync just made. Net effect: every grants_sync run doubled its # AWS API call volume. On a real prod-shaped integration (one AWS # Org with 50+ member accounts × 20 permission sets) this # routinely exceeded 90s and stalled the sync. The orphan-scan # cost is acceptable as an admin-on-demand action (with a # ``scope_id`` to narrow it); it is NOT acceptable as an # uncontrolled tail-step on every cron run. The admin-on-demand # flow at ``/cmdb/cloud-grants/orphan-scan?scope_id=…`` is the # supported path; piggy-backing detection on the existing scope # iteration (no second pass) is a future refactor. return { "status": "success" if not errors else "partial_failure", "integration_id": str(integration.id), "provider": integration.provider, "scopes_scanned": len(scope_plan), # Non-tenant-level rows the IAM fan-out cannot address (today: # ``azure_resource_group`` from the AOAI auto-grant flow). Named, # not just counted, so "why does this scope have no discovered # grants" is answerable from one sync summary. NEVER an error — # these rows are legitimate cost-attribution anchors. "scopes_skipped_non_tenant": skipped_non_tenant, "upserted": upserted, "touched": touched, "skipped_unresolved": skipped_unresolved, "unresolved_principals": sorted(set(unresolved_sample)), "skipped_per_scope": skipped_per_scope, "auto_owners_assigned": auto_owners, "drift_closed": drift_closed, "drift_closed_grant_ids": closed_grant_ids, "errors": errors, } @dataclass class _AutoOwnerCandidates: """Per-scope candidate pools for auto-owner adoption. ``owner`` — people holding a cloud owner-equivalent role on this scope (the ``driver.owner_role_keys`` set). Represents the highest privilege tier. ``active`` — everyone with an active grant on this scope, regardless of role. Used as the fallback pool when ``owner`` is empty so an unowned scope still gets a sensible default. Both map ``person_id -> earliest real granted_at from the cloud``, or ``None`` if the cloud provider doesn't supply timestamps. """ owner: dict[str, datetime | None] active: dict[str, datetime | None] def _merge_candidate( pool: dict[str, datetime | None], person_id: str, ts: datetime | None, ) -> None: """Insert or update the per-person earliest cloud-supplied ``granted_at``. A real timestamp always beats ``None``; between two real timestamps, keep the earlier one.""" if person_id not in pool: pool[person_id] = ts return prev = pool[person_id] if prev is None: pool[person_id] = ts return if ts is None: return if ts < prev: pool[person_id] = ts def _pick_auto_owner_person( cands: _AutoOwnerCandidates, ) -> tuple[str | None, str]: """Choose the ``person_id`` to auto-fill as scope owner. Selection order honours the user rule "prefer the highest privilege holder; if the cloud doesn't tell us who is oldest, give it to the first one — they can hand it over later": 1. If anyone holds an owner-equivalent role, pick from that pool only (权限最大的人 wins). 2. Otherwise fall back to any active-grant holder. 3. Within the chosen pool, the earliest real ``granted_at`` wins. Real timestamps rank above ``None``. 4. Tie-break lexicographically on ``person_id`` — deterministic "first person" pick when the cloud doesn't expose timestamps. Returns ``(person_id_or_None, reason)`` where ``reason`` names the rule that fired (used in the audit trail so admins can tell why any given scope ended up with its auto-assigned owner). """ if cands.owner: pool = cands.owner base = "owner_role" elif cands.active: pool = cands.active base = "active_grant_fallback" else: return None, "no_candidates" sorted_items = sorted( pool.items(), key=lambda kv: ( 0 if kv[1] is not None else 1, kv[1].isoformat() if kv[1] is not None else "", kv[0], ), ) person_id = sorted_items[0][0] if len(pool) == 1: tiebreak = "single" elif sorted_items[0][1] is not None: tiebreak = "earliest_granted_at" else: tiebreak = "first_by_person_id" return person_id, f"{base}:{tiebreak}" # AWS managed policies every IAM user gets by default for self-service # credential rotation. Auto-attached by the AWS console / IAM Identity # Center first-login flow; ALWAYS attached to every active IAM user so # they can rotate their own password / SSH keys / service credentials. # Excluded from the discovered-grant pipeline because they are not # business-grant decisions: revoking them blocks the user from the # self-service flow without removing any access they actually use, and # tracking them as grants generated tens of false "60 days no login" # inactivity alerts per leaver (one per ``ChangePassword``-policy row # per scope). Match by ARN suffix (case-sensitive — these are managed # policy names AWS guarantees stable) and by exact policy name (covers # the Tencent / Aliyun fallback path where the IAM-sync pipeline only # stores the policy name and not the ARN). _AWS_SELF_SERVICE_MANAGED_POLICIES: frozenset[str] = frozenset({ "IAMUserChangePassword", "IAMUserSSHKeys", "IAMSelfManageServiceSpecificCredentials", }) _AWS_SELF_SERVICE_MANAGED_POLICY_ARNS: frozenset[str] = frozenset({ f"arn:aws:iam::aws:policy/{name}" for name in _AWS_SELF_SERVICE_MANAGED_POLICIES }) def _is_self_service_managed_policy(arn: str, name: str) -> bool: """Return True for AWS-console-attached self-service managed policies. Both inputs are accepted because the IAM-sync pipeline stores both fields per attached policy and either may be empty (ARN absent for Tencent CAM and RAM custom-policy cases). """ if arn and arn in _AWS_SELF_SERVICE_MANAGED_POLICY_ARNS: return True if name and name in _AWS_SELF_SERVICE_MANAGED_POLICIES: return True return False def _email_from_iam_user( user_name: str, tags: dict | None, ) -> str | None: """Best-effort principal-email resolution for a CMDB IAM-user row. Mirrors the per-driver email logic so :func:`_derive_grants_from_iam_users` matches whatever ``list_scope_grants`` would have produced. Order: 1. ``user_name`` itself if it looks like an email — covers ~90% of AWS / Tencent / RAM users at PingCAP (``shuai.feng@pingcap.com``). 2. The ``email`` / ``user_email`` / ``email_address`` tag, with the tag key matched case-insensitively. These are documented but rare. 3. The ``Owner`` tag (production AWS uses this for everyone) — but only if the value contains ``@`` (skip ``BedrockAPIKey-d6l8``). Returns lowercased email or None. The lifecycle service drops grants with ``principal_email=None`` rather than guessing. """ if user_name and "@" in user_name: return user_name.lower() tags = tags or {} # Normalise to a case-insensitive lookup. norm = {str(k).lower(): str(v) for k, v in tags.items() if v is not None} for key in ("email", "user_email", "email_address"): v = norm.get(key, "").strip() if "@" in v: return v.lower() v = norm.get("owner", "").strip() if "@" in v: return v.lower() return None async def _derive_grants_from_iam_users( session: AsyncSession, integration_id: str, scopes: list[CmdbCloudScope], ) -> list[DiscoveredGrant]: """Synthesise ``DiscoveredGrant`` rows from ``cmdb_iam_users``. The IAM-sync pipeline already pulls per-user attached-policy data every hour and stores it in ``cmdb_iam_users.attached_policies`` (a JSON array of ``{name, arn}``). When the per-driver ``list_scope_grants`` returns nothing — broken auth scope, missing Identity Center config, RAM master-only roles — we can still reconstruct an accurate grant view by walking that table. Costs zero cloud API calls. The ``scope`` field on each emitted grant is left as a placeholder pointing at the integration's first active scope; the caller fans out to every scope (see ``sync_grants_for_integration`` fallback block) because IAM users are tenant-wide for the providers this helper rescues. Self-service AWS managed policies (``IAMUserChangePassword``, ``IAMUserSSHKeys``, ``IAMSelfManageServiceSpecificCredentials``) are filtered here. They are auto-attached by the AWS console for every IAM user so they can rotate their own credentials — they do not represent business-grant decisions and were generating false inactivity-sweep alerts on every employee in every IAM-mode AWS tenant (60+ noisy "60 days no login" notifications per leaver, none of which corresponded to a real revocable permission). """ if not scopes: return [] rows = (await session.execute( select(CmdbIamUser).where( CmdbIamUser.integration_id == integration_id, CmdbIamUser.deleted_at.is_(None), ) )).scalars().all() if not rows: return [] # Pre-resolve work_email for every IAM user that already has a # bound ``owner_person_id``. This is the AliCloud-RAM rescue path: # the IAM-sync ``_resolve_iam_owner_person_id`` (sync.py) already # ran the email-local-part fallback (``li.da`` → # ``li.da@pingcap.com``) and persisted the binding, so # ``_apply_discovered_grants`` can resolve the grant if we just # plug the right email in here. bound_person_ids = sorted({ str(u.owner_person_id) for u in rows if u.owner_person_id }) person_id_to_email: dict[str, str] = {} if bound_person_ids: person_rows = (await session.execute( select(IdentityPerson.id, IdentityPerson.work_email).where( IdentityPerson.id.in_(bound_person_ids), IdentityPerson.deleted_at.is_(None), ) )).all() person_id_to_email = { str(pid): str(email or "").strip().lower() for pid, email in person_rows if email and "@" in str(email) } # Belt-and-braces: if cron fires grants_sync BEFORE the IAM-sync # owner-resolve has run, owner_person_id is still NULL on fresh # rows. Pre-compute an email-local-part lookup for the # @-less ``user_name``s in this batch so we don't lose grants to # scheduling order. Same "exactly one match" guard as # _resolve_iam_owner_person_id — collisions stay unresolved. no_email_user_names = sorted({ (u.user_name or "").lower() for u in rows if u.user_name and "@" not in u.user_name and not u.owner_person_id and not _email_from_iam_user(u.user_name or "", u.tags) }) local_part_to_email: dict[str, str] = {} if no_email_user_names: local_rows = (await session.execute( select(IdentityPerson.work_email).where( func.lower( func.substring_index(IdentityPerson.work_email, "@", 1) ).in_(no_email_user_names), IdentityPerson.deleted_at.is_(None), ) )).all() local_count: dict[str, int] = {} local_email: dict[str, str] = {} for (email_val,) in local_rows: email_str = str(email_val or "").strip().lower() if "@" not in email_str: continue local = email_str.split("@", 1)[0] local_count[local] = local_count.get(local, 0) + 1 local_email[local] = email_str # Only commit unambiguous matches. local_part_to_email = { local: email for local, email in local_email.items() if local_count.get(local, 0) == 1 } placeholder_scope = CloudScopeRef( scope_type=scopes[0].scope_type, external_id=scopes[0].external_id, display_name=scopes[0].display_name, ) out: list[DiscoveredGrant] = [] for u in rows: email = _email_from_iam_user(u.user_name or "", u.tags) # Owner-bound fallback: if the IAM-sync identity-resolution # step already linked this row to a Helix person (e.g. via # email-local-part), use that person's work_email — even if # the user_name / tags don't surface an email directly. if not email and u.owner_person_id: email = person_id_to_email.get(str(u.owner_person_id)) # Cron-order rescue: owner-resolve hasn't run yet but we can # still try the local-part lookup ourselves. if not email and u.user_name: email = local_part_to_email.get(u.user_name.lower()) if not email: continue policies = u.attached_policies or [] if not isinstance(policies, list): continue for p in policies: if not isinstance(p, dict): continue arn = str(p.get("arn") or "").strip() name = str(p.get("name") or "").strip() if not arn and not name: continue if _is_self_service_managed_policy(arn, name): continue # ``role_key`` matches whatever the driver's ``list_scope_roles`` # uses — for AWS that's the policy ARN, for Tencent / RAM # that's the policy name. Use ARN if it looks like an ARN # (``arn:`` / ``acs:`` / ``qcs:``); otherwise fall back to # name. This keeps the (scope_id, person_id, role_key) # uniqueness aligned with helix-issued grants. looks_like_arn = arn.startswith( ("arn:", "acs:", "qcs:", "/providers/"), ) role_key = arn if (looks_like_arn and arn) else (name or arn) out.append(DiscoveredGrant( principal_email=email, principal_type="user", role_key=role_key, role_display_name=name or None, external_ref=f"iam-user:{u.user_name}::{role_key}", scope=placeholder_scope, granted_at=u.create_date, raw={ "source": "cmdb_iam_users.attached_policies", "iam_user_name": u.user_name, "policy_arn": arn or None, "policy_name": name or None, "iam_user_id_external": u.user_id_external, }, )) return out async def _apply_discovered_grants( session: AsyncSession, scope: CmdbCloudScope, grants: list[DiscoveredGrant], driver: CloudDriver, sync_job_id: str, now: datetime, ) -> tuple[int, int, int, _AutoOwnerCandidates]: """Upsert every DiscoveredGrant where the principal resolves to an active ``IdentityPerson``. Returns ``(upserted, touched, skipped_unresolved, candidates)``. ``candidates`` is consumed by the caller to auto-assign the scope owner using ``_pick_auto_owner_person``. """ owner_role_keys: set[str] = set( getattr(driver, "owner_role_keys", frozenset()) or frozenset() ) cands = _AutoOwnerCandidates(owner={}, active={}) upserted = 0 touched = 0 skipped_unresolved = 0 unresolved_emails: list[str] = [] # Collect unique emails to batch-resolve to person_id. This keeps # the DB roundtrips to 1 per integration-scope regardless of grant # count. emails = sorted({ g.principal_email.strip().lower() for g in grants if g.principal_type == "user" and g.principal_email }) if not emails: return 0, 0, 0, cands, [] # Resolve emails to (person_id, employment_status) so we can: # * skip INSERTing a new ``discovered`` row for leaver_pending / left # people (otherwise we re-grant 90 days of access past the # offboarding fence — ``leaver_grant_sweep`` only catches # ``employment_status='left'`` so a freshly-resurrected row # survives the leaver_pending window); # * STILL update ``last_seen_grants_sync_*`` on EXISTING rows for # leaver_pending people so the drift sweep doesn't DB-revoke # them and leave the cloud-side assignment live (it would, # because ``leaver_grant_sweep`` skips status='revoked' rows). # We include leaver_pending rows in the resolver but tag them so # the per-grant loop only updates and never inserts. rows = ( await session.execute( select( IdentityPerson.id, IdentityPerson.work_email, IdentityPerson.employment_status, ) .where( func.lower(IdentityPerson.work_email).in_(emails), IdentityPerson.deleted_at.is_(None), ) ) ).all() email_to_person: dict[str, str] = { str(row[1]).strip().lower(): str(row[0]) for row in rows } # Person IDs whose employment_status would make a fresh discovered # grant a security regression. Used by the per-grant INSERT branch. leaver_person_ids: set[str] = { str(row[0]) for row in rows if row[2] in ("left", "leaver_pending") } # ── Bare-handle tier: IAM user name → work_email local-part ── # IAM-mode clouds (Tencent CAM, Aliyun RAM) let the admin name a # sub-user anything, and the optional ``Email`` attribute is almost # never filled in. On the prod Tencent tenant that leaves handles like # ``zhuoran.li`` / ``wuhui.zuo`` (people, holding 7 and 9 policies) # next to ``tiup-server`` / ``operator-v1-cos-upload`` (robots). The # exact-``work_email`` tier above can NEVER match the first group — # there is no domain to match on — so those principals were invisible # in the members table while their access was live. # # Resolution rule is the one the IAM-user sync already trusts for # ``cmdb_iam_users.owner_person_id`` (``_resolve_iam_owner_person_id`` # tier 2): bind only when EXACTLY ONE **active** person owns that # local-part. ``shuai`` must never silently pick between # ``shuai.feng@`` and ``shuai.zhang@`` — a wrong bind here has # security weight (sweeps revoke through these rows). Leavers are # excluded on purpose: a handle whose only match has already left # (prod: ``chujie``, still carrying 4 CAM policies) must stay # UNRESOLVED so it lands in the summary's named list, where an admin # can act on it — binding it would hide it behind a revoked row that # no sweep revisits. bare_handles = [e for e in emails if "@" not in e] if bare_handles: handle_rows = ( await session.execute( select(IdentityPerson.id, IdentityPerson.work_email).where( func.lower( func.substring_index(IdentityPerson.work_email, "@", 1) ).in_(bare_handles), IdentityPerson.deleted_at.is_(None), IdentityPerson.employment_status == "active", ) ) ).all() persons_by_local: dict[str, set[str]] = {} for person_row in handle_rows: local = str(person_row[1] or "").strip().lower().split("@", 1)[0] if not local: continue persons_by_local.setdefault(local, set()).add(str(person_row[0])) for handle in bare_handles: matched = persons_by_local.get(handle) or set() if len(matched) == 1: email_to_person.setdefault(handle, next(iter(matched))) # Perpetual holders (cloud/global admins) must NOT get a TTL stamped on a # discovered row. This was the missing check that caused the # 2026-07-29 cn68 incident — see ``load_perpetual_person_ids``. The # approve / owner-add / renew writers all consulted the perpetual # predicate; this writer did not, so a cloud admin whose access came # from a cloud-side sync got a 90-day clock and was auto-revoked # while the UI said 「永久授权」. perpetual_person_ids = await load_perpetual_person_ids(session) # A provider's grant listing does not always carry a human-readable role # name. AWS Identity Center is the most visible example: assignments # contain only a permission-set ARN, while the already-synced role catalog # holds the friendly permission-set name (for example ``SRE-Admin``). # Resolve catalog names here, at the shared discovery boundary, so every # cloud provider gets the same display behaviour and every existing # discovered row is repaired on its next grants sync. role_keys = {g.role_key for g in grants if g.role_key} catalog_names: dict[str, str] = {} if role_keys: catalog_rows = (await session.execute( select( CmdbCloudRole.role_key, CmdbCloudRole.display_name, CmdbCloudRole.scope_id, ).where( CmdbCloudRole.integration_id == scope.integration_id, CmdbCloudRole.role_key.in_(role_keys), CmdbCloudRole.deleted_at.is_(None), or_( CmdbCloudRole.scope_id == str(scope.id), CmdbCloudRole.scope_id.is_(None), ), ) )).all() # Scope-specific catalog rows override tenant-wide rows. Preserve the # provider's own display string only when the catalog has no usable # name, so custom/un-synced roles still remain intelligible. for role_key, display_name, catalog_scope_id in catalog_rows: name = str(display_name or "").strip() if not name: continue key = str(role_key) if ( key not in catalog_names or str(catalog_scope_id or "") == str(scope.id) ): catalog_names[key] = name for grant in grants: if grant.principal_type != "user": continue email = grant.principal_email.strip().lower() person_id = email_to_person.get(email) if not person_id: skipped_unresolved += 1 # Record WHO, not just how many. These principals hold live # access cloud-side that Helix has no row for at all: they are # in no members table, no 已结束 tab, and no sweep can ever # touch them. aws-account 385595570414 had 11 on # 2026-07-29 — visible only as an integer in the cron's # ``result_summary``, so nobody could act on it. Capped so a # badly-mapped tenant cannot bloat the summary JSON. if len(unresolved_emails) < _UNRESOLVED_SAMPLE_CAP: unresolved_emails.append(email) continue role_display_name = ( catalog_names.get(grant.role_key) or (grant.role_display_name or "").strip() or None ) existing = ( await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == str(scope.id), CmdbCloudAccessGrant.person_id == person_id, CmdbCloudAccessGrant.role_key == grant.role_key, ) ) ).scalars().first() if existing is not None: # Helix-issued rows stay badged ``helix``; we just update # the "last seen cloud-side" markers so drift detection # works. existing.last_seen_grants_sync_job_id = sync_job_id existing.last_seen_grants_sync_at = now # Only pull provider-side detail back when this WAS # originally a discovered row — we don't want cloud-side # drift silently overwriting detail on helix-issued rows. if existing.source == "discovered": if grant.external_ref: existing.external_ref = grant.external_ref existing.detail = dict(grant.raw or {}) existing.role_display_name = ( role_display_name or existing.role_display_name ) # Backfill expires_at on legacy discovered rows inserted # before the TTL-at-discovery change (migration 063 did # the bulk backfill; this is the safety net for rows # that somehow sneak in with NULL after that point). # # NULL is NOT always "missing" though — it is also how the # platform SPELLS "perpetual". Two populations legitimately # hold NULL and must never be re-stamped here: # # * cloud/global admins (``load_perpetual_person_ids``), and # * the scope's own owner, whose rows the auto-owner # backfill writes with ``force_perpetual=True``. # # Without this guard the nightly sync fights the expiry # sweep's normalization pass forever: sweep clears the TTL # at 08:15, sync re-stamps it at 05:23 the next morning. # It is also the most likely origin of the 140 owner rows # found carrying a long-stale ``expires_at`` — the owner # exemption hid the consequence, so nothing ever surfaced # the re-stamping. if ( existing.expires_at is None and person_id not in perpetual_person_ids and str(scope.owner_person_id or "") != person_id ): existing.expires_at = now + timedelta( days=int(scope.default_grant_ttl_days or 90), ) touched += 1 # Skip owner-candidate / active-candidate aggregation for # leaver_pending / left persons. The existing-row branch # still updates ``last_seen_grants_sync_*`` (so drift # detection sees the row as cloud-side-extant — the # offboarding pipeline is responsible for the revoke), # but a leaver MUST NOT be promoted to scope owner via # auto-owner inference. The candidate dict downstream # picks the highest-priority candidate as the new owner # when the scope has no owner_person_id. if ( existing.status == GRANT_STATUS_ACTIVE and person_id not in leaver_person_ids ): _merge_candidate(cands.active, person_id, grant.granted_at) if grant.role_key in owner_role_keys: _merge_candidate(cands.owner, person_id, grant.granted_at) else: # Don't insert a fresh discovered grant for a leaver_pending # / left person — the offboarding pipeline is responsible # for closing cloud-side access; resurrecting a grant here # would extend that access past the offboarding fence by # 90 days. (Existing rows for the same person already get # ``last_seen`` updates above so the drift sweep doesn't # close them DB-side while the cloud-side is still live; # the leaver-pipeline handles the real revoke.) if person_id in leaver_person_ids: skipped_unresolved += 1 # Distinct from the unmapped case above: we DO know who # this is, and the offboarding pipeline owns their access. # Not sampled — surfacing leavers here would duplicate the # leaver sweep's own reporting. continue # Discovered grants get expires_at = now + scope's default # TTL. We deliberately do NOT use ``grant.granted_at`` (the # cloud-side creation time) because legacy grants carry an # anchor years in the past — +90d from that is already # expired, which would instantly revoke production access. ttl_days = int(scope.default_grant_ttl_days or 90) # Perpetual holders (cloud admins) and the scope's own owner # get ``expires_at = NULL`` — the same value the approve / # owner-add / renew writers produce for them. Stamping a TTL # here is what made 150 cloud-admin grants silently # revocable while the UI advertised 「永久授权」. # # The TTL that IS stamped is jittered rather than landing on # a single instant for every principal discovered in the same # sync — see ``_discovery_ttl_expires_at``. discovered_expires_at: datetime | None = ( None if ( person_id in perpetual_person_ids or str(scope.owner_person_id or "") == person_id ) else _discovery_ttl_expires_at( now=now, ttl_days=ttl_days, scope_id=str(scope.id), person_id=person_id, role_key=grant.role_key, ) ) new_row = CmdbCloudAccessGrant( id=uuid.uuid4(), scope_id=str(scope.id), person_id=person_id, role_key=grant.role_key, role_display_name=role_display_name, status=GRANT_STATUS_ACTIVE, granted_at=grant.granted_at or now, expires_at=discovered_expires_at, granted_by_person_id=None, external_ref=grant.external_ref, detail=dict(grant.raw or {}), source="discovered", last_seen_grants_sync_job_id=sync_job_id, last_seen_grants_sync_at=now, ) session.add(new_row) upserted += 1 _merge_candidate(cands.active, person_id, grant.granted_at) if grant.role_key in owner_role_keys: _merge_candidate(cands.owner, person_id, grant.granted_at) # Update scope's last-synced marker for the grants sweep — useful # for the UI "last grants check" badge. scope.last_seen_sync_job_id = sync_job_id scope.last_synced_at = now await session.flush() return upserted, touched, skipped_unresolved, cands, unresolved_emails # ── Daily billing sync + spike alert ────────────────────────── async def sync_billing_daily_for_integration( integration_id: str, *, start_date: date | None = None, end_date: date | None = None, dispatch_alerts: bool = True, ) -> dict[str, Any]: """Ingest daily billing rows + evaluate spike / threshold alerts. ``dispatch_alerts`` (default ``True``) controls whether the per- scope spike detection and user-defined-rule evaluation are run after each scope's daily rows land. The daily cron leaves this at the default so production alerts continue firing. The backfill walker (:mod:`server.scripts.backfill_cost_facts`) passes ``False`` because it replays multi-month historical windows; with alerts enabled, every backfilled chunk would fire a separate Feishu card per scope per crossed threshold (prod incident 2026-05-16 issued ~7 cards per scope across 60 scopes). Even with ``dispatch_alerts=False`` we still write to ``cmdb_cloud_billing_daily`` and still **claim** the alert dedup slots — so the next live cron tick after a backfill does not re-page on historical data. See :func:`evaluate_scope_alert_rules` (``dispatch=False`` mode) and :func:`_alert_spike` (``dispatch=False`` mode). """ async with async_session_factory() as session: integration = await session.get(Integration, uuid.UUID(integration_id)) if integration is None or integration.kind != "cloud": return { "status": "skipped", "reason": "integration missing or not cloud", "integration_id": integration_id, } # Match the ``sync_cost_facts_for_integration`` contract — a # disabled cloud integration must not burn provider API quota # (AWS Cost Explorer is billed/rate-limited; Aliyun BSS has # daily caps). Pre-fix the sibling check existed in cost-facts # only, so the disabled-integration billing-daily cron kept # firing and writing rows with the old credentials. if not integration.enabled: return { "status": "skipped", "reason": "integration disabled", "integration_id": integration_id, } end_dt = end_date or (datetime.now(UTC).date() - timedelta(days=1)) start_dt = start_date or (end_dt - timedelta(days=2)) # Guard against an inverted window from a misconfigured manual # backfill — silently looping 0 times would skip the eviction # path entirely but still record a "success" run, masking the # mis-config. if start_dt > end_dt: return { "status": "failed", "reason": ( f"invalid window: start={start_dt} > end={end_dt}" ), "integration_id": integration_id, } # Filter to tenant-level scope types only. Sub-tenant rows # (today: ``azure_resource_group`` minted by the Azure OpenAI # auto-grant flow) carry RG names in ``external_id`` and are # NOT queryable units of the cloud cost APIs — the Azure # driver builds ``/subscriptions/{external_id}/...`` URLs. # Same filter as ``cost_service.sync_cost_facts_for_integration``. # # Round-20 audit (A1F15): also restrict to ACTIVE scopes. # Closed (decommissioned) scopes used to fall into the # billing-daily loop and waste provider API quota — AWS Cost # Explorer is rate-limited and billed per call. Mirrors the # ``status == SCOPE_STATUS_ACTIVE`` predicate used by the # grants_sync / inactivity-sweep cron paths. scopes = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.integration_id == str(integration.id), CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.scope_type.in_(TENANT_LEVEL_SCOPE_TYPES), CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, ) )).scalars().all() driver = await _load_cloud_driver(session, integration) # Let the driver tell us up-front if the integration is mis- # configured (e.g. GCP without billing_bq_table). Without this # the job would quietly return upserted=0 and the admin would # have no idea why. try: precheck_reason = await driver.billing_daily_precheck() except Exception as exc: # noqa: BLE001 precheck_reason = ( f"billing_daily_precheck raised: " f"{format_exception_for_log(exc)}" ) if precheck_reason: try: await driver.close() except Exception: # noqa: BLE001 pass return { "status": "skipped", "reason": precheck_reason, "upserted": 0, "errors": [], "spikes_alerted": 0, } # Resolve BQ CUR source once per cron tick (mirrors the # cost_facts cron's pattern). For AWS with billing_source= # bigquery this populates the driver's bq_billing kwarg with # SA + dataset; other providers / AWS-on-CE see None and # the driver short-circuits past the BQ branch. try: bq_billing = await _resolve_aws_bq_billing(session, integration) except RuntimeError as exc: try: await driver.close() except Exception: # noqa: BLE001 pass return { "status": "failed", "reason": ( f"BQ billing source misconfigured: " f"{format_exception_for_log(exc)}" ), "upserted": 0, "errors": [str(exc)], "spikes_alerted": 0, } upserted = 0 spikes_alerted = 0 errors: list[str] = [] try: # (The registered-account index used to be built HERE and is # now built as ``covered_pairs`` just before the # reconciliation pass below — it has to be read AFTER the # per-scope loop has written this run's rows. See #2221.) # AWS CUR-in-BigQuery (per_month): hand the driver the full # target set so its billing_daily path scans all accounts in # one pass per query instead of one query per account. The CUR # month-tables are DAY-partitioned and NOT clustered by # account, so the per-account WHERE filter bills identical # bytes whether it matches one account or all — the per-account # fan-out (here ×3 queries: totals/service/owner) is a pure N× # scan-cost multiplier billed to the GCP project running the SA. # Set only when bq_billing is present (AWS-BQ mode); other # providers / the CE path leave it None and keep per-account. if bq_billing is not None: driver._billing_daily_all_account_scopes = [ CloudScopeRef( scope_type=s.scope_type, external_id=s.external_id, display_name=s.display_name, ) for s in scopes if s.external_id ] # Read once per sync run, unconditionally — two settings # lookups is nothing next to the per-scope cloud calls, and # an eager read keeps these plain floats instead of # ``float | None`` threaded through the comparisons below # (#2189-4). spike_multiplier = await _setting_float( session, "cloud.daily_spike_multiplier", 1.5, ) spike_min_usd = await _setting_float( session, "cloud.daily_spike_min_usd", 10, ) for scope in scopes: try: rows = await driver.list_billing_daily( start_dt, end_dt, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), include_owner_breakdown=True, bq_billing=bq_billing, ) except NotImplementedError: continue except Exception as e: # noqa: BLE001 errors.append( f"{scope.external_id}: {format_exception_for_log(e)}" ) continue for row in rows: stmt = mysql_insert(CmdbCloudBillingDaily).values( id=str(uuid.uuid4()), integration_id=str(integration.id), scope_id=str(scope.id), linked_account_id=scope.external_id or "", date=row.date, total_cost_usd=row.total_cost_usd, # 双口径:None 透传 NULL(=口径未接入),读侧据此 # 区分「$0」与「没有数」。 total_cost_usd_list=row.total_cost_usd_list, total_cost_usd_public=row.total_cost_usd_public, cost_by_service=row.cost_by_service or {}, currency=row.currency, raw_response=row.raw or {}, owner_breakdown=row.owner_breakdown or None, fetched_at=datetime.now(UTC), ).on_duplicate_key_update( total_cost_usd=row.total_cost_usd, total_cost_usd_list=row.total_cost_usd_list, total_cost_usd_public=row.total_cost_usd_public, cost_by_service=row.cost_by_service or {}, currency=row.currency, raw_response=row.raw or {}, owner_breakdown=row.owner_breakdown or None, fetched_at=datetime.now(UTC), ) await session.execute(stmt) upserted += 1 # Spike detection: compare latest day vs past 7-day avg. # Round-51 R3-P0-2: admin typo on the multiplier setting # (negative / zero / 1e308) can either spam every scope # with phantom spikes or silently mute the entire alert # path. Clamp to a plausibility band, matching the # same defence applied to FX rate in # ``app.core.fx_rate``. Out-of-band values fall back to # the default + warning rather than raising — operator # mistyping a setting should not break the daily cron. multiplier = spike_multiplier min_usd = spike_min_usd if not (1.0 <= multiplier <= 100.0): logger.warning( "cloud.daily_spike_multiplier out of plausible band " "(%.4f); falling back to default 1.5", multiplier, ) multiplier = 1.5 if min_usd < 0: logger.warning( "cloud.daily_spike_min_usd negative (%.4f); " "falling back to default 10", min_usd, ) min_usd = 10.0 past = (await session.execute( select(CmdbCloudBillingDaily).where( CmdbCloudBillingDaily.integration_id == str(integration.id), CmdbCloudBillingDaily.scope_id == str(scope.id), CmdbCloudBillingDaily.date >= end_dt - timedelta(days=7), CmdbCloudBillingDaily.date < end_dt, ).order_by(CmdbCloudBillingDaily.date.asc()) )).scalars().all() today_row = next( (r for r in rows if r.date == end_dt), None, ) # Convert native-currency rows to USD before any # threshold math. CN drivers (Aliyun / Tencent / UCloud # / KSyun) write the native CNY amount into # ``total_cost_usd`` (legacy column name); without this # conversion, every threshold below — daily-spike # ``min_usd`` floor, monthly budget, user-defined alert # rules in USD — would fire at ~1/7 the intended trigger # point. Round-20 audit (A1F1) landed this. We resolve # FX once per scope here so the call is amortised across # ``min_usd`` + ``avg`` + ``evaluate_scope_alert_rules``. row_currency_today = ( (today_row.currency or "USD").upper() if today_row is not None else "USD" ) cny_per_usd: float | None = None async def _to_usd(amount: float, row_currency: str) -> float: """Convert a daily-row amount into USD using the platform FX source-of-truth. USD passes through. CNY divides by ``resolve_cny_per_usd``. Unknown currencies pass through with a warning — better to over-alert (raw value) than to silently zero-out spend on an unrecognised currency. """ nonlocal cny_per_usd cur = (row_currency or "USD").upper() if cur == "USD": return float(amount) if cur == "CNY": if cny_per_usd is None: from app.core.fx_rate import resolve_cny_per_usd cny_per_usd = await resolve_cny_per_usd( session, integration_config=( integration.config_public or {} ), ) if not cny_per_usd or cny_per_usd <= 0: # Round-51 R1-P1-6 / R3-P0-4: a missing or # non-positive FX rate must NOT silently # pass-through CNY as USD — the # ``today_usd > avg_usd * multiplier`` check # is ratio-invariant (safe), but # ``delta_usd > min_usd`` compares CNY to a # USD threshold and fires ~7× more often # than intended. ``resolve_cny_per_usd`` # raises on missing/stale; this branch is # the defence-in-depth. Raise so the cron # fails loudly + ``failure_notifier`` pages # admins; threshold alerts are too # important to compute on bad units. raise RuntimeError( "cloud.daily_spike: CNY FX rate is " "missing or non-positive; refusing to " "fall through to raw CNY" ) return float(amount) / cny_per_usd logger.warning( "cloud.billing_daily: unrecognised currency %r " "on scope %s; falling back to raw amount for " "threshold comparison", cur, scope.id, ) return float(amount) # Round-51 R3-P0-1: require a minimum of 7 baseline days # before any spike fires. On day-2 of a freshly-registered # scope (or first cron tick after migration) ``past`` had # length 1 and a totally normal $5→$50 warmup tripped the # multiplier — pages owners + IT admins for noise. The # on-demand ``cost_service.detect_anomalies`` already # enforces ``len(points) < 7`` (cost_service.py:2349); the # cron path must match. _SPIKE_BASELINE_MIN_DAYS = 7 # Round-51 regression-review #4: ``_to_usd`` RAISES on # missing / non-positive FX rate (R1-P1-6). Without # this try, one bad-FX scope abort the rest of the # integration's loop because there's no outer handler. # Catch + record + skip this scope's alert path; the # daily-row upserts above already landed. fx_failed_for_this_scope = False avg_usd = 0.0 today_usd = 0.0 # 口径一致性 (2026-07-28): 尖峰是「比值」告警,分子分母 # 必须同口径。此前的按行 COALESCE 取值 (list→净额,与读侧 # daily_engineering_amount 同语义) 在双口径回填过渡期会产 # 生混合基线——2026-07-27 # 生产事故: gcp-autonomous-prod 折扣 ~55%,3 天同步窗口刚写 # 出 list 值、基线还剩 5 天净额行,昨日 list $1,163 vs 混合 # 基线 $706 = 1.65× 假尖峰(真实 list 花费三天全平稳)。 # 只有当「昨日 + 全部基线行」都有 list 值才用 list 口径, # 否则整组回落净额——两侧永远同列,比值不受回填进度影响。 spike_measure = "net" if today_row is not None and len(past) >= _SPIKE_BASELINE_MIN_DAYS: use_list = ( today_row.total_cost_usd_list is not None and all( p.total_cost_usd_list is not None for p in past ) ) spike_measure = "list" if use_list else "net" # ``r`` duck-types: DB row (CmdbCloudBillingDaily) # for baseline days, driver row (CloudDailyCost) # for today — both expose the two measure attrs. def _spike_value(r: Any) -> float: if use_list: return float(r.total_cost_usd_list) return float(r.total_cost_usd or 0) try: # Past 7d average converted row-by-row in case the # provider mid-period switched currency (e.g. an # admin re-bound an intl scope to a CN integration). past_usd = [ await _to_usd( _spike_value(p), (p.currency or "USD"), ) for p in past ] avg_usd = float(sum(past_usd) / len(past_usd)) today_usd = await _to_usd( _spike_value(today_row), row_currency_today, ) except RuntimeError as fx_exc: # FX missing/stale — record + skip spike alert # for this scope. Other scopes on the same # integration may have legit USD currency and # should not be blocked by this one's CN # FX-rate gap. The cron's failure_notifier picks # up the FX-cron itself via its own path. errors.append( f"{scope.external_id}: fx-conversion-failed: " f"{fx_exc}" ) logger.warning( "cloud.daily_spike: skipping scope %s due to " "FX rate failure: %s", scope.id, fx_exc, ) fx_failed_for_this_scope = True if fx_failed_for_this_scope: # Daily-row upsert already done above; commit it # (see the per-scope commit note at the end of the # loop) before skipping this scope's alert path so # the FX-fail path doesn't carry the rows into the # next scope's transaction. await session.commit() continue if today_row is not None and len(past) >= _SPIKE_BASELINE_MIN_DAYS: delta_usd = today_usd - avg_usd if ( avg_usd > 0 # Refund / credit days are stored as negative # amounts; ``today_usd <= 0`` should never be # called a "spike high". Without this gate a # post-refund return-to-normal (avg negative, # today positive) would still hit the # ``today_usd > avg_usd * multiplier`` rule and # spam an alert on what is just normal usage. and today_usd > 0 and today_usd > avg_usd * multiplier and delta_usd > min_usd # Legacy stamp fast-path skip — deploy-day # back-compat. Pre-migration-171 the only # spike dedup was this column. A scope whose # last_spike_alert_date is already end_dt on # the FIRST post-deploy cron tick must NOT # claim a fresh slot in the (empty-on-deploy) # alert-fired table and re-fire today's card. and scope.last_spike_alert_date != end_dt ): # Threshold-independent pre-claim gate — admin # tuning the daily-spike multiplier mid-day # (1.5× → 1.2×) would otherwise open a NEW dedup # slot under the new multiplier and re-page the # same scope on the same data. Refuse second # spike alert per (scope, day) regardless of # multiplier value at trigger time. Use a flag # rather than ``continue`` so the post-spike # user-defined rule evaluation below still runs # — they're independent alert pipelines. already_spiked = await _period_already_alerted( session, scope_id=str(scope.id), rule_type=CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY_SPIKE, period_key=end_dt.isoformat(), ) # Atomically claim the (scope, daily_spike, date, # multiplier) dedup slot in # cmdb_cloud_scope_alert_fired. The legacy # ``scope.last_spike_alert_date`` stamp is still # updated below for back-compat, but the new # table is authoritative and persists immediately # via SAVEPOINT — so a later rollback can't drop # the dedup state while the Feishu card is # already in flight. spike_claimed = ( False if already_spiked else await _try_claim_cloud_scope_alert_slot( session, scope_id=str(scope.id), rule_type=CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY_SPIKE, period_key=end_dt.isoformat(), threshold_usd=Decimal(str(multiplier)), current_usd=Decimal(str(today_usd)), ) ) if spike_claimed: # Round-51 R3-P1-7: stamp the legacy # ``last_spike_alert_date`` only when an # alert actually went out. Backfill walks # (``dispatch_alerts=False``) used to stamp # the legacy column with the backfill # end_dt, leaving any operator UI that # still reads this column with a wrong # "last spike was N months ago" label. The # authoritative dedup is the new # ``cmdb_cloud_scope_alert_fired`` row # which is still claimed above. if dispatch_alerts: if ( scope.last_spike_alert_date is None or end_dt > scope.last_spike_alert_date ): scope.last_spike_alert_date = end_dt # Round-51 R1-P0-3: card MUST render in # USD-normalised units. The previous # branch passed native_today (whatever # currency today's row happens to be) # and a past-7d native average — but # ``past`` may carry rows in a # DIFFERENT currency (admin rebound the # scope mid-week). The result was # operator-confusing: "今日 ¥10,000 vs # 7日均值 ¥1,500 — 6.67×" when the # truth was USD 1,400 vs USD 1,500 (no # spike). Use the already-computed # ``today_usd`` and ``avg_usd`` so the # multiplier the operator reads # matches the multiplier that # triggered the alert. await _alert_spike( session, integration, scope, today_cost=today_usd, avg=avg_usd, multiplier=multiplier, currency="USD", measure=spike_measure, ) spikes_alerted += 1 # Round-51 R3-P0-3: forensic trail for # "why did this card fire at 3am". await _audit_cloud_scope_alert_fired( scope_id=str(scope.id), integration_id=str(integration.id), rule_type=CLOUD_SCOPE_ALERT_RULE_TYPE_DAILY_SPIKE, period_key=end_dt.isoformat(), threshold_usd=Decimal(str(avg_usd * multiplier)), current_usd=Decimal(str(today_usd)), baseline_usd=Decimal(str(avg_usd)), multiplier=multiplier, fx_rate_cny_per_usd=cny_per_usd, dispatched=True, # Same basis the comparison above ran # on. On ``net`` the audit row keeps # the ratio and withholds the # absolute figures — see the # function's docstring. measure=spike_measure, ) # Per-scope user-defined alert rules. Computed off the # freshly written daily rows so we don't re-query. # USD-converted values feed the comparator since rule # thresholds live in ``threshold_usd``. if today_row is not None and scope.alert_rules: # Round-666 hardening: the spike block above wraps its # ``_to_usd`` calls in ``try/except RuntimeError`` only # when ``len(past) >= _SPIKE_BASELINE_MIN_DAYS`` (7). For # a freshly-registered CN-cloud scope (currency='CNY') # with default alert rules but < 7 baseline days, that # guard is SKIPPED, ``cny_per_usd`` stays unresolved, and # the MTD/prev-month ``_to_usd`` calls below would RAISE # on a missing/stale FX rate. The only enclosing handler # is the loop's ``finally`` (driver.close) — no ``except`` # — so the RuntimeError would propagate out of the whole # ``for scope`` loop, skip this scope's per-scope commit # AND every later scope AND the unregistered-account # reconciliation pass, and abort the run instead of # returning the partial_failure envelope. Mirror the # spike path's FX guard so one bad-FX scope is skipped # gracefully regardless of baseline-day count. try: first_of_month = end_dt.replace(day=1) mtd_rows = (await session.execute( select(CmdbCloudBillingDaily).where( CmdbCloudBillingDaily.scope_id == str(scope.id), CmdbCloudBillingDaily.date >= first_of_month, CmdbCloudBillingDaily.date <= end_dt, ) )).scalars().all() # 口径一致性 (2026-07-28,与上面尖峰块同一根因): # 一次比较的输入行必须整组同口径。按行 COALESCE 在 # 回填过渡期把 MTD 变成「净额旧天 + list 新天」的混 # 合和,而 threshold_usd 全部是按净额口径定的 —— # 2026-07-24 list 写侧上线当天,pingcap-gardener 的 # 日阈值 ($250,净额从未超过) 即连续三天假触发,一批 # 月度阈值被混合 MTD 提前顶过线。 # # 2026-07-29 复审收紧(#2079 三轮):口径是「每条 # 规则」的属性,不是「整组」的属性。整组 all() 闸有 # 两个自相矛盾的后果:(a) 任何一条未标记规则(scope # owner 通过 PUT /scopes/{id}/alerts 就能加,retune # 还会掉标记)把同 scope 其余 list 规则全部拖回净额 # 评估 —— 阈值高 2.7×,永远打不到;(b) 数据齐全本身 # 不构成切口径依据(月初翻月后 MTD 全是新双口径行, # 「齐全」自动成立)。 # # 现在:按需要的口径各算一份**整组同口径**聚合,每条 # 规则跟自己标记的那一份比。DECISIONS 的「整组同口径」 # 约束的是喂给一次比较的数据行,恰好由此保持不变。 # 某口径数据不齐 → 该口径不产出聚合,标了它的规则整条 # 跳过并显式记日志(绝不退回另一口径 —— 那是把 #2079 # 的假触发翻成永久假阴性)。 # # 齐全度按**各自窗口**判(2026-07-29 复审 F5):日规则 # 只比今天这一行,月规则比 MTD 整段。此前两者共用 MTD # 闸,3 号一天 list 为 NULL(回填缺口 / 驱动抖动)就把 # 整个 list 口径抹掉,该 scope 上所有 list 日阈值此后 # ~28 天一律哑火 —— 而闸本身刚确认过今天这行有 list。 _active_threshold_rules = [ r for r in (scope.alert_rules or []) if isinstance(r, dict) and r.get("type") in ("daily", "monthly") and r.get("active", True) ] list_complete_daily = ( today_row.total_cost_usd_list is not None ) list_complete_mtd = ( list_complete_daily and all( r.total_cost_usd_list is not None for r in mtd_rows ) ) needed_measures = { rule_measure(r) for r in _active_threshold_rules } if MEASURE_LIST in needed_measures: if not list_complete_daily: # Today's own row has no list value → NEITHER # window is available in that basis. needed_measures.discard(MEASURE_LIST) logger.event( "CLOUD_BILLING_DAILY_NO_LIST_VALUE", {"scope_id": scope.id, "date": end_dt}, level=logging.WARNING, ) elif not list_complete_mtd: logger.event( "CLOUD_BILLING_MTD_NOT_LIST_COMPLETE", { "scope_id": scope.id, "start_date": first_of_month, "end_date": end_dt, }, level=logging.WARNING, ) needs_monthly = any( r.get("type") == "monthly" for r in _active_threshold_rules ) def _row_value(r: Any, basis: str) -> float: if basis == MEASURE_LIST: return float(r.total_cost_usd_list) return float(r.total_cost_usd or 0) # Previous calendar month rows — read once, reused # per basis for the monthly card's "↑X% vs last # month" trend chip. Cheap: 0-1 month of daily rows # already in this session's view of the table. prev_rows: list[Any] = [] if needs_monthly and needed_measures: prev_first = ( first_of_month - timedelta(days=1) ).replace(day=1) prev_last = first_of_month - timedelta(days=1) prev_rows = list((await session.execute( select(CmdbCloudBillingDaily).where( CmdbCloudBillingDaily.scope_id == str(scope.id), CmdbCloudBillingDaily.date >= prev_first, CmdbCloudBillingDaily.date <= prev_last, ) )).scalars().all()) measure_aggregates: dict[str, MeasureAggregate] = {} for basis in sorted(needed_measures): # Only the MTD window can be incomplete here — # a basis whose TODAY row is missing never enters # ``needed_measures`` at all. ``None`` unarms the # monthly rules on this basis and nothing else. mtd_complete = ( basis != MEASURE_LIST or list_complete_mtd ) mtd_total_usd: float | None = None if mtd_complete: mtd_total_usd = 0.0 for r in mtd_rows: mtd_total_usd += await _to_usd( _row_value(r, basis), (r.currency or "USD"), ) today_usd_for_rules = await _to_usd( _row_value(today_row, basis), row_currency_today, ) # Trend chip must not mix measures either: a # list-basis MTD against a net prev-month renders # a phantom "↑76% vs last month". Drop the chip # when the prev month isn't complete in THIS # basis rather than blending. prev_total: float | None = None if prev_rows and mtd_complete and ( basis != MEASURE_LIST or all( r.total_cost_usd_list is not None for r in prev_rows ) ): t = 0.0 for r in prev_rows: t += await _to_usd( _row_value(r, basis), (r.currency or "USD"), ) prev_total = t measure_aggregates[basis] = MeasureAggregate( today_cost=today_usd_for_rules, month_to_date_cost=mtd_total_usd, prev_month_total_usd=prev_total, ) except RuntimeError as fx_exc: # FX missing/stale during the MTD/prev-month # conversion. Record + skip this scope's rule # evaluation; commit the already-upserted daily # rows first (same posture as the spike path's # ``fx_failed_for_this_scope`` branch) so the FX # gap doesn't carry into the next scope's txn or # abort the integration's whole run. errors.append( f"{scope.external_id}: rules-fx-conversion-" f"failed: {fx_exc}" ) logger.warning( "cloud.billing_daily: skipping user-defined " "alert rules for scope %s due to FX rate " "failure: %s", scope.id, fx_exc, ) await session.commit() continue if measure_aggregates: # ``measure`` is the primary basis + its aggregate; # the rest ride along in ``extra_measures``. Net goes # primary when present so the (overwhelmingly common) # all-net scope keeps the exact call shape it had. primary = ( MEASURE_NET if MEASURE_NET in measure_aggregates else next(iter(sorted(measure_aggregates))) ) primary_agg = measure_aggregates[primary] fired = await evaluate_scope_alert_rules( session, scope, today_cost=primary_agg.today_cost, today_date=end_dt, month_to_date_cost=primary_agg.month_to_date_cost, integration=integration, dispatch=dispatch_alerts, prev_month_total_usd=( primary_agg.prev_month_total_usd ), measure=primary, extra_measures={ k: v for k, v in measure_aggregates.items() if k != primary }, ) spikes_alerted += fired # Per-scope commit. Pre-fix the entire multi-scope loop # ran in ONE transaction committed only after the last # scope + the reconciliation pass below. On large # integrations that transaction stayed open across # minutes of slow external billing-API calls (prod runs # 122s–894s), holding locks on notification_outbox long # enough that a concurrent alert writer tripped a (1205) # lock-wait-timeout on the alert-card INSERT — which a # swallowed dispatch handler then masked as an opaque # PendingRollbackError that aborted the whole run and # discarded every scope's already-ingested rows (prod # 2026-05-26). Committing per scope keeps each # transaction short so those locks release promptly and # isolates a single scope's progress. Mirrors # cost_facts' per-batch commit (cost_service. # ingest_cost_facts). Safe mid-loop: async_session_factory # sets expire_on_commit=False, so the remaining ``scope`` # rows keep their loaded attributes after the commit. await session.commit() # ── Unregistered-account reconciliation pass ────────────── # Call the driver once with scope=None to get ALL linked accounts # visible to the credentials. Any account whose external_id is # NOT covered by a per-scope row for that DATE is spend the # per-scope loop dropped — either no ``cmdb_cloud_scope`` row # exists for it yet, or its scope is suspended / closed and so # is not visited by the ACTIVE-only loop above (#2221). # # We store these as scope_id=NULL rows so the billing overview # total matches the AWS Cost Explorer org-level total. Idempotency # uses DELETE + INSERT (not UPSERT) because MySQL/TiDB treats # NULL != NULL in unique indexes, making ON DUPLICATE KEY UPDATE # unreliable when scope_id is NULL. The billing cron runs once # per day and the reconciliation window is fixed to [start_dt, # end_dt], so the DELETE is always bounded and safe. covered_pairs: set[tuple[date, str]] = { (d, acct) for d, acct in (await session.execute( select( CmdbCloudBillingDaily.date, CmdbCloudBillingDaily.linked_account_id, ).where( CmdbCloudBillingDaily.integration_id == str(integration.id), CmdbCloudBillingDaily.scope_id.is_not(None), CmdbCloudBillingDaily.date >= start_dt, CmdbCloudBillingDaily.date <= end_dt, ).distinct() )).all() if acct } try: all_rows = await driver.list_billing_daily( start_dt, end_dt, scope=None, bq_billing=bq_billing, ) except NotImplementedError: all_rows = [] except Exception as e: # noqa: BLE001 errors.append( f"reconciliation_pass: {format_exception_for_log(e)}" ) all_rows = [] # Group unregistered costs by (date, account_id). Preserve # the driver's currency label: CN clouds (Aliyun / Tencent / # UCloud) write the NATIVE CNY amount into the legacy # ``total_cost_usd`` column with ``currency='CNY'`` and rely # on downstream ``_to_usd`` to convert. Hard-coding # ``currency='USD'`` on the reconciliation (scope_id=NULL) # row would mislabel CNY as USD and inflate the admin # billing-overview total ~7×. Mirror the per-scope path: # store the native amount + its true currency. unreg: dict[tuple[date, str], float] = {} unreg_currency: dict[tuple[date, str], str] = {} # 双口径:未注册账号行同样带上 list / public(None=未接入, # 任一行有值即 seen —— 与 per-scope 路径同一 None≠0 语义)。 unreg_list: dict[tuple[date, str], float] = {} unreg_list_seen: set[tuple[date, str]] = set() unreg_public: dict[tuple[date, str], float] = {} unreg_public_seen: set[tuple[date, str]] = set() for row in all_rows: acct = (row.scope.external_id if row.scope else "") or "" if acct and (row.date, acct) not in covered_pairs: key = (row.date, acct) unreg[key] = unreg.get(key, 0.0) + float( row.total_cost_usd or 0, ) if row.total_cost_usd_list is not None: unreg_list[key] = unreg_list.get(key, 0.0) + float( row.total_cost_usd_list, ) unreg_list_seen.add(key) if row.total_cost_usd_public is not None: unreg_public[key] = unreg_public.get(key, 0.0) + float( row.total_cost_usd_public, ) unreg_public_seen.add(key) # All rows from one driver/integration share a # currency; last-writer-wins is fine. Default USD # when the driver omits the field. unreg_currency[key] = (row.currency or "USD") # Always run the reconciliation DELETE — even when this # tick's ``unreg`` is empty. Pre-fix: ``if unreg:`` skipped # the DELETE on the day every previously-unregistered # account became registered, leaving stale ``scope_id=NULL`` # rows from prior runs to double-count in tenant-level # billing totals (admin overview ``billing overview total`` # would inflate by the prior day's unregistered figure # forever). The DELETE bounds are tight (one integration × # the same date window we just fetched) so running it on an # empty unreg is a cheap no-op. try: async with session.begin_nested(): await session.execute( delete(CmdbCloudBillingDaily).where( CmdbCloudBillingDaily.integration_id == str(integration.id), CmdbCloudBillingDaily.scope_id.is_(None), CmdbCloudBillingDaily.date >= start_dt, CmdbCloudBillingDaily.date <= end_dt, ) ) for (d, acct_id), total in unreg.items(): _ukey = (d, acct_id) await session.execute( mysql_insert(CmdbCloudBillingDaily).values( id=str(uuid.uuid4()), integration_id=str(integration.id), scope_id=None, linked_account_id=acct_id, date=d, total_cost_usd=total, total_cost_usd_list=( unreg_list.get(_ukey) if _ukey in unreg_list_seen else None ), total_cost_usd_public=( unreg_public.get(_ukey) if _ukey in unreg_public_seen else None ), cost_by_service={}, currency=unreg_currency.get( (d, acct_id), "USD", ), raw_response={}, owner_breakdown=None, fetched_at=datetime.now(UTC), ) ) upserted += 1 except Exception as e: # noqa: BLE001 # A transaction-aborting error (deadlock 1213 / lock-wait # 1205) rolls back the WHOLE transaction on TiDB and # destroys the savepoint — swallowing it here would only # defer the failure to the ``session.commit()`` two lines # below, which then raises on the dead transaction and # bypasses the graceful ``partial_failure`` envelope. Worse, # it can't be recovered in-place. Re-raise so the cron's # fresh-session retry handles it (per-scope progress is # already committed above; the reconciliation DELETE+insert # is idempotent and re-runs cleanly next attempt). Mirrors # the same-class fix in the permission / domain_registration # / ai_tool batch endpoints (#1066) and HRIS advance (#1055). if is_transaction_aborting_db_error(e): raise # Non-aborting reconciliation error: the savepoint cleanly # rolled back and the outer txn is alive — log + record into # the per-job errors list so the operator sees it in # scheduled_job_runs.detail without losing the # already-committed per-scope progress. errors.append( f"reconciliation_savepoint: {format_exception_for_log(e)}" ) logger.exception( "billing_daily reconciliation savepoint failed " "integration=%s window=%s..%s", integration.id, start_dt, end_dt, ) finally: try: await driver.close() except Exception: # noqa: BLE001 pass await session.commit() # Match ``sync_cost_facts_for_integration``'s envelope so the # admin scheduled-job execution-history view ranks # billing_daily runs the same way (ok / partial_failure / # failed) instead of always showing the cron handler's outer # status='ok' regardless of whether any per-scope path errored. status = ( "ok" if not errors else "partial_failure" if upserted > 0 else "failed" ) return { "status": status, "integration_id": integration_id, "start_date": start_dt.isoformat(), "end_date": end_dt.isoformat(), "upserted": upserted, "spikes_alerted": spikes_alerted, "errors": errors, } async def _alert_spike( session: AsyncSession, integration: Integration, scope: CmdbCloudScope, *, today_cost: float, avg: float, multiplier: float, measure: str, currency: str = "USD", ) -> None: """Post a Feishu alert to owner + IT admins when the daily spike threshold is crossed. Idempotent-per-day via ``notification_outbox.(recipient_id, related_resource_id)`` — the outbox dedupe keeps a duplicate alert for the same grant/day from firing twice. ``currency`` labels the rendered amount with the correct symbol so CN-cloud (Aliyun / Tencent / UCloud / KSyun) scopes don't render CNY values with a misleading ``$`` prefix. Round-20 audit (A1F2) landed this — a 7× over-stated daily figure trained admins to distrust the alert. ``measure`` (``"list"`` / ``"net"``) names the cost basis the caller compared on, so the card can tell the reader WHICH number to reconcile against. 2026-07-27 incident follow-up: a recipient cross-checked the card against the GCP console (contract / discounted cost), saw a flat curve, and reasonably concluded the alert was reading wrong data — the card never said it renders the pre-discount list basis. REQUIRED (2026-07-28 review): the previous ``= "list"`` default meant a future caller that forgot to pass it would stamp a net comparison with the list 口径说明 — the exact mislabel the note exists to prevent. ``measure == "net"`` additionally makes the card PER-RECIPIENT (2026-07-29 confidentiality sweep, notification surface): the absolute figures are invoice-basis, so only circle recipients (:func:`_invoice_measure_recipient_ids`) see them. Everyone else gets the same alert carrying only the DIMENSIONLESS ratio — today vs the 7-day baseline, both sides on the same basis, so the discount cancels and no invoice number is recoverable. The ratio IS the spike alert's signal, so the redacted card stays fully actionable; suppressing it for non-circle recipients instead would leave a non-prod scope's owner — usually the only person who can act — with no spike alert at all. """ service = NotificationService(session) symbol = "¥" if (currency or "USD").upper() == "CNY" else "$" cur_label = (currency or "USD").upper() # Multiplier divide-by-near-zero rendering: previous shape used # ``today_cost / max(avg, 0.01)`` which produced "500000×" cards on # brand-new scopes with no past usage. Render a human-readable # baseline-too-low marker when avg is essentially zero. mult_render: LocalizedText mult_render_redacted: LocalizedText if avg < 1e-3: mult_render = LocalizedText( zh=( f"基线 ≈ 0（{symbol}{today_cost:,.2f} {cur_label} " f"vs 近 7 日 ≈0）" ), en=( f"baseline ≈ 0 ({symbol}{today_cost:,.2f} {cur_label} " f"vs ≈0 over past 7d)" ), ja=( f"ベースライン ≈ 0（{symbol}{today_cost:,.2f} {cur_label} " f"vs 過去 7 日間 ≈0）" ), ) # Redacted twin: the same "no baseline to compare against" fact # without the absolute figure the normal render embeds. mult_render_redacted = LocalizedText( zh="基线 ≈ 0（近 7 日几乎无花费）", en="baseline ≈ 0 (almost no spend over the past 7d)", ja="ベースライン ≈ 0（過去 7 日間はほぼコストなし）", ) else: mult_render = LocalizedText( zh=f"{today_cost / avg:.2f}× (阈值 {multiplier}×)", en=f"{today_cost / avg:.2f}× (threshold {multiplier}×)", ja=f"{today_cost / avg:.2f}×（しきい値 {multiplier}×）", ) mult_render_redacted = mult_render msg_body = ( f"`{scope.display_name or scope.external_id}` 昨日花费明显高于" f"过去 7 日平均，请尽快确认是否合理。" ) # Same recipient policy as the per-rule billing alerts: only the # scope owner + billing-class grant holders. The previous fan-out # to every ``global_admin`` / ``it_admin`` user spammed the whole # IT pool on every spike; admins who actually want it should hold # a billing-class grant on the scope so the routing is auditable. recipients = await _resolve_billing_alert_recipients(session, scope) if not recipients: # The spike slot is already claimed by the caller (the daily- # sync block) so this won't refire per period — see the matching # comment in ``evaluate_scope_alert_rules``. Log loudly so the # silent-drop path is visible in the log aggregator. logger.warning( "cloud.spike_alert empty recipient set: scope_id=%s " "external_id=%s integration=%s today_cost=%.2f avg=%.2f " "currency=%s — owner/billing-grant unassigned, spike alert " "silently dropped.", scope.id, scope.external_id, integration.name, today_cost, avg, (currency or "USD").upper(), ) return from app.notifications.service import CardSection, CardStat trend_pct: str | None = None if avg > 0: diff = (today_cost - avg) / avg * 100 trend_pct = f"▲ {diff:.0f}%" spike_account_hint = LocalizedText( zh=f"云账号: {integration.name}", en=f"Account: {integration.name}", ja=f"クラウドアカウント: {integration.name}", ) card_stats = [ CardStat( label=LocalizedText( zh="昨日花费", en="Yesterday's spend", ja="昨日のコスト", ), value=f"{symbol}{today_cost:,.2f} {cur_label}", trend=trend_pct, trend_is_good=False, hint=spike_account_hint, ), CardStat( label=LocalizedText( zh="近 7 日均值", en="7-day average", ja="直近 7 日間の平均", ), value=f"{symbol}{avg:,.2f} {cur_label}", ), CardStat( label=LocalizedText(zh="环境", en="Environment", ja="環境"), value=scope.environment or "—", ), CardStat( label=LocalizedText( zh="触发倍数", en="Trigger multiple", ja="発動倍率", ), value=mult_render, trend_is_good=False, ), ] # Amount-free twin for non-circle recipients of a NET-basis spike: keep # the ratio + the trend chip (dimensionless), drop both absolutes. redacted_card_stats = [ CardStat( label=LocalizedText( zh="昨日 vs 近 7 日均值", en="Yesterday vs 7-day average", ja="昨日 vs 直近 7 日間の平均", ), value=mult_render_redacted, trend=trend_pct, trend_is_good=False, hint=spike_account_hint, ), CardStat( label=LocalizedText(zh="环境", en="Environment", ja="環境"), value=scope.environment or "—", ), ] measure_note = ( "口径说明：金额为业务折前（list 牌价）口径，通常高于云控制台 / " "发票展示的折后金额，请勿与控制台绝对值直接对比。" if measure == "list" else "口径说明：金额为发票净额（折后）口径。" ) measure_note_en = ( "Basis note: amounts are on the pre-discount list-price basis, " "usually HIGHER than the discounted figure shown in the cloud " "console / invoice — do not compare absolute values against the " "console directly." if measure == "list" else "Basis note: amounts are on the invoice-net (post-discount) basis." ) measure_note_ja = ( "基準について：金額は割引前(リスト価格)基準です。クラウドコンソール / " "請求書に表示される割引後の金額より高くなるのが通常ですので、コンソール" "の絶対値と直接比較しないでください。" if measure == "list" else "基準について：金額は請求ネット(割引後)基準です。" ) redacted_measure_note = ( "口径说明：本次比较按发票净额（实付）口径评估。实付金额仅财务 / " "云管可见，本卡因此只给出倍数、不展示绝对金额；请在「云账单」页按" "业务折前口径查看明细，或联系 IT 获取实付数字。" ) redacted_measure_note_en = ( "Basis note: this comparison was evaluated on the invoice-net " "basis. Invoice-net amounts are restricted to finance / cloud " "admins, so this card shows only the multiple and omits absolute " "figures — open the cloud billing page (pre-discount list basis) " "for details, or ask IT for the invoice figure." ) redacted_measure_note_ja = ( "基準について：今回の比較は請求ネット(実支払)基準で評価しました。" "実支払金額は経理 / クラウド管理者のみが参照できるため、このカードでは" "倍率のみを表示し、絶対金額は表示していません。詳細は「クラウド請求」" "ページで割引前(リスト価格)基準の内訳をご確認いただくか、IT まで" "お問い合わせください。" ) def _sections(redacted: bool) -> list[CardSection]: return [ CardSection( title=LocalizedText( zh="建议下一步", en="Recommended next steps", ja="推奨される次のステップ", ), body=LocalizedText( zh=( "1) 打开 [云账单详情] 查看昨日新增的服务 / 资源\n" "2) 如属正常迁移 / 灰度，可在 scope 备注里记录\n" "3) 如不预期，及时止血或联系 IT 排查\n\n" f"{redacted_measure_note if redacted else measure_note}\n" "如对帐单有任何疑问，请联系 IT 团队进行确认。" ), en=( "1) Open [cloud billing details] to review the services / " "resources added yesterday\n" "2) If this is a planned migration / canary, note it in the " "scope remarks\n" "3) If unexpected, act quickly to reduce spend or escalate " "to IT\n\n" f"{redacted_measure_note_en if redacted else measure_note_en}\n" "If you have any questions about the bill, please contact " "the IT team for confirmation." ), ja=( "1) [クラウド請求明細] を開き、昨日追加されたサービス / " "リソースを確認してください\n" "2) 想定内の移行 / カナリアであれば、スコープの備考に記録" "してください\n" "3) 想定外の場合は、速やかにコストを抑えるか IT に" "エスカレーションしてください\n\n" f"{redacted_measure_note_ja if redacted else measure_note_ja}\n" "請求内容についてご不明な点があれば、IT チームまで" "ご確認ください。" ), ), tone="warn", ), ] circle_ids = ( set() if measure == "list" else await _invoice_measure_recipient_ids(session, recipients) ) for r in recipients: redacted = measure != "list" and str(r.id) not in circle_ids try: await service.dispatch( NotificationEnvelope( recipient=r, title=f"☁️ 云账单日环比异常 — {scope.display_name or scope.external_id}", body_text=msg_body, channels=["feishu", "email"], url=f"/cloud/scopes/{scope.id}", card_subtitle=LocalizedText( zh=f"{integration.name} / {scope.environment or '—'} 环境", en=f"{integration.name} / {scope.environment or '—'}", ja=f"{integration.name} / {scope.environment or '—'} 環境", ), card_stats=( redacted_card_stats if redacted else card_stats ), card_sections=_sections(redacted), related_resource_type="cmdb_cloud_scope", related_resource_id=scope.id, ), trace_id="cloud.billing_daily_sync", ) except Exception: # noqa: BLE001 logger.exception("spike alert dispatch failed recipient=%s", r.id) # ── Inactive grant last-login pull (used by inactivity sweep) ─ async def refresh_grant_last_login( grant: CmdbCloudAccessGrant, *, session: AsyncSession, ) -> None: """Per-grant wrapper that delegates to the per (scope, person) refresh and applies the result to ALL grants in the same group. Kept as the public entry point so existing callers (and tests) that pass a single grant still work, but internally we coalesce by (scope_id, person_id) — see :func:`refresh_person_last_login_in_scope` for the rationale: cloud IAM tracks login at the user identity level (PasswordLastUsed / AccessKeyLastUsed on AWS, equivalents on other clouds), not per role / policy. Refreshing one grant's ``last_login_at`` while leaving the other grants of the same person on the same scope at NULL was the 2026-05-05 false-alert bug: ``IAMUserChangePassword`` got flagged ``60+ 天无登录`` while the user logged in 7 days ago, because the user-level activity got written to ONE grant only and the others stayed NULL. """ await refresh_person_last_login_in_scope( scope_id=str(grant.scope_id), person_id=str(grant.person_id), session=session, ) async def refresh_person_last_login_in_scope( *, scope_id: str, person_id: str, session: AsyncSession, ) -> None: """Pull cloud-side last activity for one (scope, person) pair and apply the timestamp to every active grant in that group. Why this is the right unit: * Cloud IAM tracks login per IDENTITY, not per ROLE / POLICY. AWS ``PasswordLastUsed`` is per IAM user; ``AccessKeyLastUsed`` is per access key (which belongs to the IAM user, not the policy). Aliyun RAM is the same shape. There is no "last time this user exercised role X" data on any provider we ship. * Therefore the meaningful inactivity unit is (cloud scope, IAM user / person), not (scope, person, role). One driver call covers every grant the person holds on that scope. * Driver call cost: refreshing N grants for one (scope, person) one-by-one issues N IAM lookups against the cloud. Coalescing to one driver call also keeps the cron job under its 10 min budget at scale. * Determinism: all grants in the group end up with the same ``last_login_at``, so the downstream sweep can group-evaluate without a per-grant time-skew race. Errors are swallowed: ``last_login_at`` keeps its prior value on any provider failure. Driver close is best-effort. """ scope = await session.get(CmdbCloudScope, uuid.UUID(scope_id)) if scope is None: return person = await session.get(IdentityPerson, uuid.UUID(person_id)) if person is None or not person.work_email: return grants_in_group = list((await session.execute( select(CmdbCloudAccessGrant).where( CmdbCloudAccessGrant.scope_id == scope_id, CmdbCloudAccessGrant.person_id == person_id, CmdbCloudAccessGrant.status == GRANT_STATUS_ACTIVE, ) )).scalars().all()) if not grants_in_group: return integration, driver = await _load_driver_for_scope(session, scope) # Pick a representative grant for ``role_key`` / ``external_ref`` # context. The driver implementations only use these as fallback # hints for principal lookup — for AWS / Aliyun IAM they're # ignored (the IAM user is identified by email or ARN). Pick the # most-recently-renewed row so any driver that does consult the # external_ref hits the freshest one. representative = max( grants_in_group, key=lambda g: ( g.last_renewed_at or g.granted_at or g.created_at or datetime.fromtimestamp(0, tz=UTC) ), ) try: last = await driver.get_last_login(CloudAccessContext( principal_email=person.work_email, principal_display_name=person.display_name, scope=CloudScopeRef( scope_type=scope.scope_type, external_id=scope.external_id, display_name=scope.display_name, ), role_key=representative.role_key, existing_external_ref=representative.external_ref, )) if last is not None: for g in grants_in_group: g.last_login_at = last except Exception as e: # noqa: BLE001 logger.info( "last_login refresh skipped scope=%s person=%s: %s", scope_id, person_id, e, ) finally: try: await driver.close() except Exception: # noqa: BLE001 pass # ── Scope idle-candidate sweep (cmdb.scope_idle_sweep) ─────────────── @dataclass class IdleScopeCandidate: """One row of the weekly idle-scope report. Pure read-only — surfaced to the admin as a notification + via the ``GET /cmdb/scopes/idle-candidates`` endpoint. Decommissioning is always a manual decision. """ scope_id: str integration_id: str integration_name: str | None provider: str | None scope_type: str external_id: str display_name: str | None owner_person_id: str | None last_synced_at: datetime days_since_any_grant_activity: int | None cost_30d_usd: float # Suppression state — populated even when ``include_suppressed=False`` # (in which case all returned rows have ``is_snoozed=False`` and # ``recently_notified=False`` by construction, but callers still get # the raw timestamps for display / debug logs). snoozed_until: datetime | None = None last_notified_at: datetime | None = None is_snoozed: bool = False recently_notified: bool = False async def list_idle_scope_candidates( session: AsyncSession, *, no_grants_for_days: int = 30, no_cost_for_days: int = 30, dedup_days: int = IDLE_SWEEP_DEDUP_DAYS, include_suppressed: bool = False, ) -> list[IdleScopeCandidate]: """Find ``status='active'`` scopes that look abandoned. Rule (deliberately conservative — both conditions must hold): * 0 active grants right now AND * $0 spend in the trailing ``no_cost_for_days`` days Plus the scope must have been first seen at least ``no_grants_for_days`` ago (don't flag scopes the integration synced in this morning). Cost data may be missing entirely for clouds whose billing sync hasn't run yet — those scopes are NOT flagged (we can't tell idle from "no data"). Notification suppression (``include_suppressed=False``, the default for the weekly sweep): scopes are skipped when EITHER * ``idle_snoozed_until > now`` (owner explicitly snoozed), OR * ``last_idle_notified_at >= now - dedup_days`` (auto-backoff after a previous sweep already sent a notification). Callers that want to surface ALL idle scopes for an admin overview (e.g. the planned ``GET /cmdb/scopes/idle-candidates`` admin listing) pass ``include_suppressed=True``. """ now = datetime.now(UTC) cost_cutoff = (now - timedelta(days=no_cost_for_days)).date() age_cutoff = now - timedelta(days=no_grants_for_days) dedup_cutoff = now - timedelta(days=dedup_days) # Active scopes whose first_seen is old enough to consider. # # The snooze + dedup filters used to live in the Python loop below. # That left a race window (Round-3 logic audit): sweep SELECTed # scope X as "not snoozed", owner committed a snooze on X with # ``FOR UPDATE`` while the sweep was still building candidates, # the sweep proceeded to dispatch a notification about a freshly- # snoozed scope. Moving the filter into the SQL WHERE collapses # the window to "between this SELECT and the subsequent dispatch # commit" — the snooze writer holds the row lock for that span so # MySQL/TiDB serialises any concurrent reader-then-stamper against # the snooze commit at the storage layer. # # ``include_suppressed=True`` callers (admin overview, future # debug surfaces) drop the WHERE clauses by branching, so they # still see the full active-scope set including snoozed / recently- # notified ones — the dataclass exposes ``is_snoozed`` / # ``recently_notified`` so UIs can pill-render the state. scope_filter = [ CmdbCloudScope.status == SCOPE_STATUS_ACTIVE, CmdbCloudScope.deleted_at.is_(None), CmdbCloudScope.first_seen_at < age_cutoff, ] if not include_suppressed: # NULL-safe: a scope never snoozed has ``idle_snoozed_until IS # NULL``; ``NULL > now`` evaluates to NULL → filtered by the # OR-with-IS-NULL. Same shape for ``last_idle_notified_at``. scope_filter.extend([ # A scope whose decommission is already confirmed and # mid-cleanup (migration 624) is ``status='active'`` on # purpose — but "这个 scope 好像没人用了,考虑下线?" is the # one thing its owner must not be asked while Helix is # actively deleting its contents. CmdbCloudScope.decommission_pending_since.is_(None), or_( CmdbCloudScope.idle_snoozed_until.is_(None), CmdbCloudScope.idle_snoozed_until <= now, ), or_( CmdbCloudScope.last_idle_notified_at.is_(None), CmdbCloudScope.last_idle_notified_at < dedup_cutoff, ), ]) scope_rows = (await session.execute( select(CmdbCloudScope).where(*scope_filter) )).scalars().all() if not scope_rows: return [] scope_ids = [str(s.id) for s in scope_rows] # Active+pending grant counts per scope. active_counts: dict[str, int] = {} for scope_id, n in (await session.execute( select( CmdbCloudAccessGrant.scope_id, func.count(CmdbCloudAccessGrant.id), ).where( CmdbCloudAccessGrant.scope_id.in_(scope_ids), CmdbCloudAccessGrant.status.in_( (GRANT_STATUS_ACTIVE, GRANT_STATUS_PENDING), ), ).group_by(CmdbCloudAccessGrant.scope_id) )).all(): active_counts[str(scope_id)] = int(n) # Sum of trailing-30d cost per scope. Scopes with NO billing rows at # all (provider doesn't sync billing yet, or fresh integration) stay # out of the candidates list — we can't claim "no cost" without data. # # NOTE: ``total_cost_usd`` is misnamed — CN-billed tenants store the # native CNY amount in it (see CloudCostUsdConverter). A bare SUM here # therefore mixes currencies, BUT this sum is decision-only: it feeds # the zero-vs-nonzero idle test below, never a displayed USD figure. # CNY inflation can only make a nonzero amount stay nonzero — it can # never turn a zero into a nonzero or vice-versa — so the idle # decision is currency-mix-invariant and no FX conversion is needed # (adding one would cost an extra lookup per integration for no # behaviour change). Do not "fix" this to a converter. cost_rows = (await session.execute( select( CmdbCloudBillingDaily.scope_id, func.sum(CmdbCloudBillingDaily.total_cost_usd), func.count(CmdbCloudBillingDaily.id), ).where( CmdbCloudBillingDaily.scope_id.in_(scope_ids), CmdbCloudBillingDaily.date >= cost_cutoff, ).group_by(CmdbCloudBillingDaily.scope_id) )).all() cost_30d: dict[str, tuple[float, int]] = { str(sid): (float(total or 0), int(n)) for sid, total, n in cost_rows } # Most-recent grant activity (for the report column — informational # only, not a filter). last_grant_rows = (await session.execute( select( CmdbCloudAccessGrant.scope_id, func.max(CmdbCloudAccessGrant.updated_at), ).where( CmdbCloudAccessGrant.scope_id.in_(scope_ids), ).group_by(CmdbCloudAccessGrant.scope_id) )).all() last_grant_at: dict[str, datetime] = { str(sid): ts for sid, ts in last_grant_rows if ts is not None } integ_rows = (await session.execute( select(Integration).where(Integration.id.in_( [uuid.UUID(str(s.integration_id)) for s in scope_rows], )) )).scalars().all() integ_by_id = {str(i.id): i for i in integ_rows} def _aware(value: datetime | None) -> datetime | None: # MySQL DATETIME loses tzinfo on fresh load; normalise here so # the comparison against ``now`` (always UTC-aware) is honest. # See memory ``feedback_mysql_datetime_naive_on_fresh_load``. if value is None: return None return value if value.tzinfo else value.replace(tzinfo=UTC) out: list[IdleScopeCandidate] = [] for s in scope_rows: sid = str(s.id) if active_counts.get(sid, 0) > 0: continue cost_total, cost_row_count = cost_30d.get(sid, (0.0, 0)) # Hard skip: no billing rows in the last 30d means we can't # judge — abstain. Once billing_daily catches up the candidate # will surface in a later sweep. if cost_row_count == 0: continue # ``> 0`` skips a spending scope. A NEGATIVE total is a # credit-only month (refunds / RI cancellations netting below # zero) — real billing activity, not "$0 spend" (#2189-7). It # still qualifies as idle by the rule, but the report must not # call it $0 when it isn't; ``cost_30d_usd`` carries the real # signed total and the caller renders that. if cost_total > 0: continue snoozed_until = _aware(s.idle_snoozed_until) last_notified_at = _aware(s.last_idle_notified_at) is_snoozed = snoozed_until is not None and snoozed_until > now recently_notified = ( last_notified_at is not None and last_notified_at >= dedup_cutoff ) # The weekly sweep call site passes ``include_suppressed=False`` # so snoozed / recently-notified scopes are dropped here rather # than at the dispatch loop — keeps the candidate count in the # job summary honest about "what is actually being notified # this run". Admin overview / debug callers can opt back in. if not include_suppressed and (is_snoozed or recently_notified): continue last_act = last_grant_at.get(sid) days_idle: int | None if last_act is not None: naive_last = last_act.replace(tzinfo=None) if last_act.tzinfo else last_act naive_now = now.replace(tzinfo=None) days_idle = max(0, (naive_now - naive_last).days) else: days_idle = None integ = integ_by_id.get(str(s.integration_id)) out.append(IdleScopeCandidate( scope_id=sid, integration_id=str(s.integration_id), integration_name=integ.name if integ else None, provider=integ.provider if integ else None, scope_type=s.scope_type, external_id=s.external_id, display_name=s.display_name, owner_person_id=s.owner_person_id, last_synced_at=s.last_synced_at, days_since_any_grant_activity=days_idle, cost_30d_usd=cost_total, snoozed_until=snoozed_until, last_notified_at=last_notified_at, is_snoozed=is_snoozed, recently_notified=recently_notified, )) out.sort( key=lambda c: ( -(c.days_since_any_grant_activity or 0), c.display_name or c.external_id, ), ) return out async def sweep_idle_scopes(session: AsyncSession) -> dict[str, Any]: """Weekly job — notify IT admins AND each scope owner about idle scopes. Recipient model (changed 2026-05-25): the digest now goes to the scope OWNER per candidate (they're the only person who can decide "keep / transfer / decommission") AND to all IT admins (visibility + fallback when no owner is set). A single person who happens to be both IT admin and owner of one of the candidates gets ONE Feishu card, not two — dedup is by person_id. Dedup (also new 2026-05-25): scopes still inside the ``IDLE_SWEEP_DEDUP_DAYS`` window or under explicit owner snooze (``idle_snoozed_until > now``) are excluded by ``list_idle_scope_candidates`` itself, so this function only sees "actually-needs-a-notification" rows. After a successful dispatch the scope's ``last_idle_notified_at`` is stamped so the next Monday's sweep skips it. Returns the run summary used by ``scheduled_job_runs.result_summary``. Still no auto-decommission: this only writes notifications + state. Owners / admins still have to click 下线 (or 忽略 30 天) on each candidate they want resolved. """ candidates = await list_idle_scope_candidates(session) summary: dict[str, Any] = { "candidates": len(candidates), "notified_recipients": 0, "owners_notified": 0, "admins_notified": 0, "stamped_scopes": 0, "timeline_rows": 0, "errors": [], } if not candidates: return summary # IT admin recipient set — same membership as before. admin_ids = list((await session.execute( select(AuthUserRole.user_id) .join(AuthRole, AuthRole.id == AuthUserRole.role_id) .where(AuthRole.name.in_(("global_admin", "it_admin"))) )).scalars().all()) # Scope-owner recipient set — only the owners OF THIS DIGEST'S # candidates (not all scope owners platform-wide). owner_ids: set[str] = { c.owner_person_id for c in candidates if c.owner_person_id } # Union, then dedup by person_id so a single person who is both # admin and one of the candidate-owners receives one card. recipient_ids: set[str] = {str(a) for a in admin_ids} | owner_ids if not recipient_ids: return summary recipients: list[IdentityPerson] = [] admin_id_str = {str(a) for a in admin_ids} for rid in recipient_ids: # AuthUser.id and IdentityPerson.id share the same UUID by # construction; owner_person_id is already a person id. p = await session.get(IdentityPerson, rid) if (p and p.work_email and p.deleted_at is None and p.employment_status == "active"): recipients.append(p) if not recipients: return summary title = f"[云治理] {len(candidates)} 个闲置子账号建议确认处理" def _format_body(*, for_owner_id: str | None) -> str: # When sent to a specific owner, only their candidates are # listed (avoids leaking other teams' scope names to every # owner). Admins see the full list. visible = ( [c for c in candidates if c.owner_person_id == for_owner_id] if for_owner_id else candidates ) lines: list[str] = [] if for_owner_id is not None: lines.append( f"以下 {len(visible)} 个子账号过去 30 天没有任何使用 " "—— 0 个生效 grant + $0 花费,请确认下一步处理:" ) else: lines.append( f"本周扫到 {len(candidates)} 个长期闲置子账号 " "—— 0 个生效 grant + 过去 30 天 $0 花费,请确认下一步处理:" ) lines.append("") for c in visible[:20]: days = ( f"{c.days_since_any_grant_activity} 天前" if c.days_since_any_grant_activity is not None else "无 grant 活动记录" ) prov = c.provider or "?" lines.append( f" · [{prov}] {c.display_name or c.external_id}" ) lines.append( f" 上次 grant 活动:{days};过去 30 天花费:$0" ) if len(visible) > 20: lines.append(f" ... 另有 {len(visible) - 20} 个未列出") lines.append("") lines.append("处理建议(任选其一):") lines.append(" · 转让 Owner ——账号已交接给别人") lines.append(" · 下线 ——确实不再使用") lines.append(" · 在详情页点「忽略 30 天」——短期保留,暂时无需提醒") lines.append("") lines.append("数据与提醒说明:") lines.append( " · 花费数据来自云服务商账单导出,通常滞后 1-2 天," "非平台同步异常" ) lines.append(" · 仅供决策参考,平台不会自动下线任何账号") lines.append( " · 同一账号 14 天内不会重复打扰;owner 可主动「忽略 30 天」" ) return "\n".join(lines) # Per-scope dispatch tracker — a scope's dedup clock starts only # when at least ONE recipient who would actually SEE that scope in # their card was successfully reached. Crucially: admin recipients # see every candidate (full digest), but owner recipients only see # their own scopes (filtered body). If admin dispatch fails and # only an unrelated owner is reached, scopes BELONGING to other # owners (whose owners weren't reached either) MUST NOT be stamped # — otherwise their dedup window starts despite no one being # notified about them, and the actual owner never hears about it. # Pre-fix this was a global "any recipient succeeded → stamp all" # check; the BLK-1/BLK-2 audit finding flagged it. seen_scope_ids: set[str] = set() # Pre-compute the per-recipient "scopes I would see" set once so we # don't recompute it on every dispatch success. candidate_owner_index: dict[str, list[str]] = {} for c in candidates: candidate_owner_index.setdefault( c.owner_person_id or "__unowned__", [], ).append(c.scope_id) all_scope_ids: list[str] = [c.scope_id for c in candidates] # Per-scope candidate lookup so the timeline row we write below can # snapshot "last grant activity N days ago" alongside the $0/30d fact. candidate_by_scope_id: dict[str, IdleScopeCandidate] = { c.scope_id: c for c in candidates } notify = NotificationService(session) notified_person_ids: set[str] = set() for person in recipients: try: # Owners only see THEIR candidates so cross-team scope names # aren't leaked. Admins see the full digest. person_id_str = str(person.id) is_admin = person_id_str in admin_id_str is_owner_only = ( person_id_str in owner_ids and not is_admin ) body = _format_body( for_owner_id=person_id_str if is_owner_only else None, ) await notify.dispatch( NotificationEnvelope( recipient=person, title=title, body_text=body, # Owner lands on 我的云订阅 (the list of scopes they # hold, each row drilling into the detail page that # carries their snooze button); admin lands on the # admin list. NB ``/my-cloud-scopes`` was emitted here # until 2026-07-28 and has never been a route — every # owner tapping this idle-sweep card got a 404. url=( "/my/cloud/grants" if is_owner_only else "/admin/cloud/scopes" ), channels=["feishu", "email"], related_resource_type="cmdb_cloud_scope", related_resource_id=None, ), trace_id="cloud.scope_idle_sweep", ) notified_person_ids.add(person_id_str) summary["notified_recipients"] += 1 if person_id_str in owner_ids: summary["owners_notified"] += 1 if is_admin: summary["admins_notified"] += 1 # Admin saw the full digest → every candidate is now # "on someone's radar". A single admin who is also one # of the candidate-owners still saw the full digest # (the body branch chose ``for_owner_id=None``); this # counts for every scope, not just their own. seen_scope_ids.update(all_scope_ids) elif is_owner_only: # Owner-only recipient — only the scopes they own # were listed in their card body. seen_scope_ids.update( candidate_owner_index.get(person_id_str, []), ) except Exception as e: # noqa: BLE001 summary["errors"].append( f"notify {person.work_email}: {e}" ) # Batch-load all candidate scope rows once (replaces per-scope # session.get() in the previous version, which could return cached # instances). Single SELECT, indexed by id-as-str so we can pair # against ``seen_scope_ids`` cheaply. if seen_scope_ids: now = datetime.now(UTC) scope_rows = (await session.execute( select(CmdbCloudScope).where( CmdbCloudScope.id.in_( [uuid.UUID(sid) for sid in seen_scope_ids], ), ) )).scalars().all() for scope in scope_rows: scope.last_idle_notified_at = now # Leave a per-scope governance-timeline row so the detail # page's 治理 tab shows "system flagged this scope idle on # date D" — previously the sweep only stamped # ``last_idle_notified_at`` (a single overwrite-on-each-run # timestamp) + sent ONE digest notification with # ``related_resource_id=None``, so a scope owner had no # per-scope record that they'd ever been pinged. Written as a # leaf ``responded`` event (a notice, not a prompt the owner # must answer); the sweep's own 14-day dedup keeps it to at # most one row per scope per ~2 weeks. cand = candidate_by_scope_id.get(str(scope.id)) days = ( cand.days_since_any_grant_activity if cand is not None else None ) reason = ( "0 个生效 grant + 过去 30 天 $0 花费" + ( f";上次 grant 活动 {days} 天前" if days is not None else ";无 grant 活动记录" ) ) session.add(CmdbCloudScopeOwnerConfirmation( id=uuid.uuid4(), scope_id=str(scope.id), owner_person_id=scope.owner_person_id, kind=OWNER_CONFIRMATION_KIND_IDLE_NOTIFIED, status=OWNER_CONFIRMATION_STATUS_RESPONDED, response_kind=OWNER_CONFIRMATION_KIND_IDLE_NOTIFIED, spend_recent_30d_usd=Decimal("0"), member_count_snapshot=0, # No ``notified_at`` — ``created_at`` already marks the # ping instant for a leaf notice; setting both makes the # detail-page "when" column render the same timestamp # twice (it shows ``notified_at`` only when it differs # from ``created_at``). reason=reason, )) summary["stamped_scopes"] += 1 summary["timeline_rows"] = len(scope_rows) return summary __all__ = [ "CloudLifecycleService", "GrantDispatchError", "IdleScopeCandidate", "LifecycleGrantOutcome", "ScopeUnownedError", "list_idle_scope_candidates", "refresh_grant_last_login", "refresh_person_last_login_in_scope", "sweep_active_grants_on_left_persons", "sweep_expiring_grants", "sweep_idle_scopes", "sweep_inactive_grants", "sync_billing_daily_for_integration", "sync_grants_for_integration", "sync_roles_for_integration", "sync_roles_for_scope", "sync_scopes_for_integration", ]