"""Identity Sync Center — orchestration service. This module is the single entry-point the rest of the platform uses to: * CRUD targets (admin UI / migrations) * Build a runner instance for a target (driver factory) * Run a target end-to-end: candidate selection → mapping → upsert → state reconciliation → run record (sweep mode) * Push a single person on-demand (lifecycle event hook, per-person admin button, MCP tool) * Compute the next cron fire time after each run The dispatcher handler ( ``identity_sync.dispatch`` registered in ``scheduler_handler.py``) is a thin wrapper around :meth:`IdentitySyncService.run_due_targets`. """ from __future__ import annotations import asyncio import json import logging import re import time import uuid import zlib from collections.abc import Iterable from datetime import UTC, datetime, timedelta from typing import Any, NamedTuple from uuid import UUID from sqlalchemy import and_, desc, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.audit.service import AuditService from app.core.database import async_session_factory from app.core.exceptions import AppException, ConflictError, NotFoundError from app.core.safe_logging import get_safe_logger from app.core.txn import best_effort from app.core.working_hours import ( get_provision_lead_hours, is_future_hire, ) from app.domains.identity_sync.destructive_confirmation import ( DestructivePlanService, department_mass_action_brake_required, user_mass_action_brake_required, ) from app.domains.identity_sync.filters import ( CandidatePerson, FilterConfig, FilterError, expand_department_subtree, fails_employee_no_gate, is_non_corporate_email, is_test_employee_no, matches, ) from app.domains.identity_sync.mapping import ( MappingConfig, MappingError, build_person_context, collect_render_diagnostics, hash_payload, render_payload, ) from app.domains.identity_sync.models import ( IdentitySyncDepartmentState, IdentitySyncRun, IdentitySyncRunItem, IdentitySyncState, IdentitySyncTarget, IdentitySyncVendorRecord, ) from app.domains.identity_sync.runners.base import ( DeptOutcome, PersonOutcome, RunSummary, SyncRunner, ) from app.identity.leaver_notice import leaver_last_working_day_passed from app.identity.person_models import ( IdentityOrgMembership, IdentityOrgNode, IdentityPerson, ) from app.integrations.models import Integration from app.integrations.service import _resolve as _resolve_integration from app.integrations.service import _scrub_message as _scrub_secrets logger = get_safe_logger(__name__) class HuilianyiDeptAttrs(NamedTuple): """Derived 汇联易 department attributes (all optional → field omitted). ``leader_no`` / ``hrbp_no`` / ``budget_owner_no`` are 工号 pushed as ``departmentPositionList`` roles (positionCode 6103 / 6105 / 6102); ``cost_center_label`` is pushed as ``customFormValues`` fieldCode CC. """ leader_no: str | None cost_center_label: str | None hrbp_no: str | None budget_owner_no: str | None # SF models 13 top-level BUs (Engineering Group / Business Operations / …) with # NO "PingCAP" department — "PingCAP" is only the ``cust_L0`` company label, so # Helix's ``IdentityOrgNode`` tree has those BUs as roots (parent=None). The # 汇联易 tenant, however, has a real top-of-tree dept ``custDeptNumber=RootOrg`` # named "PingCAP" (verified live 2026-06-16). To keep the vendor tree # single-rooted (and mirror the legacy Authing/北森 structure the 费控 config # hangs off of), the dept push re-parents each Helix-root BU under this vendor # root code instead of sending parent=None. Reference-only: Helix never # creates/updates the RootOrg node itself (it pre-exists; pushing it could # blank the vendor's existing config) — overridable per target via # ``runner_options.huilianyi_root_dept_code`` if a tenant's root differs. HUILIANYI_DEFAULT_ROOT_DEPT_CODE = "RootOrg" # SF ``EmpJob.customString24`` picklist code meaning "不推送到下游系统" # (do NOT provision this person to any downstream). HR-confirmed # 2026-07-07 (the field's SF picklist labels are legacy-garbage — only # the code carries meaning; 2059 = 推送, 2061 = 不推送). Stored RAW in # ``identity_persons.sf_downstream_push_code`` (migration 390) and gated # in ``_load_candidates`` so it applies to EVERY sync target. _SF_NO_PUSH_CODE = "2061" # Periodic drift re-verification window. Even when a linked person's / # dept's HRIS-rendered payload hash is unchanged (the cheap steady-state # skip), a row whose ``last_verified_at`` is older than this is NOT # hash-skipped: the runner is called so it re-asserts the Helix-managed # fields against LIVE vendor state, correcting anything edited on the # vendor side outside Helix (a manually-changed Feishu email / name / # dept) and retrying best-effort slots that were dropped on the last # push (leader / HRBP / mobile). This is the throttled analogue of the # predecessor Authing sync's "compare live vendor state every run": # re-assert at most every N days instead of every sweep, so steady-state # API load stays low while drift still self-heals within the window. _DRIFT_REVERIFY_INTERVAL = timedelta(days=7) # Per-row deterministic jitter added on top of the base interval. Without it, # a nightly-synced target's whole population crosses the 7-day threshold in the # SAME sweep and then all get a fresh ``last_verified_at`` in that sweep — so # they re-push as a recurring clump every 7 days (a periodic full-target vendor # spike on large targets). Spreading each row's effective window by a stable # hash of its key over this extra span smooths the re-verify load into a # rolling wave instead of a weekly spike. _DRIFT_REVERIFY_JITTER = timedelta(days=3) # ── Scheduled-run failure / cancellation re-arm policy ─────────────── # The cron cursor (``IdentitySyncTarget.next_run_at``) is re-armed in # ``_finalise_run`` based on how a SCHEDULED run ended: # # ok / partial → advance to the next cron slot; reset failure streak. # cancelled → an infra event (worker restart / deploy killed the # sweep). Re-arm to ``now + _SCHED_CANCELLED_RETRY_DELAY`` # so the sweep re-runs almost immediately after the new # container is up — the day's reconcile is NEVER lost. # Streak untouched (not a data failure). # failed → re-arm to a short exponential backoff and retry the # SAME day, escalating with the streak, up to # ``_SCHED_FAILURE_MAX_SAMEDAY_RETRIES`` attempts; then # fall back to the next cron slot (stop hammering a # hard-down vendor — the staleness watchdog + failure # notifier still cover a persistently-broken target). # # Before this policy, EVERY scheduled finalise advanced the cursor to # tomorrow's slot, so one transient failure — or any deploy landing # mid-sweep — cost a full 24h of downstream staleness. _SCHED_CANCELLED_RETRY_DELAY = timedelta(seconds=60) _SCHED_FAILURE_BACKOFF_BASE = timedelta(minutes=5) _SCHED_FAILURE_BACKOFF_CAP = timedelta(hours=1) _SCHED_FAILURE_MAX_SAMEDAY_RETRIES = 5 # Run statuses that mean "this sweep left unfinished work behind". # # ``ok`` is the ONLY status that means nothing was left: ``_sweep`` # assigns ``partial`` the moment a person (or a Phase-A dept) fails while # the target is still reachable, and ``failed`` when not one vendor call # got through. Both leave rows to redo and both must reach a human — so # this one tuple drives BOTH the cron cursor re-arm (retry the same day) # and the dispatcher's raise-on-failure gate (``run_due_targets``). # # Keeping them on one predicate is deliberate. They used to disagree: # ``partial`` advanced the cursor to tomorrow AND was skipped by the # dispatcher gate, so a live target with per-person casualties both lost # a day of staleness for those people and told nobody. Same tuple as # ``alert_notifier._ALERTING_STATUSES`` — the per-target alert and the # admin page must not disagree about what counts as a failure. _RUN_STATUSES_WITH_FAILURES = ("failed", "partial") # Event-push must-deliver: when a single-person push (hire/offboard event, # admin/MCP "Sync this person") finds the target locked by an in-flight # sweep/push, re-enqueue it up to this many times with this delay between # attempts, so it lands on its own once the lock frees — instead of being # dropped (event_only targets have no sweep to reconcile them; cron targets # would otherwise wait until the next daily sweep to revoke a leaver). _EVENT_PUSH_LOCKBUSY_RETRIES = 3 _EVENT_PUSH_LOCKBUSY_RETRY_DELAY_S = 45 def _sched_failure_backoff(streak: int) -> timedelta: """Exponential backoff for the Nth consecutive scheduled failure (streak ≥ 1): 5m, 10m, 20m, 40m, 60m (capped).""" exp = max(0, streak - 1) delay = _SCHED_FAILURE_BACKOFF_BASE * (2 ** exp) return min(delay, _SCHED_FAILURE_BACKOFF_CAP) def _drift_reverify_stale( last_verified_at: datetime | None, *, jitter_key: str | None = None, ) -> bool: """True when a row hasn't been vendor-verified within the drift window (or never has → re-verify on next sweep). Tolerates naive datetimes: MySQL/TiDB ``DATETIME`` drops tzinfo on read, and stored values are UTC, so a naive value is treated as UTC. ``jitter_key`` (a stable per-row key — person id / dept code) spreads the effective window across ``[INTERVAL, INTERVAL + JITTER]`` deterministically, so a whole population synced at once doesn't clump into one re-verify sweep. """ if last_verified_at is None: return True lv = last_verified_at if lv.tzinfo is None: lv = lv.replace(tzinfo=UTC) window = _DRIFT_REVERIFY_INTERVAL if jitter_key: # zlib.crc32 is stable across processes (unlike hash()); spread over # the jitter span in whole seconds. extra = zlib.crc32(jitter_key.encode()) % int( _DRIFT_REVERIFY_JITTER.total_seconds() ) window = window + timedelta(seconds=extra) return lv < datetime.now(UTC) - window def _leaver_last_working_day_passed(person: IdentityPerson) -> bool: """True iff a leaver's last working day has fully ended in their LOCAL timezone — i.e. we are at/after the first moment of ``termination_date + 1`` local. Mirrors offboarding's ``_leaver_revoke_not_before`` semantics (revoke fires AFTER the last working day ends in the employee's wall-clock) so identity-sync never deactivates a ``leaver_pending`` person before their notice period closes. HRIS-confirm flips a future-dated leaver to ``leaver_pending`` the moment IT confirms — often weeks early — and that status drops them out of the standard ``active``/``on_leave`` filter; without this guard the nightly full-reconcile ``_sweep`` would deactivate a still-working employee (the 2026-05-28 Thalie Zhuang incident; see the ``account_backstop`` 'left'-only invariant in ``scheduler_handler.py``). A missing ``termination_date`` is treated as NOT passed (conservative: never early-deactivate a leaver_pending person whose last day HR has not populated) — the status flip to ``left`` clears the guard. Implementation moved to :mod:`app.identity.leaver_notice` (2026-08-11) so the seat-sync reconcilers share ONE definition of the invariant — they had their own "exclude leaver_pending outright" rule and drift-revoked still-working leavers' seats weeks early. """ return leaver_last_working_day_passed(person) def _person_downstream_gate_reason(person: IdentityPerson) -> str | None: """If ``person`` is excluded from downstream sync by a DESIGN gate (not by absence), return a human-readable reason; else ``None``. Mirrors the WHERE-clause exclusions in :meth:`_load_candidates`: the P99XX test band, SF ``customString24=2061`` (不推送), and ``hris_record_state='purged'``. Used by :meth:`_push_one_person` to distinguish "person exists but is gated by design" (→ excluded, run stays green) from "person genuinely absent" (→ NotFoundError). Without it a 2061 / purged person's lifecycle event lands a misleading ``failed`` run + false-red alert on every event-aware target. """ if is_test_employee_no(person.employee_no) or is_test_employee_no( person.hris_person_id ): return ( "HR reserved P99XX test band (employee_no in P9900-P9999): " "test record, never synced to downstream; skipped by design" ) if is_non_corporate_email(person.work_email): return ( "non-corporate work_email (not a @pingcap.com domain): HRIS " "junk / test record with no corporate identity; never synced " "to downstream; skipped by design" ) if (person.sf_downstream_push_code or "") == _SF_NO_PUSH_CODE: return ( "SF downstream-push gate (customString24=2061 不推送): person is " "flagged never-sync-to-downstream; skipped by design" ) if (person.hris_record_state or "present") == "purged": return ( "HRIS record purged (hris_record_state='purged'): person dropped " "from the HRIS feed; skipped by design" ) return None SUPPORTED_KINDS: frozenset[str] = frozenset({ "scim", "navan", "google_schema", "feishu_contact", "huilianyi", # ``netsuite`` is wired end-to-end (preset, runner_options validator, # runner build, sweep precompute, dept phase) but was missing from this # create gate — so a NS target could run (seeded out-of-band) but never # be (re)created via the API/UI (DR / second-instance dead-end). Add it. "netsuite", }) SUPPORTED_TRIGGERS: frozenset[str] = frozenset({"cron", "manual", "event_only"}) # Dispatch ordering: deterministic by-slug within the same tick. No # cross-target dependencies today — ``feishu_contact`` self-orders # its dept-vs-user phases internally so it doesn't need to outrank # anything. Kept as a dict so a future kind that DOES have a cross- # target dependency can claim a slot without resurrecting the # ordering machinery from scratch. _KIND_DISPATCH_PRIORITY: dict[str, int] = { "google_schema": 50, "scim": 60, "navan": 60, } def _safe_error(value: str | None) -> str | None: """Scrub credential-shaped substrings from a free-form error text before it lands on a DB column (``IdentitySyncRun.error_message``, ``IdentitySyncState.last_error``, ``RunSummary.errors[…]``) or an audit row that the admin UI later renders. Reuses ``_scrub_message`` from the integrations service — strips AWS access-key IDs, PEM blocks, ``password=… / api_key=… / secret=…``-style key=value pairs, JWT shapes, and long base64 blobs. Defense in depth: SCIM servers shouldn't echo the Authorization header in 4xx bodies, but a misbehaving proxy or a vendor's verbose validation message could surface the bearer token. Without this scrub, that token would land in ``identity_sync_state.last_error`` and render in a Tooltip on the admin's runs / state page. """ if value is None: return None return _scrub_secrets(value) # Display-boundary PII masking for the admin read surfaces (vendor # snapshot raw / run-item payloads / drift values). The platform's # pattern is masked-by-default + audited reveal (app/core/masking); # these surfaces are admin/auditor-readable AND PAT-reachable, so a # stolen read token must not dump the population's clear-text mobile # numbers. Masking happens at SERIALIZATION, never in storage — the # drift comparator needs the real values to compare correctly. _SENSITIVE_PHONE_KEY_RE = re.compile( r"mobile|phone|cell", re.IGNORECASE, ) _SENSITIVE_ID_KEY_RE = re.compile( r"certificate|id_?card|identity_?no|national_?id", re.IGNORECASE, ) # NetSuite ``record_type`` runner_option flows into the REST URL path # (``/record/v1/{record_type}/…``) — constrain to a safe identifier. _RECORD_TYPE_RE = re.compile(r"^[a-z][a-z0-9_]*$") def mask_sensitive_display(value: Any) -> Any: """Recursively mask phone- / id-number-shaped values for display.""" from app.core.masking import mask_head_tail, mask_phone if isinstance(value, dict): out: dict[str, Any] = {} for k, v in value.items(): key = str(k) if isinstance(v, str) and v: if _SENSITIVE_PHONE_KEY_RE.search(key): out[k] = mask_phone(v) continue if _SENSITIVE_ID_KEY_RE.search(key): out[k] = mask_head_tail(v) continue out[k] = mask_sensitive_display(v) return out if isinstance(value, list): return [mask_sensitive_display(x) for x in value] return value # Per-row cap for vendor-controlled JSON landing in our tables. A # pathological vendor row (megabyte attribute) must degrade to a # truncation marker, not blow max_allowed_packet and 500 the whole # snapshot refresh / sweep commit. _JSON_STORAGE_CAP = 32 * 1024 def _cap_json_for_storage(obj: Any, *, limit: int = _JSON_STORAGE_CAP) -> Any: if obj is None: return None try: s = json.dumps(obj, ensure_ascii=False, default=str) except (TypeError, ValueError): return {"_unserializable": True} if len(s) <= limit: return obj return {"_truncated": True, "_size": len(s), "_preview": s[:2048]} def _scim_user_snapshot_record(u: dict[str, Any]) -> dict[str, Any]: """One SCIM user resource → one vendor-snapshot record. Shared by every roster source that speaks SCIM shape (the SCIM endpoint itself and the AWS Identity Store puller), so a target that switches transport can't drift in ``external_key`` / ``is_active`` semantics — both feed leaver assurance and revocation. """ user_name = u.get("userName") or "" emails = u.get("emails") or [] email = ( user_name if "@" in str(user_name) else next( (e.get("value") for e in emails if isinstance(e, dict) and e.get("value")), None, ) ) return { "record_type": "user", "external_key": str(u.get("id") or user_name), "display_name": u.get("displayName") or user_name, "email": email, # Absent ``active`` counts as active — under-reporting an account # as inactive would make leaver assurance believe a live account # was already revoked. "vendor_status": ( "active" if u.get("active") is not False else "inactive" ), "is_active": u.get("active") is not False, "raw": u, } def _scim_group_snapshot_record(g: dict[str, Any]) -> dict[str, Any]: """One SCIM group resource → one vendor-snapshot record.""" return { "record_type": "group", "external_key": str(g.get("id") or ""), "display_name": g.get("displayName"), "raw": g, } # ── Validation helpers ──────────────────────────────────────────────── def _validate_schedule(raw: dict[str, Any] | None) -> dict[str, Any]: if raw is None: return {"trigger": "manual", "cron_expr": None, "timezone": "UTC"} trigger = raw.get("trigger") or "manual" if trigger not in SUPPORTED_TRIGGERS: raise AppException( code="IDENTITY_SYNC_INVALID_SCHEDULE", message=f"trigger must be one of {sorted(SUPPORTED_TRIGGERS)}", status_code=400, ) cron_expr = raw.get("cron_expr") if trigger == "cron": if not cron_expr: raise AppException( code="IDENTITY_SYNC_INVALID_SCHEDULE", message="trigger='cron' requires cron_expr", status_code=400, ) # Validate via apscheduler's CronTrigger — same validator the # ScheduledJobs router uses, so syntax expectations stay # consistent across both surfaces. from apscheduler.triggers.cron import CronTrigger try: CronTrigger.from_crontab(cron_expr) except (ValueError, KeyError) as e: raise AppException( code="IDENTITY_SYNC_INVALID_CRON", message=f"invalid cron expression: {e}", status_code=400, ) from e timezone = raw.get("timezone") or "UTC" # Validate the tz at save time. Without this, a typo like # ``Asia/Shanhai`` (Shanghai missing the 'g') silently falls # back to UTC at compute_next_run time and the admin sees # cron firings 8 hours off with no signal in the UI. try: from zoneinfo import ZoneInfo, ZoneInfoNotFoundError ZoneInfo(timezone) except (ZoneInfoNotFoundError, ValueError) as e: raise AppException( code="IDENTITY_SYNC_INVALID_TIMEZONE", message=( f"unknown timezone {timezone!r}. Use IANA tz names " "(e.g. UTC, Asia/Shanghai, America/Los_Angeles)." ), status_code=400, ) from e return {"trigger": trigger, "cron_expr": cron_expr, "timezone": timezone} def _validate_ca_bundle_option(value: Any) -> str | None: if value is None: return None if not isinstance(value, str) or not value.strip(): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="ca_bundle_path must be a non-empty absolute path", status_code=400, ) path = value.strip() from app.domains.identity_sync.runners.tls import ( validate_identity_sync_ca_bundle_path, ) try: validate_identity_sync_ca_bundle_path(path) except ValueError as exc: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=str(exc), status_code=400, ) from exc return path def _validate_runner_options( kind: str, raw: dict[str, Any] | None, ) -> dict[str, Any]: raw = raw or {} if kind in ("scim", "navan"): base_url = (raw.get("base_url") or "").strip() # Validate scheme + host. SSRF defence: reject loopback / # link-local / RFC1918 / IPv6 ULA so an admin (or compromised # admin session) can't point an integration's bearer token # at internal Helix routes (``http://localhost:8000/...``) # or the cloud metadata endpoint (``169.254.169.254``). # Production also requires https://; http:// is allowed only # against loopback for local dev. from urllib.parse import urlparse from app.core.config import settings as _settings is_loopback_name = False if base_url: parsed = urlparse(base_url) if parsed.scheme not in ("http", "https"): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="base_url must start with http:// or https://", status_code=400, ) if not parsed.hostname: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="base_url must include a hostname", status_code=400, ) host = parsed.hostname.lower() is_loopback_name = host in ("localhost", "127.0.0.1", "::1") # IP literal? Reject private / loopback / link-local # ranges. DNS-resolvable hostnames are NOT pre-resolved # here (DNS rebinding would need a separate runtime # check) — the IP-literal check covers the common # SSRF vectors (cloud metadata 169.254.169.254, raw # IPs to internal services). try: import ipaddress ip = ipaddress.ip_address(host) if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: if not ( is_loopback_name and getattr(_settings, "APP_ENV", "development") == "development" ): raise AppException( code="IDENTITY_SYNC_INVALID_BASE_URL", message=( f"base_url host {host!r} is in a " "private / loopback / link-local " "range — not allowed for outbound " "identity sync (SSRF guard)." ), status_code=400, ) except ValueError: # Not an IP literal — DNS hostname. Allow. pass # Production: require https except to loopback. if parsed.scheme == "http" and not ( is_loopback_name and getattr(_settings, "APP_ENV", "development") == "development" ): raise AppException( code="IDENTITY_SYNC_INVALID_BASE_URL", message=( "base_url must be https:// in production " "(http:// only allowed against localhost " "for local dev)." ), status_code=400, ) auth = raw.get("auth_scheme") or "bearer" if auth not in ("bearer", "basic"): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="auth_scheme must be 'bearer' or 'basic'", status_code=400, ) timeout = raw.get("timeout_seconds", 30) if not isinstance(timeout, int) or timeout < 5 or timeout > 300: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="timeout_seconds must be int between 5 and 300", status_code=400, ) rl = raw.get("rate_limit_per_minute", 60) if not isinstance(rl, int) or rl < 1 or rl > 1000: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="rate_limit_per_minute must be int between 1 and 1000", status_code=400, ) verify_ssl_raw = raw.get("verify_ssl", True) if not isinstance(verify_ssl_raw, bool): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="verify_ssl must be a boolean", status_code=400, ) verify_ssl = verify_ssl_raw if not verify_ssl: raise AppException( code="IDENTITY_SYNC_INSECURE_TLS", message=( "verify_ssl=false is forbidden; install an approved CA " "bundle instead." ), status_code=400, ) ca_bundle_path = _validate_ca_bundle_option(raw.get("ca_bundle_path")) # Reject auth-overriding entries in extra_headers. The runner # ``_headers()`` already layers the canonical Authorization # header on top, but admin-supplied auth-shaped headers are a # foot-gun (typo silently nukes auth) and a potential token- # carry-over vector if combined with a swapped base_url. extra_headers = raw.get("extra_headers") or {} if not isinstance(extra_headers, dict): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="extra_headers must be an object", status_code=400, ) forbidden_keys = {"authorization", "cookie", "proxy-authorization"} cleaned_headers: dict[str, str] = {} for k, v in extra_headers.items(): if not isinstance(k, str) or not isinstance(v, str): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="extra_headers keys and values must be strings", status_code=400, ) if k.strip().lower() in forbidden_keys: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( f"extra_headers key {k!r} is reserved — " "Authorization / Cookie / Proxy-Authorization " "are managed by the runner's auth_scheme + " "Integration row api_token, not by extra_headers." ), status_code=400, ) cleaned_headers[k] = v # ``schemas_user_create`` — override the SCIM ``schemas`` list sent # on POST create (some vendors, e.g. Alicloud CloudSSO, reject the # default core+enterprise pair). Documented + a real ScimRunner # ctor param, but was silently DROPPED by this validator (never # forwarded) — an admin who set it got the default with no error. schemas_uc = raw.get("schemas_user_create") if schemas_uc is not None: if not isinstance(schemas_uc, list) or not all( isinstance(x, str) and x for x in schemas_uc ): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="schemas_user_create must be a list of non-empty strings", status_code=400, ) # ``drop_root_segments`` — the sweep honours it for ANY kind when # building org-path context; accept it on SCIM/navan too (was only # validated for google_schema, so an admin who set it here had it # silently dropped). drop_root = raw.get("drop_root_segments") or [] if not isinstance(drop_root, list): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="drop_root_segments must be a list of strings", status_code=400, ) # Snapshot roster source. Validated strictly: a typo'd value would # silently leave the target on the truncating SCIM path, which is # the exact failure mode this knob exists to escape. roster_source = str(raw.get("roster_source") or "scim").strip() if roster_source not in ("scim", "aws_identity_store"): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "roster_source must be 'scim' or 'aws_identity_store', " f"got {roster_source!r}." ), status_code=400, ) aws_integration_slug = str( raw.get("aws_integration_slug") or "", ).strip() identity_store_id = str(raw.get("identity_store_id") or "").strip() identity_store_region = str( raw.get("identity_store_region") or "", ).strip() if roster_source == "aws_identity_store" and not aws_integration_slug: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "roster_source='aws_identity_store' requires " "runner_options.aws_integration_slug — the slug of the " "AWS cloud integration whose credentials can call " "identitystore:ListUsers on that Identity Center " "instance." ), status_code=400, ) if roster_source != "aws_identity_store" and ( aws_integration_slug or identity_store_id or identity_store_region ): # Accepting these silently under roster_source='scim' would # look configured while changing nothing. raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "aws_integration_slug / identity_store_id / " "identity_store_region are only valid with " "roster_source='aws_identity_store'." ), status_code=400, ) out = { "base_url": base_url, "auth_scheme": auth, "extra_headers": cleaned_headers, "verify_ssl": verify_ssl, "timeout_seconds": timeout, "rate_limit_per_minute": rl, # Dept-tree → SCIM Groups push (AWS IAM Identity Center # parity with the legacy Authing downstream sync). Runs as # a group phase AFTER the user sweep (members reference # the user sweep's SCIM ids). Default OFF. "push_groups": bool(raw.get("push_groups", False)), "drop_root_segments": [str(s) for s in drop_root if s], # GET-merge before a full-replace PUT (preserve vendor attrs the # mapping didn't render). OPT-IN — turn on for AWS IdC-style # servers that clear omitted attributes; default off. "put_merge": bool(raw.get("put_merge", False)), # Where the VENDOR SNAPSHOT roster is read from. Writes always # stay on SCIM; this only swaps the read side. ``scim`` # (default) = GET /Users. ``aws_identity_store`` = AWS # Identity Store ListUsers/ListGroups, the only way to # enumerate an AWS IdC tenant past 100 people (its SCIM # endpoint ignores startIndex entirely — see #2734). "roster_source": roster_source, } if roster_source == "aws_identity_store": # AWS credentials live on a separate ``aws_cloud`` integration # (the target's single integration_id is already spoken for by # the SCIM token), so the target points at it by slug. out["aws_integration_slug"] = aws_integration_slug # Both default to the AWS integration's own ``sso_*`` config at # collect time; set here only to override. if identity_store_id: out["identity_store_id"] = identity_store_id if identity_store_region: out["identity_store_region"] = identity_store_region if schemas_uc is not None: out["schemas_user_create"] = [str(x) for x in schemas_uc] if ca_bundle_path is not None: out["ca_bundle_path"] = ca_bundle_path.strip() # Reject UNKNOWN keys instead of silently dropping them — a typo'd # or dead knob (like the previously-dropped schemas_user_create) # gave the admin a false sense it took effect. The SCIM/navan admin # form round-trips the whole runner_options JSON, so every legit key # is in ``out``; anything else is a mistake worth a 400. unknown = set(raw) - set(out) if unknown: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "unknown runner_options key(s) for a scim/navan target: " f"{sorted(unknown)}. Allowed: {sorted(out)}." ), status_code=400, ) return out if kind == "google_schema": schema_id = (raw.get("schema_id") or "").strip() if not schema_id: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="google_schema requires runner_options.schema_id", status_code=400, ) drop_root = raw.get("drop_root_segments") or [] if not isinstance(drop_root, list): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="drop_root_segments must be a list of strings", status_code=400, ) return { "schema_id": schema_id, "drop_root_segments": [str(s) for s in drop_root if s], } if kind == "feishu_contact": # Phase A (departments) is an admin-invariant projection from # IdentityOrgNode → Feishu /contact/v3/departments; Phase B # (users) consults ``mapping_config.rules``. The only admin # knobs are the Authing-parity provisioning policy: # * reopen_frozen (default off): reopen a frozen-but-HRIS-active # in-scope person (打开). Off by default — some frozen # accounts are intentionally suspended (security / leaver # leftover) and must NOT be blindly reopened. # * skip_provision_person_ids: admin skip-list — person_ids that # are never CREATEd or REOPENed (an existing active record is # still updated). Populated from the 同步前预览 create/reopen # candidate list. Stored as canonical person_id strings. opts: dict[str, Any] = { "reopen_frozen": bool(raw.get("reopen_frozen", False)), } skip_ids: list[str] = [] for c in raw.get("skip_provision_person_ids") or []: try: skip_ids.append(str(UUID(str(c)))) except (ValueError, TypeError) as _e: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "skip_provision_person_ids must be a list of " f"person UUID strings; got {c!r}" ), status_code=400, ) from _e opts["skip_provision_person_ids"] = list(dict.fromkeys(skip_ids)) return opts if kind == "huilianyi": # Credentials + endpoint knobs live on the integration row; field # shaping lives in mapping_config.rules. The only runner_options are # the in-place-recode operational keys, set by cutover tooling (NOT # the admin form): ``huilianyi_dept_code_map`` (Helix SF code → # existing 汇联易 custDeptNumber) and ``huilianyi_root_dept_code`` # (vendor PingCAP root). Preserve them when present so an explicit # API write keeps them; the admin-form save path additionally # re-injects them from the existing row (see ``update_target``) so # a plain edit — whose payload omits these — never wipes them. opts: dict[str, Any] = {} dept_map = raw.get("huilianyi_dept_code_map") if isinstance(dept_map, dict): opts["huilianyi_dept_code_map"] = { str(k): str(v) for k, v in dept_map.items() if k and v } root = raw.get("huilianyi_root_dept_code") if root is not None: opts["huilianyi_root_dept_code"] = str(root) return opts if kind == "netsuite": # Credentials live on the integration row (Account ID + TBA # keys); field shaping lives in mapping_config.rules. The # runner_options are operational knobs: # * ``netsuite_default_subsidiary_id`` — OneWorld fallback # injected as subsidiary.id when the company_code value_map # misses (lets a single-person test push before the full map # is filled). # * ``record_type`` — REST record type (default "employee"). # * verify_ssl / timeout_seconds / rate_limit_per_minute — # transport knobs (same bounds as the SCIM kind). opts: dict[str, Any] = {} sub = raw.get("netsuite_default_subsidiary_id") if sub not in (None, ""): opts["netsuite_default_subsidiary_id"] = str(sub) # Update-only safety (default OFF = update-only). When absent/false # the runner PATCHes existing records and returns ``excluded`` on a # 404 instead of creating — so a sweep over an un-adopted population # can never mass-duplicate existing NetSuite employees. Enable only # AFTER the population is adopted (externalId backfilled), when a # 404 genuinely means "new hire". See DECISIONS.md 2026-06-24. if raw.get("netsuite_create_missing") is True: opts["netsuite_create_missing"] = True # Per-person CREATE skip-list (mirrors feishu_contact). person_ids # that must NEVER be PUT-created even with create_missing on: NS # Administrator records (a blind create spawns a duplicate non-admin # record), people finance keeps as multiple subsidiary records, and # test / junk-email rows. Only meaningful with create_missing on; # harmless otherwise. Stored as canonical person_id strings — the # runner build resolves them to 工号 (NS is keyed by externalId=工号). ns_skip_ids: list[str] = [] for c in raw.get("skip_provision_person_ids") or []: try: ns_skip_ids.append(str(UUID(str(c)))) except (ValueError, TypeError) as _e: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "skip_provision_person_ids must be a list of " f"person UUID strings; got {c!r}" ), status_code=400, ) from _e if ns_skip_ids: opts["skip_provision_person_ids"] = list(dict.fromkeys(ns_skip_ids)) # Phase A (department sweep) gate. MUST be carried through here — # the dept phase is triggered solely by this flag in run_target, so # dropping it (as an earlier version did) makes the whole dept sync # unreachable. Default OFF (departments stay Authing-era reference # data until the cutover is adopted). if raw.get("netsuite_sync_departments") is True: opts["netsuite_sync_departments"] = True rt = raw.get("record_type") if rt: # ``record_type`` flows into the REST URL path # (``/record/v1/{record_type}/…``); constrain it to a safe # identifier so a stray value can't break out of the path. rt = str(rt) if not _RECORD_TYPE_RE.match(rt): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "record_type must match ^[a-z][a-z0-9_]*$ " "(e.g. 'employee', 'department')" ), status_code=400, ) opts["record_type"] = rt if "verify_ssl" in raw: verify_ssl_raw = raw.get("verify_ssl") if not isinstance(verify_ssl_raw, bool): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="verify_ssl must be a boolean", status_code=400, ) if not verify_ssl_raw: raise AppException( code="IDENTITY_SYNC_INSECURE_TLS", message=( "NetSuite verify_ssl=false is forbidden; install an " "approved CA bundle instead." ), status_code=400, ) opts["verify_ssl"] = True ca_bundle_path = _validate_ca_bundle_option(raw.get("ca_bundle_path")) if ca_bundle_path is not None: opts["ca_bundle_path"] = ca_bundle_path if "timeout_seconds" in raw: timeout = raw.get("timeout_seconds") if not isinstance(timeout, int) or timeout < 5 or timeout > 300: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="timeout_seconds must be int between 5 and 300", status_code=400, ) opts["timeout_seconds"] = timeout if "rate_limit_per_minute" in raw: rl = raw.get("rate_limit_per_minute") if not isinstance(rl, int) or rl < 1 or rl > 1000: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message="rate_limit_per_minute must be int between 1 and 1000", status_code=400, ) opts["rate_limit_per_minute"] = rl return opts if kind == "zoom": # The ``zoom`` kind shipped 2026-07-24 with NO branch here, which # made its ``runner_options`` unreachable through the service: the # target editor and `POST /targets` both 400'd with "unsupported # kind 'zoom'", and only migration 428's raw seed could set them. # So the tier could not be corrected from the UI even by someone # who knew it was wrong — it took a script in the prod container. # * ``zoom_plan_type`` — the tier a brand-new hire's account is # created as. Validated against the driver's own map so a typo # is a 400 AT SAVE TIME rather than a per-person vendor # rejection at 02:00 UTC. Basic is tenant-capped (20 seats # here) and is NOT the free-for-everyone tier it looks like; # the uncapped baseline is ``no_meetings``. # * ``zoom_max_creates_per_run`` — per-sweep create ceiling. A # create makes Zoom email the person an invitation, so this # bounds the invitation fan-out when the create gate is first # opened on a backlog. 0 disables the ceiling. from app.integrations.drivers.zoom_software import _TYPE_BY_PLAN opts: dict[str, Any] = {} plan = raw.get("zoom_plan_type") if plan not in (None, ""): plan = str(plan).strip().lower() if plan not in _TYPE_BY_PLAN: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( f"zoom_plan_type must be one of " f"{sorted(_TYPE_BY_PLAN)}; got {plan!r}. Note " "'basic' is capped per Zoom tenant — use " "'no_meetings' for an every-employee baseline." ), status_code=400, ) opts["zoom_plan_type"] = plan if "zoom_max_creates_per_run" in raw: budget = raw.get("zoom_max_creates_per_run") if not isinstance(budget, int) or isinstance(budget, bool) or ( budget < 0 or budget > 1000 ): raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "zoom_max_creates_per_run must be an int between 0 " "and 1000 (0 = no per-run ceiling)" ), status_code=400, ) opts["zoom_max_creates_per_run"] = budget unknown = set(raw) - {"zoom_plan_type", "zoom_max_creates_per_run"} if unknown: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( "unknown runner_options key(s) for a zoom target: " f"{sorted(unknown)}. Allowed: " "['zoom_max_creates_per_run', 'zoom_plan_type']." ), status_code=400, ) return opts raise AppException( code="IDENTITY_SYNC_INVALID_KIND", message=f"unsupported kind {kind!r}", status_code=400, ) WRITE_MODES = ("create_update", "create_only", "update_only") # Lifecycle-event reasons whose single-person push is a DEACTIVATION # (``events.enqueue_person_sync`` stamps ``event:`` on # trigger_source). Today only ``offboard`` is fired (leaver-blocked # completion + ``advance_service.process_advance``); the synonyms are # defensive so a future offboard-shaped reason inherits the same # "never silently drop on an event_only target" guarantee in # ``_record_inflight_collapse``. _OFFBOARD_EVENT_REASONS = frozenset({ "offboard", "leave", "terminate", "deactivate", }) def _validate_write_mode(write_mode: str) -> None: """Reject an unknown write-mode (the sync-template operation dimension).""" if write_mode not in WRITE_MODES: raise AppException( code="IDENTITY_SYNC_INVALID_WRITE_MODE", message="write_mode must be create_update|create_only|update_only", status_code=400, ) def _validate_on_offboard(kind: str, on_offboard: str) -> None: """Reject policies the kind's runner cannot execute. The sweep maps an UNSUPPORTED policy to ``last_status='excluded'`` with NO vendor call — for ``on_offboard=delete`` on a runner with ``supports_delete=False`` (huilianyi, feishu_contact) that would silently skip leaver lockout on the expense / directory system while the run reports ok. Refuse at save time instead. """ if on_offboard not in ("deactivate", "delete", "keep"): raise AppException( code="IDENTITY_SYNC_INVALID_ON_OFFBOARD", message="on_offboard must be deactivate|delete|keep", status_code=400, ) if on_offboard == "delete": from app.domains.identity_sync.runners import RUNNER_REGISTRY runner_cls = RUNNER_REGISTRY.get(kind) if runner_cls is not None and not runner_cls.supports_delete: raise AppException( code="IDENTITY_SYNC_ON_OFFBOARD_UNSUPPORTED", message=( f"kind {kind!r} 不支持硬删除(供应商无删除语义)," "on_offboard 请用 deactivate(停用)。" ), status_code=400, ) def _compute_next_run( schedule_config: dict[str, Any], *, after: datetime, ) -> datetime | None: """Compute next fire time for a cron schedule. ``manual`` and ``event_only`` triggers return None — they don't auto-fire.""" if schedule_config.get("trigger") != "cron": return None cron_expr = schedule_config.get("cron_expr") if not cron_expr: return None from apscheduler.triggers.cron import CronTrigger tz_name = schedule_config.get("timezone") or "UTC" try: from zoneinfo import ZoneInfo tz = ZoneInfo(tz_name) except Exception: # noqa: BLE001 # ZoneInfoNotFoundError on weird tz strings — fall back to UTC # rather than crashing the whole dispatch tick. # Pre-launch P2 fix (Round 1, CR-017): log a warning so a # corrupted ``schedule_config.timezone`` row (manual SQL edit, # migration mishap) surfaces in runtime logs instead of # silently dispatching at the wrong wall-clock time. # ``_validate_schedule`` rejects bad tz at create/update time, # so this fallback should never fire — when it does, the row # is already corrupted and operator intervention is needed. logger.warning( "_compute_next_run: invalid timezone %r in schedule_config " "— falling back to UTC; row is corrupted, operator must " "fix the target's schedule_config.timezone", tz_name, ) from zoneinfo import ZoneInfo tz = ZoneInfo("UTC") trigger = CronTrigger.from_crontab(cron_expr, timezone=tz) # APScheduler's get_next_fire_time returns the first fire AFTER # ``previous_fire_time`` and AT/AFTER ``now``. We pass after as # both so an immediate-due target ('* * * * *' just fired) gets # the NEXT slot, not the same slot. next_fire = trigger.get_next_fire_time(after, after) return next_fire async def _retry_person_push_after_delay( *, target_id: UUID, person_id: UUID, trigger_source: str, force_in_scope: bool, retries_left: int, delay_s: int, ) -> None: """Delayed re-enqueue of a single-person push that hit a target lock (event-push must-deliver). Sleeps, then re-runs the push in its OWN session via the public chokepoint (which re-takes the per-(person, target) in-flight guard, so a genuine duplicate still collapses). Never raised back — a failure is recorded on the run/state rows as usual. """ import asyncio as _asyncio await _asyncio.sleep(delay_s) async with async_session_factory() as session: svc = IdentitySyncService(session) # Runs on its OWN session (async_session_factory above) in a # detached retry task — a failure cannot poison any caller's # transaction because the session is discarded on exit. # savepoint-exempt: dedicated retry session try: await svc.sync_person_to_target( target_id=target_id, person_id=person_id, trigger_source=trigger_source, force_in_scope=force_in_scope, lockbusy_retries_left=retries_left, ) await session.commit() except Exception: # noqa: BLE001 — best-effort re-enqueue logger.exception( "identity_sync lockbusy re-enqueue for person=%s target=%s " "failed", person_id, target_id, ) async def _execute_manual_run( *, target_id: UUID, run_id: UUID, trigger_source: str, push_only: bool | None, force_reverify: bool, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str | None = None, ) -> None: """Background body of a manual "Sync now" — runs the sweep against a pre-created ``run_id`` under the per-target lock, in its own session (the HTTP request session is long gone). Never raised back to a caller: every terminal path finalises the run row so the UI polling the run_id always converges. """ from app.core.distributed_lock import LockBusy, distributed_lock async with async_session_factory() as session: svc = IdentitySyncService(session) try: target = await svc.get_target(target_id) except Exception: # noqa: BLE001 — target vanished between click and run await svc._finalise_run( run_id, status="failed", error_message="target not found at run time", summary=None, ) return # Lock on the SLUG — every other sweep path (dispatcher, single-person # push, router preview) keys the per-target lock on ``target.slug``. # Keying on the UUID here would put this manual run in a different lock # namespace, so it would NOT be mutually exclusive with a concurrent # scheduled tick → two full reconciles hitting the same vendor at once # (double writes / 409 storms / racing out-of-scope deactivations). lock_key = f"identity_sync:target:{target.slug}" try: # auto_renew: a manual full reconcile is the same long sweep the # cron path runs; renew so the lock can't lapse mid-run and let a # scheduled tick start a second concurrent sweep. async with distributed_lock(lock_key, ttl=900, auto_renew=True): await svc.run_target( target, trigger_source=trigger_source, push_only=push_only, force_reverify=force_reverify, run_id=run_id, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=trace_id, ) except LockBusy: # A scheduled tick / another manual run holds the target lock. # The lock lives OUTSIDE run_target, so run_target never got a # chance to finalise — do it here. ``cancelled`` (not failed): # nothing went wrong, this run just didn't execute; the in-flight # sweep covers the work. await svc._finalise_run( run_id, status="cancelled", error_message=( "another sweep for this target was already running; " "this manual run did not execute" ), summary=None, ) # Any other exception (incl. CancelledError) is left to propagate: # run_target finalises its OWN run row before raising for every # error it handles, so finalising again here would double-fire the # failure alert and re-arm the cursor twice. The spawn() done- # callback logs the propagated exception; the pre-created row is # already terminal, and the stale-run cleanup is the backstop for # the (row-created-but-run_target-never-entered) edge. # ── Service ────────────────────────────────────────────────────────── class IdentitySyncService: """Façade for the entire identity-sync subsystem.""" def __init__(self, db: AsyncSession): self.db = db self.audit = AuditService(db) # ── CRUD ───────────────────────────────────────────────────────── async def list_targets( self, *, kind: str | None = None, enabled: bool | None = None, include_deleted: bool = False, ) -> list[IdentitySyncTarget]: q = select(IdentitySyncTarget) if not include_deleted: q = q.where(IdentitySyncTarget.deleted_at.is_(None)) if kind: q = q.where(IdentitySyncTarget.kind == kind) if enabled is not None: q = q.where(IdentitySyncTarget.enabled.is_(enabled)) q = q.order_by(IdentitySyncTarget.created_at) return list((await self.db.execute(q)).scalars().all()) async def get_target(self, target_id: UUID) -> IdentitySyncTarget: row = await self.db.get(IdentitySyncTarget, target_id) if row is None or row.deleted_at is not None: raise NotFoundError("IdentitySyncTarget", str(target_id)) return row async def get_target_by_slug(self, slug: str) -> IdentitySyncTarget | None: return ( await self.db.execute( select(IdentitySyncTarget).where( IdentitySyncTarget.slug == slug, IdentitySyncTarget.deleted_at.is_(None), ) ) ).scalars().first() async def _normalize_alert_recipients( self, raw: list[str] | None, ) -> list[str] | None: """Validate + canonicalise an alert-recipient id list. Returns ``None`` for an empty / None input (the notifier then notifies nobody — there is no IT-admin fallback). Otherwise a de-duplicated, order-preserving list of canonical UUID strings, after verifying each references an existing non-deleted person. Raises a clean 400 ``AppException`` on a malformed UUID or an unknown / deleted person so the admin gets actionable feedback instead of a silent drop or a delivery-time storm. """ if not raw: return None cleaned: list[str] = [] seen: set[str] = set() for item in raw: try: canonical = str(UUID(str(item))) except (ValueError, AttributeError, TypeError) as e: raise AppException( code="IDENTITY_SYNC_INVALID_ALERT_RECIPIENT", message=f"alert_recipient_ids contains non-UUID value: {item!r}", status_code=400, ) from e if canonical not in seen: seen.add(canonical) cleaned.append(canonical) if not cleaned: return None rows = ( await self.db.execute( select(IdentityPerson.id).where( IdentityPerson.id.in_([UUID(c) for c in cleaned]), IdentityPerson.deleted_at.is_(None), ) ) ).scalars().all() found = {str(r) for r in rows} missing = [c for c in cleaned if c not in found] if missing: raise AppException( code="IDENTITY_SYNC_UNKNOWN_ALERT_RECIPIENT", message=( "alert_recipient_ids references unknown / deleted " f"person(s): {', '.join(missing)}" ), status_code=400, ) return cleaned async def create_target( self, *, name: str, slug: str, kind: str, mapping_config: dict[str, Any], filter_config: dict[str, Any], schedule_config: dict[str, Any], runner_options: dict[str, Any] | None = None, integration_id: UUID | None = None, enabled: bool = False, on_offboard: str = "deactivate", write_mode: str = "create_update", description: str | None = None, alert_enabled: bool = True, alert_recipient_ids: list[str] | None = None, actor_id: UUID | None = None, ) -> IdentitySyncTarget: if kind not in SUPPORTED_KINDS: raise AppException( code="IDENTITY_SYNC_INVALID_KIND", message=f"unsupported kind {kind!r}", status_code=400, ) # Slug uniqueness — explicit check so we return a clean 409 # rather than a SQL IntegrityError on commit. NOTE this only sees # *live* rows: ``get_target_by_slug`` filters ``deleted_at IS NULL``, # but ``uq_identity_sync_targets_slug`` is unconditional and # ``delete_target`` only soft-deletes (keeps the slug). Reusing a # soft-deleted target's slug therefore passes this pre-check and would # otherwise trip 1062 on flush → poisoned session → 500. The flush is # wrapped in a SAVEPOINT below to catch that (and any concurrent # create race) and re-raise it as the same clean 409. existing = await self.get_target_by_slug(slug) if existing is not None: raise ConflictError( f"target with slug {slug!r} already exists", ) # Validate the three nested configs by parsing them with their # respective dataclasses; serialize back to ensure round-trip # cleanliness (no stray unsupported keys saved). try: parsed_mapping = MappingConfig.from_dict(mapping_config) except MappingError as e: raise AppException( code="IDENTITY_SYNC_INVALID_MAPPING", message=str(e), status_code=400, ) from e try: parsed_filter = FilterConfig.from_dict(filter_config) except FilterError as e: raise AppException( code="IDENTITY_SYNC_INVALID_FILTER", message=str(e), status_code=400, ) from e sched = _validate_schedule(schedule_config) opts = _validate_runner_options(kind, runner_options) _validate_on_offboard(kind, on_offboard) _validate_write_mode(write_mode) recipients = await self._normalize_alert_recipients(alert_recipient_ids) row = IdentitySyncTarget( id=uuid.uuid4(), name=name, slug=slug, description=description, kind=kind, integration_id=integration_id, enabled=enabled, on_offboard=on_offboard, write_mode=write_mode, mapping_config=parsed_mapping.to_dict(), filter_config=parsed_filter.to_dict(), schedule_config=sched, runner_options=opts, alert_enabled=alert_enabled, alert_recipient_ids=recipients, created_by=actor_id, updated_by=actor_id, next_run_at=( _compute_next_run(sched, after=datetime.now(UTC)) if enabled else None ), ) try: async with self.db.begin_nested(): self.db.add(row) await self.db.flush() except IntegrityError as e: # slug collision against a soft-deleted row (or a concurrent # create). The SAVEPOINT rolls back only this INSERT, leaving the # session clean; surface the documented 409 instead of a 500. raise ConflictError( f"target with slug {slug!r} already exists", ) from e await self.audit.log( trace_id=uuid.uuid4(), actor_type="user" if actor_id else "system", actor_user_id=actor_id, action="identity_sync.target.create", status="success", resource_type="identity_sync_target", resource_id=str(row.id), detail={"slug": slug, "kind": kind, "enabled": enabled}, ) return row async def update_target( self, target_id: UUID, *, name: str | None = None, description: str | None = None, integration_id: UUID | None = None, integration_id_set: bool = False, enabled: bool | None = None, on_offboard: str | None = None, write_mode: str | None = None, mapping_config: dict[str, Any] | None = None, filter_config: dict[str, Any] | None = None, schedule_config: dict[str, Any] | None = None, runner_options: dict[str, Any] | None = None, alert_enabled: bool | None = None, alert_recipient_ids: list[str] | None = None, alert_recipient_ids_set: bool = False, actor_id: UUID | None = None, ) -> IdentitySyncTarget: row = await self.get_target(target_id) changes: dict[str, Any] = {} if name is not None: row.name = name changes["name"] = name if description is not None: row.description = description changes["description"] = description if integration_id_set: row.integration_id = integration_id changes["integration_id"] = str(integration_id) if integration_id else None if on_offboard is not None: _validate_on_offboard(row.kind, on_offboard) row.on_offboard = on_offboard changes["on_offboard"] = on_offboard if write_mode is not None: _validate_write_mode(write_mode) row.write_mode = write_mode changes["write_mode"] = write_mode # Track whether the rendered-payload-affecting config changed # — if so we must invalidate every IdentitySyncState row's # ``last_payload_hash`` for this target. Without this, the # payload-hash skip at sweep time would silently keep # pushing the OLD rendered shape (the hash matches), which # is the "I edited my mapping rules and nothing happens" # foot-gun. invalidate_state_hashes = False if mapping_config is not None: try: parsed = MappingConfig.from_dict(mapping_config) except MappingError as e: raise AppException( code="IDENTITY_SYNC_INVALID_MAPPING", message=str(e), status_code=400, ) from e new_mapping = parsed.to_dict() if new_mapping != row.mapping_config: invalidate_state_hashes = True row.mapping_config = new_mapping changes["mapping_config"] = "updated" if filter_config is not None: try: parsed_f = FilterConfig.from_dict(filter_config) except FilterError as e: raise AppException( code="IDENTITY_SYNC_INVALID_FILTER", message=str(e), status_code=400, ) from e row.filter_config = parsed_f.to_dict() changes["filter_config"] = "updated" if schedule_config is not None: row.schedule_config = _validate_schedule(schedule_config) changes["schedule_config"] = "updated" # #1943/#1938 — person ids REMOVED from the admin skip-list # (``runner_options.skip_provision_person_ids``) whose state rows # must be re-armed below. Captured while the old options are # still on the row. unskipped_person_ids: list[UUID] = [] if runner_options is not None: new_opts = _validate_runner_options(row.kind, runner_options) # Preserve server-managed operational keys the admin UI never # surfaces, so a plain "save target" (whose payload round-trips # only the form fields) can't silently wipe them. The huilianyi # in-place-recode map is set by cutover tooling via the API, not # the form — dropping it would re-create SF-coded parallels and # collide (the 2026-06-16 incident: every UI save wiped the map). # NOTE: the netsuite gate flags (``netsuite_sync_departments`` / # ``netsuite_create_missing``) are deliberately NOT preserved # here. Unlike the huilianyi map (whose old validator returned # ``{}`` and wiped everything), the netsuite validator already # KEEPS true flags, and the admin form round-trips the entire # ``runner_options`` JSON — so a plain save can't drop them. They # also use absence to mean "off", so blanket-preserving them # would make the flags impossible to turn off through any # API/form path (absence and explicit-false both re-inject ON). _PRESERVED_RUNNER_OPTS = ( "huilianyi_dept_code_map", "huilianyi_root_dept_code", ) existing_opts = row.runner_options or {} for _k in _PRESERVED_RUNNER_OPTS: if _k not in new_opts and existing_opts.get(_k) is not None: new_opts[_k] = existing_opts[_k] if new_opts != row.runner_options: # ``drop_root_segments`` (google_schema) and the SCIM # ``base_url`` / ``extra_headers`` all affect the # rendered payload OR the lookup target — invalidate. invalidate_state_hashes = True # Skip-list SHRINK: un-skipping a person must re-enable # provisioning NOW, not after the 7-day drift window. The # suppressed attempt stored ``last_status='skipped'`` WITH a # payload hash, so (a) the sweep's unchanged-hash fast path # re-affirms skipped without invoking the runner, and (b) the # hourly onboarding backfill treats ``last_status='skipped'`` # as settled — the person stayed unprovisioned with a green # surface. Collect the removed ids; their state rows are # re-armed after the flush below (hash → NULL so the next # sweep must call the runner; a 'skipped' status → 'pending' # so the hourly backfill re-drives them immediately). _old_skip = set( (existing_opts or {}).get("skip_provision_person_ids") or [] ) _new_skip = set( new_opts.get("skip_provision_person_ids") or [] ) for _removed in _old_skip - _new_skip: try: unskipped_person_ids.append(UUID(str(_removed))) except (TypeError, ValueError): # Malformed legacy entry — nothing to re-arm. continue row.runner_options = new_opts changes["runner_options"] = "updated" if enabled is not None: row.enabled = enabled changes["enabled"] = enabled if alert_enabled is not None: row.alert_enabled = alert_enabled changes["alert_enabled"] = alert_enabled if alert_recipient_ids_set: row.alert_recipient_ids = await self._normalize_alert_recipients( alert_recipient_ids, ) changes["alert_recipient_ids"] = ( len(row.alert_recipient_ids) if row.alert_recipient_ids else 0 ) # Recompute next_run_at ONLY when schedule, enabled state, or # cron expression actually changed. Without this guard, an # admin who edits an unrelated field (name / description) at # 09:59:30 on a target whose ``next_run_at`` was 10:00:00 # would silently overwrite the upcoming fire to "next slot # from now" — possibly skipping a fire that was 30s away. schedule_actually_changed = ( schedule_config is not None or enabled is not None ) if schedule_actually_changed: if row.enabled and row.schedule_config.get("trigger") == "cron": row.next_run_at = _compute_next_run( row.schedule_config, after=datetime.now(UTC), ) else: row.next_run_at = None row.updated_by = actor_id await self.db.flush() # Bust the per-person payload-hash cache when the rendered # shape might have changed. This is a deliberate eager # invalidation — at ~500 employees × N targets we'd rather # re-push than keep pushing the old shape. if invalidate_state_hashes: from sqlalchemy import update as sa_update await self.db.execute( sa_update(IdentitySyncState) .where(IdentitySyncState.target_id == target_id) .values(last_payload_hash=None) ) changes["state_hashes_invalidated"] = True # Re-arm the state rows of persons REMOVED from the skip-list # (see the runner_options branch above). The hash null is # belt-and-suspenders on top of the target-wide invalidation; # the ``skipped`` → ``pending`` reset is the part nothing else # does — the hourly onboarding backfill keys "settled" on # ``last_status='skipped'``, so without it an un-skipped hire # waited for the next full sweep instead of the next hour. if unskipped_person_ids: from sqlalchemy import update as sa_update await self.db.execute( sa_update(IdentitySyncState) .where( IdentitySyncState.target_id == target_id, IdentitySyncState.person_id.in_(unskipped_person_ids), ) .values(last_payload_hash=None) ) await self.db.execute( sa_update(IdentitySyncState) .where( IdentitySyncState.target_id == target_id, IdentitySyncState.person_id.in_(unskipped_person_ids), IdentitySyncState.last_status == "skipped", ) .values(last_status="pending") ) changes["skip_list_removed_states_rearmed"] = len( unskipped_person_ids, ) if changes: await self.audit.log( trace_id=uuid.uuid4(), actor_type="user" if actor_id else "system", actor_user_id=actor_id, action="identity_sync.target.update", status="success", resource_type="identity_sync_target", resource_id=str(target_id), detail=changes, ) return row async def delete_target( self, target_id: UUID, *, actor_id: UUID | None = None, ) -> None: row = await self.get_target(target_id) row.deleted_at = datetime.now(UTC) row.enabled = False row.next_run_at = None row.updated_by = actor_id await self.db.flush() await self.audit.log( trace_id=uuid.uuid4(), actor_type="user" if actor_id else "system", actor_user_id=actor_id, action="identity_sync.target.delete", status="success", resource_type="identity_sync_target", resource_id=str(target_id), detail={"slug": row.slug}, ) # ── Runner construction ────────────────────────────────────────── async def _build_runner( self, target: IdentitySyncTarget, ) -> SyncRunner: """Construct a person-keyed runner for a target. Resolves the integration row + decrypts credentials. Caller is responsible for closing the runner via ``await runner.aclose()``. For ``feishu_contact``, this builds ONLY the user (Phase B) runner — the dept (Phase A) runner is constructed inline by :meth:`_run_feishu_dept_phase` so it can pre-prime the dept catalog and refresh ``identity_sync_department_state`` rows before this user runner sees its dept index. On the single-person event-hook path Phase A doesn't run, and the runner's ``helix_dept_to_external`` comes purely from the already-persisted state rows. """ if target.kind == "feishu_contact": if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires an " "identity_sync_feishu integration; create one " "in 数据连接器 first." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) app_id = resolved.config.get("app_id") app_secret = resolved.config.get("app_secret") if not app_id or not app_secret: raise AppException( code="IDENTITY_SYNC_NO_TOKEN", message=( f"integration {integ.slug!r} missing " "app_id / app_secret; configure under 凭证." ), status_code=400, ) from app.approval.feishu_client import FeishuApprovalClient client = FeishuApprovalClient(app_id=app_id, app_secret=app_secret) # Load the HRIS dept_code → Feishu od-XXX adoption map from # the persistent state table. The user runner uses it to: # * translate mapped HRIS dept_codes into Feishu's # actual department_ids on push # * identify which slice of an existing user's # department_ids Helix manages (vs Authing-built # geographic depts) for the merge step # On the single-person event-hook path, this index is the # ONLY way the runner sees the adoption map (Phase A # doesn't run on per-person triggers). On the full-sweep # path (`run_target`), Phase A refreshes the index BEFORE # Phase B runs by writing fresh state rows + reloading # this map from DB. helix_dept_to_external = await self._load_helix_dept_index( target_id=target.id, ) # SECOND dept dimension — office name → od-XXX. We scan # the target's mapping_config for the ``office_dept_name`` # rule, collect its ``value_map`` values (curated Feishu # office-dept names), and resolve each by name against the # Feishu catalog. Read-only — safe on prod. office_name_to_external = await self._build_office_name_index( client=client, mapping_config=target.mapping_config or {}, ) from app.domains.identity_sync.runners.feishu_contact import ( FeishuContactRunner, ) # Provisioning policy (Authing parity) from runner_options: # * reopen_frozen (default off): reopen an in-scope, # HRIS-active person who is frozen on Feishu. # * skip_provision_person_ids: admin skip-list — never # create or reopen these people (still update if active). # Resolve the skip person_ids to lower-cased work_emails here # (the runner is email-keyed). Malformed ids are dropped, not # fatal, so a stray value can't take the whole sweep down. _opts = target.runner_options or {} _reopen_frozen = bool(_opts.get("reopen_frozen", False)) _skip_emails: set[str] = set() _skip_uuids: list[UUID] = [] for _c in _opts.get("skip_provision_person_ids") or []: try: _skip_uuids.append(UUID(str(_c))) except (ValueError, TypeError): continue if _skip_uuids: _rows = (await self.db.execute( select(IdentityPerson.work_email).where( IdentityPerson.id.in_(_skip_uuids), ) )).all() _skip_emails = {r[0].lower() for r in _rows if r[0]} runner = FeishuContactRunner( client=client, helix_dept_to_external=helix_dept_to_external, office_name_to_external=office_name_to_external, reopen_frozen=_reopen_frozen, skip_provision_emails=frozenset(_skip_emails), ) # Prime the custom_attrs catalog so the runner can drop # mapped ``custom_attrs.`` entries that aren't # configured on the Feishu side. Failure-tolerant (the # method swallows + logs). Reads only — safe on prod. await runner.prime_custom_attrs() return runner if target.kind == "huilianyi": client = await self._build_huilianyi_client(target) from app.domains.identity_sync.runners.huilianyi import ( HuilianyiRunner, ) return HuilianyiRunner( client=client, dept_code_map=(target.runner_options or {}).get( "huilianyi_dept_code_map", ) or {}, ) if target.kind == "netsuite": client = await self._build_netsuite_client(target) from app.domains.identity_sync.runners.netsuite import ( NetSuiteRunner, ) opts = target.runner_options or {} # Resolve the create skip-list (person UUIDs) → 工号 for the # runner (NetSuite is keyed by externalId=工号, not email). # Mirrors the Feishu path's skip_provision_person_ids handling; # malformed ids are dropped, not fatal. Only meaningful with # create_missing on — gates the PUT-create path so admins / # finance-multi-subsidiary / test rows are never auto-created. _skip_uuids: list[UUID] = [] for _c in opts.get("skip_provision_person_ids") or []: try: _skip_uuids.append(UUID(str(_c))) except (ValueError, TypeError): continue _skip_work_nums: set[str] = set() if _skip_uuids: _rows = (await self.db.execute( select(IdentityPerson.employee_no).where( IdentityPerson.id.in_(_skip_uuids), ) )).all() _skip_work_nums = { str(r[0]).strip() for r in _rows if r[0] and str(r[0]).strip() } return NetSuiteRunner( client=client, record_type=str(opts.get("record_type") or "employee"), default_subsidiary_id=opts.get("netsuite_default_subsidiary_id"), create_missing=bool(opts.get("netsuite_create_missing", False)), skip_create_work_nums=frozenset(_skip_work_nums), ) if target.kind == "zoom": # Reuse the existing Zoom Server-to-Server OAuth integration # (``zoom-primary`` — same S2S app / token cache that backs the # seat flow). The runner delegates all HTTP to the driver, so # credential decryption + auth stay in one place. Account # lifecycle here is decoupled from the (decommissioned) Authing # SSO cascade — Helix owns create-on-hire / deactivate-on-leave. if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires a zoom integration; " "pick the Zoom connector in the target editor before " "enabling." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) from app.domains.identity_sync.runners.zoom import ( _DEFAULT_MAX_CREATES_PER_RUN, _DEFAULT_PLAN_TYPE, ZoomRunner, ) from app.integrations.drivers.zoom_software import ZoomSoftwareDriver driver = ZoomSoftwareDriver( integration_id=str(integ.id), config=resolved.config, ) opts = target.runner_options or {} # Fall back to the runner's own defaults rather than restating # them here — the baseline tier moved off "basic" (tenant-capped) # once prod proved Basic creates get refused, and a second copy # of the default would have silently kept the old behaviour for # every target whose runner_options omit the key. try: max_creates = int( opts.get("zoom_max_creates_per_run") if opts.get("zoom_max_creates_per_run") is not None else _DEFAULT_MAX_CREATES_PER_RUN ) except (TypeError, ValueError): max_creates = _DEFAULT_MAX_CREATES_PER_RUN return ZoomRunner( driver=driver, plan_type=str(opts.get("zoom_plan_type") or _DEFAULT_PLAN_TYPE), max_creates_per_run=max_creates, ) if target.kind == "google_schema": # Resolve via the existing integrations service so credential # decryption + driver instantiation stay in one place. if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires a " "google_workspace integration; pick one in the " "target editor before enabling." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) from app.domains.identity_sync.runners.google_schema import GoogleSchemaRunner from app.integrations.drivers.google_workspace import GoogleWorkspaceDriver driver = GoogleWorkspaceDriver( integration_id=str(integ.id), config=resolved.config, ) opts = target.runner_options or {} return GoogleSchemaRunner( driver=driver, schema_id=opts.get("schema_id") or "Department_Information", ) if target.kind in ("scim", "navan"): # SCIM credentials live on the integration row's encrypted # secrets. Convention: ``api_token`` (and optionally # ``api_username`` for basic auth). The integration's # ``config_public.base_url`` overrides the runner_options # default if set — so admins can manage URL on the # integration row without re-saving the target. if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires an integration " "with api_token configured." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) opts = dict(target.runner_options or {}) base_url = resolved.config.get("base_url") or opts.get("base_url") ca_bundle_path = ( resolved.config.get("ca_bundle_path") or opts.get("ca_bundle_path") ) if not base_url: raise AppException( code="IDENTITY_SYNC_NO_BASE_URL", message=( "no base_url set; configure on the integration " "row's config or on the target's runner_options." ), status_code=400, ) api_token = resolved.config.get("api_token") if not api_token: raise AppException( code="IDENTITY_SYNC_NO_TOKEN", message=( f"integration {integ.slug!r} has no 'api_token' " "field; add it under credentials." ), status_code=400, ) # SSRF guard — base_url is admin-entered integration config; a # compromised admin session must not be able to point the authed # SCIM client (now used for outbound group push, #956) at IMDS / # internal services. Mirrors the huilianyi builder below. from app.domains.identity_sync.runners.tls import ( build_identity_sync_tls_verify, development_loopback_tls_exception, ) from app.integrations._url_safety import ( UnsafeURLError, safe_async_http_transport, ) tls_verify = build_identity_sync_tls_verify( base_url=str(base_url), verify_ssl=bool(opts.get("verify_ssl", True)), ca_bundle_path=ca_bundle_path, ) transport = None if not development_loopback_tls_exception(str(base_url)): try: transport = await safe_async_http_transport( str(base_url), verify=tls_verify, ) except UnsafeURLError as e: # ``e.message`` is host + reason only — str(e) would echo # the full url incl. any pasted token material. raise AppException( code="IDENTITY_SYNC_UNSAFE_BASE_URL", message=f"SCIM base_url unsafe: {e.message:.200}", status_code=400, ) from e from app.domains.identity_sync.runners.scim import ScimRunner label = "Navan" if target.kind == "navan" else "SCIM" return ScimRunner( base_url=base_url, auth_scheme=opts.get("auth_scheme") or "bearer", api_token=api_token, api_username=resolved.config.get("api_username"), extra_headers=opts.get("extra_headers") or {}, verify_ssl=bool(opts.get("verify_ssl", True)), ca_bundle_path=ca_bundle_path, timeout_seconds=int(opts.get("timeout_seconds", 30)), rate_limit_per_minute=int(opts.get("rate_limit_per_minute", 60)), # Now forwarded (was validated-then-dropped): a vendor that # rejects the default core+enterprise schemas (CloudSSO) can # override the POST-create ``schemas`` list. schemas_user_create=opts.get("schemas_user_create"), put_merge=bool(opts.get("put_merge", False)), label=label, transport=transport, ) raise AppException( code="IDENTITY_SYNC_INVALID_KIND", message=f"no runner registered for kind {target.kind!r}", status_code=500, ) async def _build_huilianyi_client(self, target: IdentitySyncTarget): """Resolve the ``identity_sync_huilianyi`` integration row and construct an authed :class:`HuilianyiClient`. Shared by the Phase B user-runner builder and the Phase A dept phase (each phase builds its OWN client so aclose() stays per-phase).""" if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires an " "identity_sync_huilianyi integration; create one " "in 数据连接器 first." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) base_url = resolved.config.get("base_url") client_id = resolved.config.get("client_id") client_secret = resolved.config.get("client_secret") if not base_url or not client_id or not client_secret: raise AppException( code="IDENTITY_SYNC_NO_TOKEN", message=( f"integration {integ.slug!r} missing base_url / " "企业ID / 企业密钥; configure under 凭证." ), status_code=400, ) # SSRF guard — base_url is admin-entered integration config; a # compromised admin session must not be able to point the # authed client at IMDS / internal services. Async variant # (the sync one does a blocking DNS resolve). Same helper the # scim-endpoint / SuccessFactors drivers use. from app.domains.identity_sync.runners.tls import ( build_identity_sync_tls_verify, ) from app.integrations._url_safety import ( UnsafeURLError, safe_async_http_transport, ) try: tls_verify = build_identity_sync_tls_verify( base_url=str(base_url), verify_ssl=True, ca_bundle_path=resolved.config.get("ca_bundle_path"), ) transport = await safe_async_http_transport( str(base_url), verify=tls_verify, ) except (UnsafeURLError, ValueError) as e: # ``e.message`` carries host + reason only — str(e) is the # args tuple incl. the FULL url, which can embed pasted # token material (the redaction the class exists for). message = e.message if isinstance(e, UnsafeURLError) else str(e) raise AppException( code="IDENTITY_SYNC_UNSAFE_BASE_URL", message=f"汇联易 base_url/TLS 配置无效: {message:.200}", status_code=400, ) from e from app.domains.identity_sync.runners.huilianyi import ( HuilianyiClient, ) return HuilianyiClient( base_url=str(base_url), client_id=str(client_id), client_secret=str(client_secret), token_path=str( resolved.config.get("token_path") or "/oauth/token" ), token_auth_style=str( resolved.config.get("token_auth_style") or "basic" ), ca_bundle_path=resolved.config.get("ca_bundle_path"), # The cutover full sweep is lookup+write per person # (~1000 calls for ~500 people) — at the conservative # 60/min default it would outlive the 900s per-target # dispatch lock and void single-flight. 240/min (4 rps) # finishes in ~4 min and is far under any SaaS gateway's # real ceiling; steady-state sweeps hash-skip to near # zero calls. rate_limit_per_minute=240, transport=transport, ) async def _build_netsuite_client(self, target: IdentitySyncTarget): """Resolve the ``identity_sync_netsuite`` integration row and construct an authed :class:`NetSuiteClient`. The REST host is derived from the Account ID inside the client (always ``*.suitetalk.api.netsuite.com``), so there is no admin-entered base_url to SSRF-guard — the account-id format check in the client constructor is the boundary.""" if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} requires an " "identity_sync_netsuite integration; create one " "in 数据连接器 first." ), status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) account_id = resolved.config.get("account_id") consumer_key = resolved.config.get("consumer_key") consumer_secret = resolved.config.get("consumer_secret") token_id = resolved.config.get("token_id") token_secret = resolved.config.get("token_secret") if not ( account_id and consumer_key and consumer_secret and token_id and token_secret ): raise AppException( code="IDENTITY_SYNC_NO_TOKEN", message=( f"integration {integ.slug!r} missing Account ID / " "Consumer Key / Consumer Secret / Token ID / Token " "Secret; configure under 凭证." ), status_code=400, ) opts = target.runner_options or {} ca_bundle_path = ( resolved.config.get("ca_bundle_path") or opts.get("ca_bundle_path") ) from app.domains.identity_sync.runners.netsuite import NetSuiteClient try: return NetSuiteClient( account_id=str(account_id), consumer_key=str(consumer_key), consumer_secret=str(consumer_secret), token_id=str(token_id), token_secret=str(token_secret), verify_ssl=bool(opts.get("verify_ssl", True)), ca_bundle_path=ca_bundle_path, timeout_seconds=int(opts.get("timeout_seconds", 30)), rate_limit_per_minute=int(opts.get("rate_limit_per_minute", 60)), ) except ValueError as e: raise AppException( code="IDENTITY_SYNC_INVALID_INTEGRATION", message=f"NetSuite 连接器配置无效: {e}", status_code=400, ) from e async def _build_office_name_index( self, *, client: Any, mapping_config: dict[str, Any], ) -> dict[str, str]: """Resolve curated Feishu office-dept names → ``od-XXX``. The mapping rule with ``target_field=office_dept_name`` carries a ``value_map`` (HRIS work_location → Feishu office-dept name); the VALUES are the names we need to look up against the live Feishu catalog so the user runner can substitute the appropriate ``od-XXX`` into the rendered user's department_ids. Implementation: reuses :class:`FeishuDepartmentRunner.prime_catalog` — BFS over ``/contact/v3/departments`` — and filters the primed ``name_to_external`` to the curated set. Failure of any single API call is tolerated; the office dimension simply degrades to "no office sync" for that sweep, and the user push still goes through with HRIS BU dept only. Called from BOTH the cron-sweep path (after Phase A) AND the per-person event-hook path (Phase A doesn't run there). The re-prime is ~2 sec for the PingCAP-size dept tree — acceptable cost for per-person triggers and absorbed by Phase A's lock on cron sweeps. """ wanted = self._extract_office_names_from_mapping(mapping_config) if not wanted: return {} # Collision guard: warn loudly if the admin's office-name # value_map points at a name that's ALSO an HRIS BU. Both axes # would then resolve to the same od-XXX, and the relocation- # drop logic in the user runner becomes ambiguous (we couldn't # tell whether the dept came from the BU axis or the office # axis). We don't BLOCK — admin might genuinely want the # overlap — but they should know. # Use ``_normalise_name`` so "Engineering Group" vs # "engineering group" (double space) DO collide for this # check (matches the runner's lookup normalisation). async with best_effort( self.db, label="identity_sync: feishu dept collision check", log=logger, ) as collision_step: from app.domains.identity_sync.runners.feishu_department import ( _normalise_name, ) from app.identity.person_models import IdentityOrgNode hris_names = { _normalise_name(n.org_node_name) for n in (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() if n.org_node_name } collisions = { w for w in wanted if _normalise_name(w) in hris_names } if collisions: logger.warning( "feishu office mapping collides with HRIS BU names: " "%s. Both axes will resolve to the same od-XXX; " "relocation-drop semantics become ambiguous. Pick " "distinct names per axis if this is unintentional.", sorted(collisions), ) if not collision_step.ok: # Collision check is advisory — never fail the sweep on # an HRIS-tree read error here. logger.error( "feishu office-name collision check failed (advisory " "only); sweep continues.", ) from app.domains.identity_sync.runners.feishu_department import ( FeishuDepartmentRunner, ) dept_runner = FeishuDepartmentRunner(client=client) try: await dept_runner.prime_catalog() out: dict[str, str] = {} for name in wanted: # Public lookup — no private-attribute access. od = dept_runner.lookup_external_id_by_name(name) if od: out[name] = od else: logger.info( "feishu office-name index: %r not found in the live " "Feishu catalog — extend the office mapping or wait " "for the dept to exist before syncing users.", name, ) return out except Exception: # noqa: BLE001 logger.exception( "identity_sync: office-name index build failed; " "office dimension will be empty for this sweep.", ) return {} finally: try: await dept_runner.aclose() except Exception: # noqa: BLE001 logger.warning( "office-name index probe runner.aclose() raised", exc_info=True, ) # Sentinel substring written by Phase A's mass-action brake into # ``IdentitySyncDepartmentState.last_error``. Used by # :meth:`_load_helix_dept_index` to distinguish brake-blocked rows # (must be excluded from the index — that's the whole point of the # brake) from transiently-failed rows (should remain visible to # preserve relocation semantics on the next healthy sweep). _DEPT_BRAKE_ERROR_MARKER = "mass-action brake (dept):" async def _load_helix_dept_index( self, *, target_id: UUID, ) -> dict[str, str]: """Build the HRIS ``dept_code → Feishu od-XXX`` index from the persistent ``identity_sync_department_state`` rows owned by ``target_id``. **Scoping**: the filter is ``target_id == this target`` so adoptions made by ANOTHER ``feishu_contact`` target (e.g. a staging-vs-prod split, or a future second tenant) DON'T leak across. Without scoping, target B's Phase B would pick up target A's od-XXX (from target A's Feishu tenant) and PATCH target B's tenant with foreign dept ids — hard 230003 every PATCH until target B's Phase A catches up. Used in two places: * Phase B (user sweep) initial build via ``_build_runner`` — seeds the user runner with whatever's been adopted up to this point. * After Phase A completes, the orchestrator calls this again and rebuilds the user runner so the freshly-adopted ``od-XXX`` ids are visible to Phase B's ``department_ids`` push. **Filter**: * Always require ``external_id IS NOT NULL`` (the dept has actually been adopted at some point). * Always exclude ``last_status='pending'`` (never processed — no real adoption). * Always exclude ``last_status='soft_deleted'`` (intentionally retired). * Exclude ``last_status='failed' AND last_error LIKE '%mass-action brake (dept):%'`` — these are brake-blocked and MUST not appear in the user-side Helix-managed set (the brake's whole point is preventing destructive Phase B action on these depts). * KEEP ``last_status='failed'`` rows whose error is a transient per-dept push failure (network, 5xx, etc.) — previous successful adoption is still valid and should be honoured for relocation semantics; the next healthy sweep re-pushes the dept anyway. Round-55: previously broad-except → return ``{}`` on any DB infra failure. That masked a real outage as "no Helix-managed depts" — the user runner then preserved everything in ``department_ids`` and silently no-op'd the entire adoption map for the sweep, with no admin-visible signal. Now: re-raise so the orchestrator's existing systemic-failure path lands the run as failed and the failure-notifier pages ops. """ from sqlalchemy import or_ index: dict[str, str] = {} state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target_id, IdentitySyncDepartmentState.external_id.is_not(None), IdentitySyncDepartmentState.last_status.not_in( ("pending", "soft_deleted"), ), # Exclude the brake-blocked subset of 'failed'. # SQL: NOT (status='failed' AND error LIKE marker). # Phrased as OR so rows with NULL last_error pass. or_( IdentitySyncDepartmentState.last_status != "failed", IdentitySyncDepartmentState.last_error.is_(None), ~IdentitySyncDepartmentState.last_error.like( f"%{self._DEPT_BRAKE_ERROR_MARKER}%", ), ), ) )).scalars().all() for s in state_rows: if s.dept_code and s.external_id: index[s.dept_code] = s.external_id return index # ── Candidate selection ────────────────────────────────────────── async def _load_candidates( self, ) -> tuple[list[CandidatePerson], dict[UUID, IdentityPerson], dict[UUID, IdentityOrgMembership]]: """Snapshot of all non-deleted persons + their primary org membership. Returned as a triple: (slim CandidatePerson list, person_by_id, membership_by_person) so the orchestrator can use the slim list for filtering and the dict for mapping context construction. """ # HRIS-tombstone guard: exclude people the upstream HRIS has # PURGED (``hris_record_state='purged'`` — migration 222). A # purged person has been dropped from the HRIS feed entirely # (e.g. a rescinded offer), even though a stale # ``employment_status`` may still read 'active'. They must never # be provisioned to a downstream (create/reopen), so they are kept # out of both the candidate list AND ``person_by_id``. # # NOTE on a previously-synced purged person (rare: got an account, # then HRIS purged the record): the out-of-scope sweep looks them up # in ``person_by_id`` (line ~2973), finds ``None``, and marks the # state ``excluded`` — it does NOT auto-deactivate the vendor account # here. That is deliberate and safe: a mis-set ``purged`` flag on a # genuinely-active employee must never silently disable their live # account. Real leavers are deactivated by the offboarding / leaver # paths (which key off ``employment_status``), not by this exclusion. # ``coalesce(...,'present')`` treats a NULL/unbackfilled state as # present so a legacy row is never mistaken for purged. # # ALSO exclude people SF flags "不推送到下游系统" — the HR/SF-owned # per-person downstream-provisioning gate ``EmpJob.customString24`` # (stored RAW in ``sf_downstream_push_code``, migration 390; the # same opt-out Authing honoured). HR-confirmed 2026-07-07: code # ``2061`` = 不推送 → never sync this person to ANY downstream # target (feishu / huilianyi / netsuite / scim / google). All # other values (``2059`` = 推送, or empty/unset) sync normally — # ``coalesce(...,'')`` so a NULL never matches the no-push code. # Candidate-set exclusion only: it never auto-deactivates an # existing vendor account (e.g. the 3 manually-created Feishu # accounts HR confirmed to leave alone stay put under # on_offboard=keep). SF is the single source of truth — flip the # flag in SF and Helix follows. from sqlalchemy import func as sql_func persons = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.deleted_at.is_(None), sql_func.coalesce( IdentityPerson.hris_record_state, "present", ) != "purged", sql_func.coalesce( IdentityPerson.sf_downstream_push_code, "", ) != _SF_NO_PUSH_CODE, ) )).scalars().all() # HR test-record band (P99XX): global exclusion for EVERY outbound # target AND the force-push / admin "sync this person" path — both # source their candidate pool here, and ``force_in_scope`` bypasses # ``matches()`` (so the per-target ``employee_no_pattern`` can't be # relied on). Filtered in Python via the shared # ``is_test_employee_no`` primitive so the P99XX rule lives in one # place, dialect-free. Mirrors the purged / no-push exclusions above. persons = [ p for p in persons if not is_test_employee_no(p.employee_no) and not is_test_employee_no(p.hris_person_id) ] # Non-corporate work_email: same-tier global exclusion. HRIS junk / # test records with a personal gmail/qq/yahoo ``work_email`` that # fall outside the P99XX band (their 工号 is P-shaped, so neither the # band gate above nor a target's ``employee_no_pattern`` catches # them) have no @pingcap.com directory identity and were creating # real SCIM accounts for non-people. Candidate-set exclusion only — # like the purged / 2061 / P99XX gates it never auto-deactivates an # existing vendor account (an out-of-scope previously-synced row # becomes ``excluded``, kept, no vendor call). persons = [p for p in persons if not is_non_corporate_email(p.work_email)] person_by_id: dict[UUID, IdentityPerson] = {p.id: p for p in persons} memberships = (await self.db.execute( select(IdentityOrgMembership).where( IdentityOrgMembership.is_primary.is_(True), IdentityOrgMembership.deleted_at.is_(None), IdentityOrgMembership.person_id.in_(person_by_id.keys()), ) )).scalars().all() membership_by_person: dict[UUID, IdentityOrgMembership] = { m.person_id: m for m in memberships } candidates: list[CandidatePerson] = [] for p in persons: if not p.work_email: continue m = membership_by_person.get(p.id) candidates.append(CandidatePerson( person_id=p.id, work_email=p.work_email, employment_status=p.employment_status, country=p.country, job_title=p.job_title, employee_no=p.employee_no, company_code=p.company_code, employee_type_code=p.employee_type_code, hire_date=p.hire_date, hris_person_id=p.hris_person_id, org_path=m.org_path if m else None, org_node_source_id=m.org_node_source_id if m else None, )) return candidates, person_by_id, membership_by_person async def _build_dept_children_index(self) -> dict[str, list[str]]: nodes = (await self.db.execute( select(IdentityOrgNode).where(IdentityOrgNode.deleted_at.is_(None)) )).scalars().all() idx: dict[str, list[str]] = {} for n in nodes: if n.parent_org_node_source_id: idx.setdefault(n.parent_org_node_source_id, []).append( n.org_node_source_id, ) return idx async def _compute_inscope_dept_ids( self, *, target: IdentitySyncTarget, nodes: list[IdentityOrgNode], ) -> set[str]: """Org node source_ids Phase A should project for ``target``. A dept is in scope iff its subtree holds ≥1 in-scope (post- filter) person — i.e. the union of every in-scope person's primary dept id, expanded UP to its full ancestor chain so the tree the synced users hang off of stays intact. Everything else (hollow shells, depts of filtered-out people) is dropped, so we never create empty / cross-language-duplicate departments on Feishu. Mirrors ``_sweep``'s filter evaluation. The CN-manager auto- include is intentionally NOT replicated: it only fires for foreigners-only targets, and ``feishu_contact`` (the sole Phase A kind) has no country filter, so every active employee — and thus every manager — is already in scope directly. """ filter_cfg = FilterConfig.from_dict(target.filter_config or {}) candidates, _person_by_id, membership_by_person = ( await self._load_candidates() ) children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) parent_of: dict[str, str | None] = { n.org_node_source_id: n.parent_org_node_source_id for n in nodes } leaf_ids: set[str] = set() for c in candidates: if not matches(c, filter_cfg, department_source_ids=dept_set): continue if c.org_node_source_id: leaf_ids.add(c.org_node_source_id) scoped: set[str] = set() for sid in leaf_ids: cur: str | None = sid guard = 0 while cur and cur not in scoped and guard < 100: scoped.add(cur) cur = parent_of.get(cur) guard += 1 return scoped async def _huilianyi_dept_attributes( self, *, target: IdentitySyncTarget, in_scope_codes: set[str], ) -> dict[str, HuilianyiDeptAttrs]: """Per-department role/cost-center attributes for 汇联易 Phase A. The 汇联易 dept create/update-V2 API (verified against opendocs + live tenant 2026-06-16) carries the cost center via ``customFormValues`` (fieldCode ``CC``) and the department roles (部门负责人 / HRBP / Budget Owner) via ``departmentPositionList`` (positionCodes 6103 / 6105 / 6102). All four derive from the HRIS golden record so a recoded dept — and any future new dept — carries them: * **leader** (→ positionCode 6103) — the DIRECT member whose manager reports OUTSIDE the dept (the org head), tie-broken toward whoever manages the most dept members. Flat/headless team → None. * **HRBP** (→ 6105) — the ``hris_hrbp_person_id`` (SF 工作关系 relationshipType 2448) shared across the dept's **entire subtree** (self + all descendants), most-common wins, resolved to a 工号. Subtree — NOT direct members — on purpose: a parent/BU node's DIRECT members are its leadership cluster (e.g. a CFO whose personal HRBP is a senior HRBP), whose HRBP differs from the operating teams below. Direct-member plurality therefore stamped a BU with the leadership's HRBP and mis-routed BU-level approvals (团建 等) away from the team's real HRBP — the incident that surfaced this (2026-07-14: Finance BU resolved to the CFO's senior HRBP 唐小丽 instead of the finance teams' HRBP Sally Hua, though the whole Finance subtree is Sally 16:1). Rolling the plurality over the subtree makes the BU HRBP reflect the population it actually covers. Leaf depts are unaffected (subtree == self). * **Budget Owner** (→ 6102) — the head of the dept's top-level **BG** (一级部门), so all ~96 depts carry one of just 14 owners. HR/finance rule confirmed 2026-09-04; NOT the dept's own head, and NOT SF's ``hris_budget_owner_person_id`` (调薪负责人 relationshipType 2447) any more — that is a BU-level comp approver, not a department budget holder, and it mis-stamped 49 of 91 prod depts. See ``_budget_owner_emp_no`` for the evidence. * **cost center** — the member-shared ``cost_center`` code (1:1 with dept in prod) rendered to 汇联易's full label via ``HUILIANYI_COST_CENTER_LABEL_MAP``; an unmapped code → None. HRBP / cost center are taken as the most-common non-null value among the relevant member set; absent → that role is omitted on the push (never blanks an existing vendor role — which is why Budget Owner resolves the BG head through a fallback rather than returning None: omitting it would freeze that dept on whatever the vendor already holds). HRBP rolls the plurality over the SUBTREE roster (see above); leader and cost center stay anchored on the node's DIRECT members (though the leader's REPORT COUNT is tallied over the subtree — see ``_leader_emp_no``), while Budget Owner is resolved once per BG and shared by everything beneath it. Membership here is the dept's FULL active roster — an INTRINSIC dept property, deliberately NOT narrowed by this run's ``person_ids`` / ``departments`` scope. A single-person test or a dept-scoped sweep still pushes a dept's ancestors (for tree integrity), and those ancestors must carry their own real leader / cost center, not blanks computed from "only the people this run touches" (which gave an empty parent dept the symptom that surfaced this). The membership-defining gates (``employment_statuses`` + ``employee_no_pattern``) are kept; only the run-scoping dims are cleared. Best-effort: returns ``{}`` semantics per-dept (missing key → caller pushes code/name/parent only). Never mutates. NOTE: re-loads candidates here rather than threading them out of ``_compute_inscope_dept_ids`` on purpose — that helper is shared by the LIVE feishu / AWS / Navan dept phases, and widening its signature to dedup one extra candidate fetch on this (currently disabled, daily-cron) huilianyi path would put those live syncs in the blast radius for an immaterial gain (~hundreds of rows, once per sweep). Protecting live paths wins over the micro-opt. """ from collections import Counter from app.domains.identity_sync.presets import ( HUILIANYI_COST_CENTER_LABEL_MAP, ) # Clear the run-scoping dims (person_ids + departments) so dept # attributes reflect the dept's FULL roster, not just whoever this # sweep happens to push — see the docstring. employment_statuses + # employee_no_pattern (the "is a real member" gates) are preserved. membership_cfg = FilterConfig.from_dict({ **(target.filter_config or {}), "person_ids": [], "departments": [], }) candidates, person_by_id, _membership = await self._load_candidates() members_by_dept: dict[str, list[UUID]] = {} for c in candidates: # departments cleared → empty subtree set → no dept-scope filter; # person_ids cleared → every active in-pattern member counts. if not matches(c, membership_cfg, department_source_ids=set()): continue sid = c.org_node_source_id # ALL active in-tree members count — deliberately NOT narrowed # to ``in_scope_codes``. The rollup below computes an # ancestor's HRBP over its ENTIRE subtree roster; on a # restricted run (the single-person dept-chain ensure passes # ``restrict_dept_codes`` = one leaf + its ancestors), an # ``in_scope_codes``-narrowed member pool would recompute the # BU's HRBP over {its leadership cluster + one leaf} — the # exact leadership-skew the subtree rollup exists to fix — # push the WRONG HRBP, and ping-pong against every full # sweep. Attribute computation is still limited to in-scope # depts (the loop below); only the MEMBER POOL is full-tree. if sid: members_by_dept.setdefault(sid, []).append(c.person_id) # Subtree rollup for the HRBP role. A parent/BU node's DIRECT members # are its leadership cluster, whose personal HRBP (often a senior HRBP) # differs from the operating teams below — so the HRBP plurality must # roll over the whole subtree, not just direct members (see docstring / # the 团建 mis-route that surfaced this). Leaf depts have no children, # so subtree == direct and they are unaffected. Uses the FULL active # node tree (not just in_scope_codes) so descendants always roll up # even on a dept-scoped / single-person run. node_rows = (await self.db.execute( select( IdentityOrgNode.org_node_source_id, IdentityOrgNode.parent_org_node_source_id, ).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).all() children_of: dict[str, list[str]] = {} parent_of: dict[str, str] = {} for code, parent_code in node_rows: if parent_code: children_of.setdefault(parent_code, []).append(code) parent_of[code] = parent_code def _subtree_member_ids(root_code: str) -> list[UUID]: """Direct members of ``root_code`` plus every descendant's, deduped. Iterative + visited-set so a cyclic / self-parented org row can't spin. Each person sits in exactly one dept, so the union is already duplicate-free; the visited set guards the DEPARTMENT walk.""" out: list[UUID] = [] seen_codes: set[str] = set() stack = [root_code] while stack: code = stack.pop() if code in seen_codes: continue seen_codes.add(code) out.extend(members_by_dept.get(code, [])) stack.extend(children_of.get(code, [])) return out def _emp_no(pid: UUID | None) -> str | None: if pid is None: return None p = person_by_id.get(pid) return p.employee_no if p else None def _mgr(pid: UUID) -> UUID | None: p = person_by_id.get(pid) return p.manager_person_id if p else None def _shared_role_emp_no(member_ids: list[UUID], attr: str) -> str | None: """Most-common role person-id among members → that person's 工号 (HRBP / Budget Owner are per-BU, member-shared like cost center). Only counts role ids that resolve to a known (non-deleted) person — ``person_by_id`` is the non-deleted roster, so a role pointing at a soft-deleted person is excluded. This prevents a stale dead id from winning ``most_common`` and silently shadowing a live minority role (it would resolve to None and drop the role).""" counter: Counter[UUID] = Counter( rid for pid in member_ids if (p := person_by_id.get(pid)) and (rid := getattr(p, attr)) and rid in person_by_id ) if not counter: return None # Deterministic plurality: ``Counter.most_common`` breaks ties # by insertion order, i.e. by the (unordered) row order the DB # happened to return — so a 1:1 tie flapped the HRBP between # sweeps (push A, push B, push A, …), churning the vendor and # the ledger. Tie-break on the holder's 工号 (stable, human- # explicable), falling back to the person UUID string. top = max(counter.values()) winner = min( (rid for rid, n in counter.items() if n == top), key=lambda rid: (_emp_no(rid) or "", str(rid)), ) return _emp_no(winner) def _leader_emp_no(dept_code: str) -> str | None: """Org head of ``dept_code`` = the DIRECT member whose manager reports OUTSIDE the dept, tie-broken toward whoever manages the most people in the dept's SUBTREE. Requires ≥1 such report so genuinely flat teams stay leaderless (no guessing). Returns 工号. **Reports are counted over the SUBTREE, not the node (2026-09-04).** A BG/parent node is almost always shaped "the head sits alone on the node, every one of their reports sits in a CHILD dept": 唐小丽 is the only member of `HR` and her 8 reports are in HR Operations / HRBP / C&B / Talent Acquisition; Liya Du is the only member of `Product` and her 8 reports are in Core Components / AI Lab / … Counting reports on the node alone scored them 0 and declared those depts headless, so 6103 was never pushed for them (汇联易 kept whatever 北森 left — for `HR`, nothing) and Budget Owner fell through to a fallback. Prod dry-run: subtree counting resolves 5 more depts (61 → 66 of 96) and changes NO existing answer — a LEAF's subtree is itself, so leaf depts are untouched by construction. Computed for ANY dept (not just in-scope ones) because the Budget-Owner BG walk below needs an out-of-scope BG's head. """ member_ids = members_by_dept.get(dept_code) or [] if not member_ids: return None member_set = set(member_ids) # How many people in the dept's SUBTREE each DIRECT member # manages (see docstring — leaf subtree == self, so unchanged). report_count: Counter[UUID] = Counter() for pid in _subtree_member_ids(dept_code): mid = _mgr(pid) if mid is not None and mid in member_set and mid != pid: report_count[mid] += 1 outside = [pid for pid in member_ids if _mgr(pid) not in member_set] if not outside: return None outside.sort( key=lambda pid: ( -report_count.get(pid, 0), _emp_no(pid) or "", ) ) head = outside[0] if report_count.get(head, 0) <= 0: return None return _emp_no(head) # Every in-pattern active member of ANY dept — the "汇联易 has this # person" predicate for a role reference. A manager outside this set # (inactive, non-P 工号, unassigned to a dept) is never referenced. all_member_ids: set[UUID] = { pid for ids in members_by_dept.values() for pid in ids } def _external_manager_emp_no(dept_code: str) -> str | None: """The person a HEADLESS dept actually reports to: the most-common manager of its SUBTREE members who sits outside that subtree (deterministic 工号 tie-break, same as the plurality helpers). Only used as the BG-head fallback below, for a BG with no internal management at all — `AI Innovation` (8 members, 7 reporting to 刘奇 P0001 and 1 to 黄东旭 P0002) and `Strategic Consultant` (2 members, both reporting to 刘奇). HR confirmed 刘奇 is indeed those BGs' owner, which is exactly what the plurality returns. """ member_ids = members_by_dept.get(dept_code) or [] if not member_ids: return None pool = _subtree_member_ids(dept_code) pool_set = set(pool) counter: Counter[UUID] = Counter( mid for pid in pool if (mid := _mgr(pid)) and mid not in pool_set and mid in all_member_ids ) if not counter: return None top = max(counter.values()) winner = min( (mid for mid, n in counter.items() if n == top), key=lambda mid: (_emp_no(mid) or "", str(mid)), ) return _emp_no(winner) def _bg_code(dept_code: str) -> str: """The dept's top-level BG (一级部门) — the root of its branch. Helix's org tree has the 14 BGs as roots (parent=None); the 汇联易 "PingCAP" RootOrg node exists only vendor-side and is re-parented onto at push time, so it never appears here. Visited set so a cyclic / self-parented org row can't spin. """ seen: set[str] = set() cur = dept_code while cur not in seen: seen.add(cur) parent = parent_of.get(cur) if not parent: return cur cur = parent return dept_code def _budget_owner_emp_no(dept_code: str) -> str | None: """Budget Owner (6102) = the head of the dept's top-level **BG**. HR/finance rule, confirmed 2026-09-04: every department's expense budget is owned at the 一级 BG level, so all ~96 depts carry one of just 14 owners (a dept under Engineering Group → Mark Xu, one under Product → Liya Du, and so on). Deliberately NOT the dept's own head — finance accepted that `Intelligent Support Enablement` moves from its own head 朱明 P0809 to Engineering Group's Mark Xu P0346 under this rule. **Not** SF ``hris_budget_owner_person_id`` (relationshipType 2447 调薪负责人) either, which is what this originally pushed: 2447 is a BU-level COMP-adjustment approver, not a budget holder. Prod evidence: the 91 depts with active members resolved to only 23 distinct 2447 holders and 49 of 91 carried someone who was neither their own head nor their BG's. That column stays populated and is still surfaced on the person profile / comp paths; it just no longer feeds this vendor role. The BG head itself is the BG's derived org head, falling back to its subtree's most-common external manager for a BG with no internal management at all (`AI Innovation`, `Strategic Consultant` — both → 刘奇 P0001, which HR confirmed). With the subtree-report relaxation in ``_leader_emp_no`` all 14 prod BGs resolve, so no dept is left frozen on a stale vendor value. """ bg = _bg_code(dept_code) return _leader_emp_no(bg) or _external_manager_emp_no(bg) attrs: dict[str, HuilianyiDeptAttrs] = {} for dept_code, member_ids in members_by_dept.items(): # Attributes are only needed (and only pushed) for depts in # THIS run's scope; the full-tree ``members_by_dept`` above # exists so subtree rollups see every descendant's members. if dept_code not in in_scope_codes: continue leader_no = _leader_emp_no(dept_code) # Cost center = member-shared code → full vendor label. cc_counter: Counter[str] = Counter( p.cost_center for pid in member_ids if (p := person_by_id.get(pid)) and p.cost_center ) cc_label: str | None = None if cc_counter: cc_code = cc_counter.most_common(1)[0][0] cc_label = HUILIANYI_COST_CENTER_LABEL_MAP.get(cc_code) attrs[dept_code] = HuilianyiDeptAttrs( leader_no=leader_no, cost_center_label=cc_label, hrbp_no=_shared_role_emp_no( _subtree_member_ids(dept_code), "hris_hrbp_person_id", ), budget_owner_no=_budget_owner_emp_no(dept_code), ) return attrs async def generate_huilianyi_dept_code_map( self, *, target: IdentitySyncTarget, ) -> dict[str, Any]: """Propose an in-place-recode map: Helix SF dept code → existing 汇联易 ``custDeptNumber``, disambiguated by FULL org PATH. SF and the Authing/北森-era 汇联易 tree share no code, only names — and names repeat across BUs (Sales / Security / PreSales appear in several). So name-only matching is unsafe; this matches the whole normalized name-path (Helix ``IdentityOrgNode.org_path`` ↔ vendor ``departmentPath``), which is unique. The SF-coded parallels Helix itself already created are EXCLUDED from the vendor index (else a recoded dept self-collides on its own path). READ-ONLY (pulls the vendor dept list, touches nothing). Returns ``{map, unmatched, ambiguous, summary}`` for human review before the map is written into ``runner_options.huilianyi_dept_code_map``. """ from app.domains.identity_sync.runners.huilianyi import ( fetch_all_departments, ) def _segs(raw: str | None, sep: str) -> tuple[str, ...]: parts = [ " ".join(str(x).split()).strip().lower() for x in str(raw or "").split(sep) ] parts = [p for p in parts if p] if parts and parts[0] == "pingcap": # drop the shared root label parts = parts[1:] return tuple(parts) nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() helix_codes = {n.org_node_source_id for n in nodes} client = await self._build_huilianyi_client(target) try: vendor = await fetch_all_departments(client) finally: await client.aclose() # Vendor path → [custDeptNumber], active only, EXCLUDING Helix's own # SF-coded parallels (their code == a Helix node code). vidx: dict[tuple[str, ...], list[str]] = {} for d in vendor: if str(d.get("status")) != "101": continue code = str(d.get("custDeptNumber") or "") if not code or code in helix_codes: continue vidx.setdefault(_segs(d.get("departmentPath"), "|"), []).append(code) code_map: dict[str, str] = {} unmatched: list[dict[str, str]] = [] ambiguous: list[dict[str, Any]] = [] for n in nodes: key = _segs(n.org_path, "/") cands = vidx.get(key, []) if len(cands) == 1: code_map[n.org_node_source_id] = cands[0] elif len(cands) > 1: ambiguous.append({ "sf_code": n.org_node_source_id, "name": n.org_node_name, "path": " / ".join(key), "vendor_codes": cands, }) else: unmatched.append({ "sf_code": n.org_node_source_id, "name": n.org_node_name, "path": " / ".join(key), }) return { "map": code_map, "unmatched": unmatched, "ambiguous": ambiguous, "summary": { "helix_nodes": len(nodes), "mapped": len(code_map), "unmatched": len(unmatched), "ambiguous": len(ambiguous), }, } async def _resolve_manager_emails( self, person_ids: Iterable[UUID], ) -> dict[UUID, tuple[str | None, str | None]]: """Map person_id → (manager_email, manager_name). Resolves through the person.manager_id FK. The manager_id field lives on IdentityPerson — see that model for definition. For people whose manager_id is None or missing from the cache, returns (None, None). """ return await self._resolve_related_emails( person_ids, "manager_person_id", ) async def _resolve_hrbp_emails( self, person_ids: Iterable[UUID], ) -> dict[UUID, tuple[str | None, str | None]]: """Map person_id → (hrbp_email, hrbp_name). Resolves through ``IdentityPerson.hris_hrbp_person_id`` (populated by the SF sync from ``EmpJobRelationships`` 2448). Feeds the Feishu contact runner's HRBP GENERIC_USER custom attribute. (None, None) when the person has no HRBP declared in SF. """ return await self._resolve_related_emails( person_ids, "hris_hrbp_person_id", ) async def _resolve_manager_identity( self, person_ids: Iterable[UUID], ) -> dict[UUID, tuple[str | None, str | None]]: """Map person_id → (manager_employee_no, manager_employment_status). Sibling of :meth:`_resolve_manager_emails` used by the dry-run preview so the admin UI can flag a departed (离职) manager via PersonTag without widening the shared 2-tuple resolver (which also feeds SCIM payload rendering). """ ids = [pid for pid in person_ids if pid is not None] if not ids: return {} person_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() mgr_ids = { p.manager_person_id for p in person_rows if p.manager_person_id } if not mgr_ids: return {pid: (None, None) for pid in ids} mgr_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(mgr_ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() mgr_by_id = {m.id: m for m in mgr_rows} out: dict[UUID, tuple[str | None, str | None]] = {} for p in person_rows: mgr = ( mgr_by_id.get(p.manager_person_id) if p.manager_person_id else None ) if mgr is None: out[p.id] = (None, None) else: out[p.id] = (mgr.employee_no, mgr.employment_status) return out async def _resolve_pushable_manager_nos( self, filter_config: dict[str, Any] | None, person_ids: Iterable[UUID], ) -> dict[UUID, str]: """Map person_id → direct manager's 工号, but ONLY when that manager is part of the population this target actually pushes. Backs 汇联易's 主岗直属领导 (``directManagerEmpoyeeId``), which is addressed by 工号 — so unlike NetSuite's ``ns_supervisor_id`` there is no vendor-id lookup, only a "does the vendor have this person?" question. The gates below are the vendor-population predicate: a P-shaped 工号 (the target's ``employee_no_pattern``), an in-scope employment status, not an HRIS tombstone, not an HR test record, not the person themselves. A manager failing ANY gate is simply omitted → the mapping rule renders empty → the field drops → 汇联易 keeps whatever it already has. Never a guess, never a blanking. Deliberately NOT narrowed by this run's ``person_ids`` / ``departments`` scope: "who is my manager" is intrinsic to the person, not to the slice of people this particular run happens to touch (same reasoning as :meth:`_huilianyi_dept_attributes`). A single-person re-push must still carry the same manager the full sweep would. """ ids = [pid for pid in person_ids if pid is not None] if not ids: return {} person_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() mgr_ids = { p.manager_person_id for p in person_rows if p.manager_person_id and p.manager_person_id != p.id } if not mgr_ids: return {} mgr_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(mgr_ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() cfg = FilterConfig.from_dict(filter_config or {}) # Same default as ``matches()``: an unconstrained status list means # active-only, NOT "any status" — a 离职 manager must never be # referenced just because the admin left the dimension empty. statuses = set(cfg.employment_statuses) or {"active"} pushable: dict[UUID, str] = {} for m in mgr_rows: emp_no = (m.employee_no or "").strip() if fails_employee_no_gate(emp_no, cfg): continue if is_test_employee_no(emp_no) or is_test_employee_no( m.hris_person_id, ): continue if m.employment_status not in statuses: continue if (m.hris_record_state or "present") == "purged": continue pushable[m.id] = emp_no out: dict[UUID, str] = {} for p in person_rows: mid = p.manager_person_id if not mid or mid == p.id: continue mgr_no = pushable.get(mid) if mgr_no and mgr_no != (p.employee_no or "").strip(): out[p.id] = mgr_no return out async def _resolve_related_emails( self, person_ids: Iterable[UUID], fk_attr: str, ) -> dict[UUID, tuple[str | None, str | None]]: """Shared resolver: map person_id → (related_email, related_name) through a self-referential person FK column named ``fk_attr`` (e.g. ``manager_person_id`` / ``hris_hrbp_person_id``). Direct attribute access (NOT ``getattr(..., None)`` fallback) so a future column rename surfaces as AttributeError at first call instead of silently rendering as "no relation." See CLAUDE.md memory ``feedback_no_getattr_default_for_model_columns``. Defence-in-depth: callers normally pass only in-scope (non-deleted) person_ids, but a stale candidate list could include a person tombstoned mid-sync. Filter deleted_at to avoid resolving a mapping for a person we should not be touching anyway. """ ids = [pid for pid in person_ids if pid is not None] if not ids: return {} person_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() # getattr with a known literal attr name + NO default — raises # loudly if the column is ever renamed (intentional, per memory). rel_ids = { getattr(p, fk_attr) for p in person_rows if getattr(p, fk_attr) } if not rel_ids: return {pid: (None, None) for pid in ids} rel_rows = (await self.db.execute( select(IdentityPerson).where( IdentityPerson.id.in_(rel_ids), IdentityPerson.deleted_at.is_(None), ) )).scalars().all() rel_by_id = {m.id: m for m in rel_rows} out: dict[UUID, tuple[str | None, str | None]] = {} for p in person_rows: rid = getattr(p, fk_attr) if rid is None: out[p.id] = (None, None) continue rel = rel_by_id.get(rid) if rel is None: out[p.id] = (None, None) continue # Fallback chain mirrors ``lookup_persons`` in the router # (name_pinyin → display_name → work_email) so display # strings stay consistent across UI surfaces. rel_name = ( rel.name_pinyin or rel.display_name or rel.work_email ) out[p.id] = (rel.work_email, rel_name) return out # ── Run a single target end-to-end ─────────────────────────────── async def run_target( self, target: IdentitySyncTarget, *, trigger_source: str = "scheduler", push_only: bool | None = None, force_reverify: bool = False, run_id: UUID | None = None, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str | None = None, ) -> IdentitySyncRun: """Sweep all candidates against this target. Per-person locked externally by the dispatcher's per-target lock — this method does NOT acquire its own lock. ``push_only`` — create/update the in-scope set only, NEVER deactivate users or soft-delete depts that are merely outside THIS run's filter. Default when unset: MANUAL / EVENT runs are push-only (a scoped test must not touch anyone else), while the SCHEDULED cron does a full reconcile (deactivating genuine leavers). Pass explicitly to override. This is what makes it safe to sync any person/group for testing without collateral deactivation of everyone else previously synced. """ if push_only is None: push_only = trigger_source.startswith(("manual", "event")) confirmation_token_hashes = confirmation_token_hashes or [] destructive_trace_id = trace_id or f"identity-sync-destructive:{run_id or target.id}" # ``run_id`` may be pre-created by the background-trigger path # (``run_target_background`` inserts the 'running' row inside the # HTTP request so the handler can return the id immediately, then # spawns this sweep off-request). When passed, adopt that row # instead of minting a second one; otherwise create it here as the # scheduled / inline path always has. if run_id is None: run_id = uuid.uuid4() started = datetime.now(UTC) run_row = IdentitySyncRun( id=run_id, target_id=target.id, started_at=started, status="running", trigger_source=trigger_source, ) self.db.add(run_row) await self.db.flush() await self.db.commit() # Reload target in a fresh session so we don't hold the row # under the long-running sweep. async with async_session_factory() as work_session: work_target = await work_session.get(IdentitySyncTarget, target.id) if work_target is None or work_target.deleted_at is not None: # Race with delete — finalise as failed. await self._finalise_run( run_id, status="failed", error_message="target deleted between dispatch and run", summary=None, ) raise NotFoundError("IdentitySyncTarget", str(target.id)) try: runner = None dept_summary: RunSummary | None = None try: # Phase A — feishu_contact runs a dept sweep first # so the user runner's department_ids merge sees # the fresh adoption map. Runs BEFORE _build_runner # constructs the user runner so the user runner # picks up the just-written od-XXX ids on # ``_load_helix_dept_index``. # Manual / event runs record hash-skip ledger rows # too (the admin is watching one specific person); # cron sweeps skip them (steady-state noise). record_skipped = trigger_source.startswith( ("manual", "event"), ) if work_target.kind == "feishu_contact": dept_summary = await IdentitySyncService(work_session)._run_feishu_dept_phase( target=work_target, run_id=run_id, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=destructive_trace_id, ) elif work_target.kind == "huilianyi": # Same Phase A contract as feishu_contact: # depts must exist before Phase B references # them via custDeptNumber on the user body. dept_summary = await IdentitySyncService(work_session)._run_huilianyi_dept_phase( target=work_target, run_id=run_id, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=destructive_trace_id, ) elif ( work_target.kind == "netsuite" and (work_target.runner_options or {}).get( "netsuite_sync_departments", ) is True ): # Phase A is GATED OFF by default: NetSuite # departments are still Authing-era reference data # until the cutover is adopted. Only the explicit # ``runner_options.netsuite_sync_departments=true`` # flag opts a target into the dept sweep; enabling # the target / running the employee sweep alone # never touches departments. (See DECISIONS.md.) dept_summary = await IdentitySyncService(work_session)._run_netsuite_dept_phase( target=work_target, run_id=run_id, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=destructive_trace_id, ) runner = await IdentitySyncService(work_session)._build_runner(work_target) if not isinstance(runner, SyncRunner): raise AppException( code="IDENTITY_SYNC_RUNNER_MISMATCH", message=( f"target {work_target.slug!r} kind=" f"{work_target.kind!r} expects " "SyncRunner but builder returned " f"{type(runner).__name__}" ), status_code=500, ) summary, status = await IdentitySyncService(work_session)._sweep( target=work_target, runner=runner, run_id=run_id, record_skipped=record_skipped, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=destructive_trace_id, ) # Group phase — scim/navan with push_groups runs # AFTER the user sweep because group members # reference the SCIM user ids the sweep just # persisted to identity_sync_state.external_id. if ( work_target.kind in ("scim", "navan") and (work_target.runner_options or {}).get("push_groups") ): if status == "failed": # The user sweep refused / hard-failed # (e.g. mass-action brake). A replace-all # group write on that state would execute # exactly the removal the brake blocked — # skip and say so. summary.errors.append( "group phase skipped: user sweep " "status=failed — fix the sweep first." ) else: dept_summary = await IdentitySyncService( work_session, )._run_scim_group_phase( target=work_target, runner=runner, run_id=run_id, ) # Merge Phase A counters into the run summary so # the admin sees both halves in one run row. if dept_summary is not None: summary.dept_in_scope = dept_summary.dept_in_scope summary.dept_created = dept_summary.dept_created summary.dept_updated = dept_summary.dept_updated summary.dept_soft_deleted = dept_summary.dept_soft_deleted summary.dept_failed = dept_summary.dept_failed summary.dept_skipped = dept_summary.dept_skipped # If Phase A had failures, downgrade run status # to "partial" — unless we already failed harder. if dept_summary.dept_failed > 0 and status == "ok": status = "partial" # max(0, …): _sweep's mass-action brake message # is appended past the per-person <50 cap, so # len(errors) can be 51 — a bare negative slice # would paste nearly ALL dept errors instead of # none. room = max(0, 50 - len(summary.errors)) for err in dept_summary.errors[:room]: summary.errors.append(f"[dept] {err}") finally: # Wrap aclose in its own try so an httpx # connection-cleanup error doesn't replace the # original sweep exception in flight. if runner is not None: try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "runner.aclose() raised for %s", target.slug, exc_info=True, ) except AppException as e: await self._finalise_run( run_id, status="failed", error_message=f"{e.code}: {e.message}", summary=None, ) raise except asyncio.CancelledError: # Worker shutdown / deploy / task cancel. ``CancelledError`` # is NOT a subclass of ``Exception`` in Python 3.8+, so the # bare-Exception catch above doesn't see it. We MUST finalize # the run row before re-raising, otherwise the row leaks at # status='running' until a watchdog cleans it up. Always # re-raise to honor cooperative cancel semantics. # # Status is ``cancelled`` (NOT ``failed``): a killed sweep is # an infra event, not a data-quality failure. ``_finalise_run`` # re-arms a cancelled SCHEDULED run's cursor to fire again in # ~60s (so the day's reconcile is not lost when a deploy lands # mid-sweep), the metrics dashboard excludes it from the # success rate, and the failure notifier does not page anyone. await self._finalise_run( run_id, status="cancelled", error_message=( "cancelled (worker shutdown / deploy / task cancel) " "— will re-run" ), summary=None, ) raise except Exception as e: # noqa: BLE001 logger.exception( "identity_sync.run_target %s failed", target.slug, ) await self._finalise_run( run_id, status="failed", error_message=f"{type(e).__name__}: {e!s:.500}", summary=None, ) raise await work_session.commit() await self._finalise_run( run_id, status=status, error_message=None, summary=summary, ) return await self._reload_finalised_run(run_id) async def _reload_finalised_run( self, run_id: UUID, ) -> IdentitySyncRun | None: """Return the run row exactly as ``_finalise_run`` committed it. ``_finalise_run`` writes through its OWN session, so reading the row back on ``self.db`` is stale in two independent ways — and the previous ``select()`` here defeated neither: * The ``running`` row ``run_target`` added at the top is still live in ``self.db``'s identity map, and ``async_session_factory`` is ``expire_on_commit=False`` (``app/core/database.py``) so the ``commit()`` never expired it. A plain ORM ``select()`` does NOT overwrite the loaded attributes of an instance already in the identity map — only ``populate_existing`` does. (The old comment called itself "defensive against future ``expire_on_commit=False`` changes"; the factory has been ``expire_on_commit=False`` all along, so the defence never once worked.) * A caller that read on ``self.db`` before the sweep — ``get_target`` in ``_execute_manual_run`` and in the MCP ``trigger_identity_sync_target`` tool — is holding a REPEATABLE READ snapshot taken before ``_finalise_run`` committed, so even a ``populate_existing`` re-read would return the pre-sweep state. Reading through a fresh session sidesteps both without a ``rollback()``/``commit()`` on the caller's session (which would discard, e.g., the MCP tool-call audit row pending on ``rt.db``). The row is returned detached with every column loaded; ``IdentitySyncRun`` has no relationships, and every caller reads plain scalars off it (``.id`` / ``.status`` / ``.result_summary``). What the staleness cost (2026-09-06, found from a NetSuite alert): ``run_target`` returned ``status='running'`` for **every** sweep, so ``run_due_targets``'s ``ran_failed`` gate — built precisely so "a revoked vendor credential could not silently kill the nightly leaver-lockout backstop for weeks" — was dead code for any run that finalises ``failed`` WITHOUT raising. In prod, netsuite's nightly sweep finalised ``failed`` on 2026-08-21/22/23 and ``identity_sync.dispatch`` reported an ok tick on all three days; the only thing that spoke up was the staleness watchdog. The MCP trigger tool had the same bug in its response, always answering ``status: "running"`` with a null summary. """ async with async_session_factory() as fresh: return (await fresh.execute( select(IdentitySyncRun).where(IdentitySyncRun.id == run_id) )).scalars().first() async def start_manual_run_background( self, target: IdentitySyncTarget, *, trigger_source: str, push_only: bool | None = None, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str | None = None, ) -> UUID: """Create the run row NOW (inside the HTTP request) and hand the actual sweep off to a retained background task, returning the run_id immediately. Why: a manual "Sync now" used to run the entire sweep INLINE in the request handler (``run_target`` under the per-target lock). Modern Starlette cancels the handler task on client disconnect — a closed browser tab or an nginx ``proxy_read_timeout`` mid-sweep then lands the run in the ``CancelledError`` path, aborting a NetSuite reconcile that legitimately takes minutes. Detaching the sweep from the request means the admin's connection lifetime no longer bounds the sweep; the UI polls the returned run_id for progress (identical observability to a scheduled run). The per-target lock is acquired INSIDE the background task (not here) so a concurrent scheduled tick is handled the same way the dispatcher handles it: LockBusy → the run finalises ``cancelled`` ("another sweep already running"), never a hard failure. """ run_id = uuid.uuid4() run_row = IdentitySyncRun( id=run_id, target_id=target.id, started_at=datetime.now(UTC), status="running", trigger_source=trigger_source, ) self.db.add(run_row) await self.db.flush() await self.db.commit() from app.core.background import spawn spawn( _execute_manual_run( target_id=target.id, run_id=run_id, trigger_source=trigger_source, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=trace_id, ), name=f"identity_sync.manual_run:{run_id}", ) return run_id async def cleanup_stale_runs(self, *, stale_minutes: int = 30) -> dict[str, Any]: """Finalise *orphaned* ``IdentitySyncRun`` rows stuck at ``status='running'`` past ``stale_minutes`` as ``cancelled``. A hard-killed worker (SIGKILL / OOM) leaves the run row pinned at ``running`` — the per-target Redis lock TTLs out but the row does not, and the web-app startup watchdog only sweeps these on boot (30-min floor). This recurring cleanup covers the runtime blind spot. ``cancelled`` (not ``failed``): an infra event, excluded from the success rate and failure alerting. The target's ``next_run_at`` is intentionally untouched — a hard kill never advanced the cron cursor, so the next dispatch tick re-runs the sweep naturally. CRITICAL — a long *healthy* sweep is NOT an orphan. A NetSuite full reconcile, a mapping-change mass re-push, or a detached manual "Sync now" can legitimately run well past ``stale_minutes`` at 60 req/min + Retry-After backoffs. Those hold the per-target lock the whole time (``auto_renew`` re-extends it); a hard-killed run's lock has lapsed. So we finalise a candidate ONLY when its target's lock is NOT held — the lock-lapsed state is exactly the orphan case. Without this guard the reaper would flip live sweeps to ``cancelled`` mid-flight (phantom cancellations in metrics + the UI polling ``GET /runs/{id}`` reporting "did not execute" while the sweep is still writing to the vendor). """ from app.core.distributed_lock import is_lock_held threshold = datetime.now(UTC) - timedelta(minutes=stale_minutes) now = datetime.now(UTC) rows = (await self.db.execute( select(IdentitySyncRun).where( IdentitySyncRun.status == "running", IdentitySyncRun.started_at < threshold, ) )).scalars().all() finalised = 0 skipped_live = 0 for run_row in rows: # Resolve the target slug to build its lock key. A live sweep # holds ``identity_sync:target:``; if the lock is (still) # held this is a healthy long run, not an orphan — leave it. target = await self.db.get(IdentitySyncTarget, run_row.target_id) if target is not None: try: if await is_lock_held( f"identity_sync:target:{target.slug}", ): skipped_live += 1 continue except Exception: # noqa: BLE001 — Redis blip: stay conservative # Can't confirm liveness → do NOT cancel (cancelling a live # sweep is worse than leaving an orphan one more cycle; the # next cleanup / startup watchdog is the backstop). During a # Redis outage no new sweep can acquire a lock anyway. skipped_live += 1 continue run_row.status = "cancelled" run_row.finished_at = now started = run_row.started_at if started is not None and started.tzinfo is None: started = started.replace(tzinfo=UTC) run_row.duration_ms = max(0, int( (now - started).total_seconds() * 1000, )) if started is not None else 0 if not run_row.error_message: run_row.error_message = ( "orphaned running row (worker hard-exit / OOM) — " "finalised cancelled by stale-run cleanup; next " "scheduled tick re-runs the sweep" ) finalised += 1 if finalised: await self.db.commit() logger.warning( "identity_sync.cleanup_stale_runs: finalised %d orphan " "'running' run row(s) older than %d min as cancelled " "(%d skipped — lock still held = live sweep)", finalised, stale_minutes, skipped_live, ) return { "finalised": finalised, "skipped_live": skipped_live, "stale_minutes": stale_minutes, } def _add_run_item( self, *, run_id: UUID | None, target_id: UUID, item_type: str, outcome: str, subject_display: str, person_id: UUID | None = None, external_key: str | None = None, request_payload: dict[str, Any] | None = None, response_detail: dict[str, Any] | None = None, error: str | None = None, duration_ms: int | None = None, ) -> None: """Append one per-run ledger row (no-op without a run_id). Rides the caller's session + batch-commit cadence — the ledger is observability, not a separate transaction; if a batch rolls back, its items roll back with the state rows they describe. ``error`` goes through the same credential scrub as state-row errors. """ if run_id is None: return self.db.add(IdentitySyncRunItem( id=uuid.uuid4(), run_id=run_id, target_id=target_id, item_type=item_type, person_id=person_id, subject_display=(subject_display or "—")[:255], external_key=(str(external_key)[:255] if external_key else None), outcome=outcome, request_payload=_cap_json_for_storage(request_payload), response_detail=_cap_json_for_storage(response_detail), error=_safe_error(error), duration_ms=duration_ms, )) def _compute_auto_included_manager_ids( self, *, in_scope_persons: list[CandidatePerson], in_scope_ids: set[UUID], person_by_id: dict[UUID, IdentityPerson], state_rows: list[IdentitySyncState], filter_cfg: FilterConfig, ) -> set[UUID]: """The set of manager person-ids auto-included into scope. A Navan/SCIM target enrols a subset of employees, but their manager may fall outside the filter (a different entity / country). For SCIM ``enterprise.manager`` to resolve — and, in Navan's case, for the manager to actually be able to approve the report's travel — the manager must exist in the vendor too. So any ACTIVE manager of an in-scope report who isn't already in scope is pulled in. Once auto-included, the manager stays in scope permanently (rehydration of prior ``auto_included_reason`` rows) — managers don't yo-yo in/out as their team changes. The ``person_ids`` override ("指定员工 / 调试·灰度") means EXACTLY those people (UI contract: "一旦填了,其它筛选条件全部失效"), so it disables auto-include entirely — a single-person debug run syncs that one person and nobody else. Extracted so :meth:`_sweep` (which actually upserts them) and :meth:`dry_run` (which must PREVIEW the same impact set, #1753) never drift. Every returned id has already passed the same active / non-deleted / has-email gate ``_sweep`` applies at materialisation — a rehydrated leaver/hard-deleted manager is dropped here so it isn't re-created on the vendor. 2026-07-07: the earlier ``is_china_employee`` gate was dropped — it was set for only ~37 people company-wide (unreliable) and silently excluded real approvers. """ if filter_cfg.person_ids: return set() managers_to_include: list[UUID] = [] for cand in in_scope_persons: p = person_by_id.get(cand.person_id) if p is None: continue mid = p.manager_person_id if mid is None or mid in in_scope_ids: continue mgr = person_by_id.get(mid) if mgr is None or mgr.deleted_at is not None or not mgr.work_email: continue if mgr.employment_status != "active": # Don't push leaver_pending / pre_hire managers; they # have other lifecycle flows that own that decision. continue managers_to_include.append(mid) # Rehydrate auto-included rows from prior sweeps — "permanently in # scope" by contract even if the original foreign report is gone, # so an attribute change months later still propagates. for s in state_rows: if s.auto_included_reason and s.person_id not in in_scope_ids: managers_to_include.append(s.person_id) # Dedupe + apply the same active/email/hard-delete gate the sweep # applies at materialisation. Leaver invariant: a departed # (left / leaver_pending) auto-included manager must NOT be # rehydrated + re-upserted — ``person_by_id`` includes left # employees and the rehydration loop above has no employment gate, # so without this a manager who leaves gets re-created every # nightly sweep, undoing offboarding's deactivation forever. result: set[UUID] = set() for mid in set(managers_to_include): mgr = person_by_id.get(mid) if mgr is None or mgr.deleted_at is not None or not mgr.work_email: continue if mgr.employment_status != "active": continue result.add(mid) return result async def _sweep( self, *, target: IdentitySyncTarget, runner: SyncRunner, run_id: UUID | None = None, record_skipped: bool = False, push_only: bool = False, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> tuple[RunSummary, str]: """Run the per-person loop for one target. Updates state rows in-place, returns the aggregate summary. ``push_only`` (manual / test syncs — see :meth:`run_target`): create/update the in-scope set ONLY; NEVER deactivate people who are merely outside THIS run's filter. Deactivating genuine leavers is the scheduled full-reconcile's job, not a scoped manual test — this is what lets an admin sync any person/group without collateral deactivation of everyone else previously synced. ``run_id`` enables the per-run ledger (one IdentitySyncRunItem per vendor-touching operation); ``record_skipped`` additionally records hash-skip rows — on for manual/event runs (the admin is watching), off for cron sweeps (steady-state noise). """ summary = RunSummary() # Validate configs (fail fast — admin may have bypassed the # router's validation by editing JSON directly via DB). try: mapping_cfg = MappingConfig.from_dict(target.mapping_config or {}) filter_cfg = FilterConfig.from_dict(target.filter_config or {}) except (MappingError, FilterError) as e: summary.errors.append(f"config invalid: {e}") return summary, "failed" # Snapshot of people + dept tree. candidates, person_by_id, membership_by_person = await self._load_candidates() children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) # Existing state rows for this target — used to detect # transitions (in_scope → out_of_scope) so we apply on_offboard # to people who fell out. state_rows = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() state_by_person: dict[UUID, IdentitySyncState] = { s.person_id: s for s in state_rows } # Manager resolution (one batch). # # ── Future-hire deferral ───────────────────────────────────── # A not-yet-started future hire is kept OUT of scope so we don't # create their vendor account (Feishu / directory / …) days # before their start date — parity with the event-time # ``is_future_hire`` gate in hr_sync_coordinator. Their account # is created on their actual start date by the daily # ``hr.hire_day_it_notify`` cron (which re-fires the event push). # We defer ONLY a first-time creation: a person already synced to # THIS target (has a state row) stays in scope so attribute # updates and out-of-scope deactivation keep flowing and we never # disturb an existing account (e.g. a rehire re-confirmed with a # future start date, or the offboarding-side deactivation of a # future-dated leaver). ``person_ids`` explicit allow-list is a # deliberate admin grant and bypasses the deferral entirely. _override_ids = set(filter_cfg.person_ids) _lead_hours = await get_provision_lead_hours(self.db) in_scope_persons: list[CandidatePerson] = [ c for c in candidates if matches(c, filter_cfg, department_source_ids=dept_set) and not ( c.person_id not in state_by_person and str(c.person_id) not in _override_ids and (_p := person_by_id.get(c.person_id)) is not None and is_future_hire(_p, lead_hours=_lead_hours) ) ] in_scope_ids: set[UUID] = {c.person_id for c in in_scope_persons} # ── Auto-include managers of in-scope reports (approvers) ──── # Compute the auto-included manager set via the shared helper (see # its docstring for the full rationale) so the dry-run preview # (:meth:`dry_run`) computes the SAME set — otherwise the "Sync # now" confirm dialog understates real impact (#1753). The helper # already applies the person_ids-override suppression AND the # active/email/hard-delete gates, so every id it returns is safe # to materialise directly. auto_included_person_ids: set[UUID] = ( self._compute_auto_included_manager_ids( in_scope_persons=in_scope_persons, in_scope_ids=in_scope_ids, person_by_id=person_by_id, state_rows=state_rows, filter_cfg=filter_cfg, ) ) # Materialise a CandidatePerson for each auto-included manager so # they go through the same render/upsert loop as filter matches. for mid in auto_included_person_ids: mgr = person_by_id.get(mid) if mgr is None: # Guaranteed present by the helper's gate, but be defensive. continue mgr_membership = membership_by_person.get(mid) in_scope_persons.append(CandidatePerson( person_id=mid, work_email=mgr.work_email, employment_status=mgr.employment_status, country=mgr.country, job_title=mgr.job_title, employee_no=mgr.employee_no, company_code=mgr.company_code, employee_type_code=mgr.employee_type_code, hire_date=mgr.hire_date, hris_person_id=mgr.hris_person_id, org_path=mgr_membership.org_path if mgr_membership else None, org_node_source_id=( mgr_membership.org_node_source_id if mgr_membership else None ), )) in_scope_ids.add(mid) summary.persons_in_scope = len(in_scope_persons) summary.auto_included = len(auto_included_person_ids) managers_by_person = await self._resolve_manager_emails(in_scope_ids) hrbp_by_person = await self._resolve_hrbp_emails(in_scope_ids) # 汇联易 per-sweep precompute: person → manager's 工号 (主岗直属领导). # One extra query for the whole sweep, empty dict for every other # kind so no other vendor's payload can move. hly_manager_nos: dict[UUID, str] = ( await self._resolve_pushable_manager_nos( target.filter_config, in_scope_ids, ) if target.kind == "huilianyi" else {} ) # ── NetSuite per-sweep precompute (NS-only). Two lookups: # * ``ns_id_by_emp_no``: 工号 → unique active NetSuite internal id, # used to resolve ``supervisor`` (the direct manager) to a vendor # id. (The department-ROLE fields — leader/budget/hrbp — are no # longer written: Helix's golden-record derivation disagrees with # the authoritative Authing-era values too often to overwrite # safely; see presets.py / DECISIONS.md 2026-06-25.) # * ``ns_adopted_dept_codes``: the set of org_node_source_ids that # actually carry an externalId in NetSuite (adopted depts). An # employee's ``department`` reference is only emitted when its # dept is in this set — a dangling department.externalId would # 400 the entire employee upsert. # Every other kind leaves both empty so the per-person injection # below is a no-op for them. ns_id_by_emp_no: dict[str, str] = {} ns_adopted_dept_codes: set[str] = set() if target.kind == "netsuite": client = getattr(runner, "_client", None) try: rows = await client.suiteql( "SELECT id, custentity_swc_work_num AS wn, isinactive " "FROM employee WHERE custentity_swc_work_num IS NOT NULL " "ORDER BY id" ) if client is not None else [] _ids: dict[str, list[str]] = {} for r in rows: wn = (r.get("wn") or "").strip() inactive = str(r.get("isinactive", "")).lower() in ( "true", "t", "1", ) if wn and not inactive: _ids.setdefault(wn, []).append(str(r.get("id"))) # Unique active record per 工号 only — an ambiguous 工号 # (multiple active records) resolves to nothing → the # reference simply drops, never points at a guess. ns_id_by_emp_no = { wn: ids[0] for wn, ids in _ids.items() if len(ids) == 1 } drows = await client.suiteql( "SELECT externalid AS eid FROM department " "WHERE externalid IS NOT NULL ORDER BY id" ) if client is not None else [] ns_adopted_dept_codes = { str(r.get("eid")).strip() for r in drows if (r.get("eid") or "").strip() } except Exception as e: # noqa: BLE001 # FAIL-LOUD (do NOT proceed with empty maps). If the supervisor # / adopted-dept precompute fails, EVERY person's rendered # payload silently loses ``supervisor.id`` + ``department`` → # the hash flips for the WHOLE population → a mass re-push that # WRITES those reduced payloads (dropping supervisor/dept on # every employee), then flips back next sweep when the map # recovers. Abort the sweep with ONE systemic error instead # (the run fails, PR1's same-day backoff retries it) — a clean # skip beats corrupting the whole population. logger.exception( "netsuite precompute (work_num→id / adopted-dept) failed " "for target=%s — aborting the sweep to avoid a " "whole-population reduced-payload re-push", target.slug, ) raise AppException( code="IDENTITY_SYNC_NS_PRECOMPUTE_FAILED", message=( "NetSuite SuiteQL precompute failed " f"({type(e).__name__}: {e!s:.200}) — aborted the sweep " "rather than push reduced payloads for everyone. " "Retries on the next scheduled backoff." ), status_code=502, ) from e runner_opts = target.runner_options or {} drop_root = runner_opts.get("drop_root_segments") or [] # Persist state mutations every ``BATCH_COMMIT_SIZE`` persons # so a runner crash mid-sweep doesn't lose every prior person's # state. The vendor side has already received those upserts; # without this commit cadence the next sweep would redo them # (each costing POST→409→GET→PUT round-trips on SCIM). BATCH_COMMIT_SIZE = 25 processed_in_batch = 0 async def _maybe_commit_batch() -> None: nonlocal processed_in_batch if processed_in_batch >= BATCH_COMMIT_SIZE: await self.db.commit() processed_in_batch = 0 # ── In-scope sweep ──────────────────────────────────────────── for cand in in_scope_persons: person = person_by_id.get(cand.person_id) if person is None: continue membership = membership_by_person.get(cand.person_id) mgr_email, mgr_name = managers_by_person.get( cand.person_id, (None, None), ) hrbp_email, hrbp_name = hrbp_by_person.get( cand.person_id, (None, None), ) ctx = build_person_context( person, org_path=membership.org_path if membership else None, manager_email=mgr_email, manager_name=mgr_name, hrbp_email=hrbp_email, hrbp_name=hrbp_name, org_path_drop_root=drop_root, org_node_source_id=( membership.org_node_source_id if membership else None ), ) # NetSuite injection (NS-only; both lookups are empty for every # other kind so nothing is added). Each token is set ONLY when it # resolves safely; otherwise it stays absent → the mapping rule # drops → the vendor field is preserved (never blanked). # * ns_supervisor_id — the direct manager's unique active NS id # (manager is an NS admin / not-yet-adopted → supervisor left # untouched this sweep). # * ns_department_code — the person's dept org_node_source_id, # but ONLY when that dept is adopted in NS (carries the # externalId); otherwise omitted so we never send a dangling # department reference that would 400 the upsert. if target.kind == "netsuite": mid = person.manager_person_id mgr_no = ( person_by_id[mid].employee_no if mid and mid in person_by_id else None ) if mgr_no and mgr_no in ns_id_by_emp_no: ctx["ns_supervisor_id"] = ns_id_by_emp_no[mgr_no] dcode = cand.org_node_source_id if dcode and dcode in ns_adopted_dept_codes: ctx["ns_department_code"] = dcode # 汇联易 injection (huilianyi-only): the direct manager's 工号, # already gated on "this manager is in the pushed population". # Absent → rule renders empty → field drops → vendor unchanged. if hly_manager_nos: mgr_no_hly = hly_manager_nos.get(cand.person_id) if mgr_no_hly: ctx["manager_employee_no"] = mgr_no_hly try: # Already-linked rows render the UPDATE shape: rules # marked ``create_only`` are omitted so merge-semantics # vendors keep their existing value (e.g. NetSuite # firstName — required non-empty on CREATE, must never # clobber a foreign hire's vendor-curated split name on # UPDATE). The hash is computed on this effective # payload, so a person transitioning unlinked→linked # re-pushes exactly once and then settles. _linked = state_by_person.get(cand.person_id) payload = render_payload( mapping_cfg, ctx, exclude_create_only=bool(_linked and _linked.external_id), ) # Data residency is a creation-time compliance decision, not # a configurable Contact field. Keep the HRIS work-location # signal alongside the rendered payload for the Feishu runner; # the runner consumes (and removes) this private key before # issuing either vendor request. if target.kind == "feishu_contact": payload["_helix_residence_country"] = ( ctx.get("residence_country") or "" ) except MappingError as e: # Update / create the state row so the admin's "State" # tab reflects the failure — without this, a render # bug only shows up in the run summary count and the # state tab still says ``success`` for a person who # has been failing every sweep. Bump retry_count too # so a persistent rendering issue surfaces in the UI. state = state_by_person.get(person.id) if state is None: state = IdentitySyncState( id=uuid.uuid4(), target_id=target.id, person_id=person.id, last_status="pending", ) self.db.add(state) state_by_person[person.id] = state state.last_status = "failed" state.last_synced_at = datetime.now(UTC) state.last_error = _safe_error(f"render: {e}") state.retry_count = (state.retry_count or 0) + 1 summary.failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{person.work_email}: render: {e}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="failed", subject_display=person.work_email, person_id=person.id, error=f"render: {e}", ) processed_in_batch += 1 await _maybe_commit_batch() continue new_hash = hash_payload(payload) # Outcome-taxonomy diagnostics (cheap, no vendor I/O): record # STRICT value_map misses for the "new unmapped code" signal, and # capture any required-field drops for the pre-vendor block below. _vm_misses, _dropped_required = collect_render_diagnostics( mapping_cfg, ctx, # Mirror render_payload above: create_only rules aren't in # the UPDATE shape, so they can't miss or drop for a # linked person. exclude_create_only=bool(_linked and _linked.external_id), ) for _tf, _src in _vm_misses.items(): summary.value_map_misses.setdefault(_tf, set()).add(_src) state = state_by_person.get(person.id) if state is None: state = IdentitySyncState( id=uuid.uuid4(), target_id=target.id, person_id=person.id, last_status="pending", ) self.db.add(state) state_by_person[person.id] = state # Mark / re-affirm the auto-include flag every sweep so a # row that was originally normal but is now also a # manager-of-foreign carries the flag too. We never CLEAR # the flag — once auto-included, permanently in-scope. if ( person.id in auto_included_person_ids and not state.auto_included_reason ): state.auto_included_reason = "manager_of_foreign" # Skip if payload unchanged AND last status was success — a # prior failed person should still retry on the next sweep # even if the rendered payload didn't change. ``excluded`` is # likewise terminal-until-the-payload-changes: re-attempting an # admin-locked NetSuite record every sweep just re-collects the # identical 400, so an unchanged-payload excluded row is a no-op # that re-affirms its excluded status (no vendor call). # # Drift re-verification: the hash covers only the HRIS-rendered # payload, NOT live vendor state — so an unchanged hash does not # prove the vendor record still matches. A row not vendor-verified # within ``_DRIFT_REVERIFY_INTERVAL`` is therefore NOT skipped: it # falls through to the runner, which re-asserts the Helix-managed # fields (idempotent PATCH — a no-op when nothing drifted) and # corrects any vendor-side edit. ``last_verified_at`` is bumped # only on a real vendor touch below, so hash-skips age it out. # force_reverify (admin "强制全量重推") bypasses BOTH the hash # match and the staleness window — re-push every in-scope row now. _verify_stale = force_reverify or _drift_reverify_stale( state.last_verified_at, jitter_key=str(person.id), ) if ( state.last_payload_hash == new_hash and state.last_status in ( "success", "skipped", "excluded", "blocked", ) and not _verify_stale ): # Re-affirm the row's prior terminal status WITHOUT a vendor # call. ``excluded`` (policy) and ``blocked`` (data-dependent # vendor refusal) both persist across an unchanged payload — # re-issuing the same push would just re-collect the identical # rejection. A payload change (data fixed) drops out of this # short-circuit and re-attempts, which is exactly right. _prior = state.last_status _short = _prior if _prior in ("excluded", "blocked") else "skipped" state.last_status = _short state.last_synced_at = datetime.now(UTC) # A blocked row keeps its actionable reason on last_error so # the admin can still see WHY it's blocked; skipped/excluded # clear it (nothing actionable). if _short != "blocked": state.last_error = None if _short == "excluded": summary.excluded += 1 elif _short == "blocked": summary.blocked += 1 else: summary.skipped += 1 if record_skipped: self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome=_short, subject_display=person.work_email, person_id=person.id, external_key=state.external_id, request_payload=payload, response_detail={ "reason": ( "payload unchanged — re-affirmed blocked" if _short == "blocked" else "payload unchanged" ), }, ) # Skips mutate state too — keep them on the 25-row # commit cadence so a mostly-skipped steady-state sweep # doesn't accumulate everything into one giant final # commit (the crash-resilience the cadence exists for). processed_in_batch += 1 await _maybe_commit_batch() continue # Write-mode gate (sync-template matrix operation dimension). # Evaluated on Helix's per-(target,person) link state, BEFORE # any runner call — so no runner internals change: # • create_only — skip anyone Helix has already linked # (external_id set): create once, never re-touch. # • update_only — skip anyone Helix hasn't linked yet # (external_id None): only maintain existing, never create. # create_update (default) never skips here. The gate does NOT # write last_payload_hash, so it re-evaluates cheaply each # sweep (no vendor call) as link state changes. _wm_skip = None if target.write_mode == "create_only" and state.external_id: _wm_skip = "write_mode=create_only (already linked)" elif target.write_mode == "update_only" and not state.external_id: _wm_skip = "write_mode=update_only (not yet linked)" if _wm_skip: state.last_status = "skipped" _wm_now = datetime.now(UTC) state.last_synced_at = _wm_now # Bump the drift-reverify clock: a write-mode skip is a # deliberate no-touch of an already-decided link state (no # vendor drift is possible for a gate that blocks the vendor), # so treating the row as perpetually "stale" would defeat the # cheap hash-skip fast-path forever (create_only linked rows). state.last_verified_at = _wm_now state.last_error = None summary.skipped += 1 if record_skipped: self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="skipped", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, request_payload=payload, response_detail={"reason": _wm_skip}, ) processed_in_batch += 1 await _maybe_commit_batch() continue # ``was_new`` decides created-vs-updated for the summary # AND the ledger row — capture before the push mutates the # state row. was_new = state.last_payload_hash is None push_t0 = time.monotonic() # Required-field preflight (outcome taxonomy): if the mapping # declared a field ``required`` and it rendered to nothing # (empty source / STRICT value_map miss, no default), DON'T burn # a vendor call that will 400 every sweep — mark the person # ``blocked`` with an actionable reason. Auto-clears when the # source data is fixed (payload hash changes → not hash-skipped). if _dropped_required: _reason = ( "required field(s) not rendered: " + "; ".join(f"{tf} ({why})" for tf, why in _dropped_required) + " — fix the person's HRIS data or the target's mapping " "value_map, then it re-syncs automatically." ) outcome = PersonOutcome( status="blocked", error=_reason, detail={ "blocked_reason": "required_field_missing", "fields": [tf for tf, _ in _dropped_required], }, ) else: # Push. try: outcome = await runner.upsert( person_email=person.work_email, payload=payload, prior_external_id=state.external_id, ) except AppException as e: # Systemic-failure propagation: bad creds / token / # tenant rate-limit blow up the whole sweep with one # alert instead of writing N "failed" state rows # under the same broken condition. Commit any # already-accumulated state first so progress isn't # lost. from app.approval.feishu_client import ( is_systemic_feishu_failure, ) if is_systemic_feishu_failure(e): if processed_in_batch > 0: await self.db.commit() raise logger.exception( "identity_sync.upsert %s for %s failed", target.slug, person.work_email, ) outcome = PersonOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) except Exception as e: # noqa: BLE001 logger.exception( "identity_sync.upsert %s for %s failed", target.slug, person.work_email, ) outcome = PersonOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) push_duration_ms = int((time.monotonic() - push_t0) * 1000) now = datetime.now(UTC) state.last_synced_at = now # Prefer the runner's REAL vendor op ("create"/"update") over # ``was_new`` (which only means "new to Helix's sync ledger"). # An already-adopted NetSuite record's first Helix sweep is an # UPDATE (PATCH eid:工号), but was_new would mislabel it # "已创建" and alarm the admin into thinking a duplicate record # was made. Fall back to was_new when the runner reports no op # (navan / scim / feishu / google). _op = outcome.detail.get("op") if isinstance(outcome.detail, dict) else None did_create = (_op == "create") if _op in ("create", "update") else was_new if outcome.status == "success": state.last_status = "success" state.last_payload_hash = new_hash if outcome.external_id: state.external_id = outcome.external_id state.retry_count = 0 # Slot-degraded success (resilient writes): the record # landed but the runner had to DROP an un-appliable slot (a # dept the vendor rejected, a bad mobile, a leader/HRBP the # vendor 404'd). That is NOT a fully-verified push. Force the # NEXT sweep to re-attempt the dropped slot by clearing # ``last_verified_at`` (``None`` ⇒ drift-stale ⇒ not # hash-skipped) — otherwise a row cleanly verified yesterday # would hash-skip today's degradation and freeze the dropped # field for the whole 7-10 day drift window with a green # state. Preserve the degradation note on the row so the # state tab shows WHICH field didn't land. A fully-clean # success resets the clock + clears the error. _dropped_slots = [ s for s in (outcome.slot_outcomes or []) if isinstance(s, dict) and s.get("status") in ("dropped", "drift", "failed") ] if _dropped_slots: state.last_verified_at = None state.last_error = _safe_error( outcome.error or "partial: " + ", ".join( str(s.get("slot") or "?") for s in _dropped_slots ) + " not applied", ) else: state.last_verified_at = now state.last_error = None if did_create: summary.created += 1 else: summary.updated += 1 # Enqueue the "your account was auto-created because # you manage a foreign report" notification once, on # first successful create of an auto-included row. # ``notified_at`` is set at enqueue time (not at # delivery) — the retry worker owns delivery from # here on, and we don't want a transient Feishu # outage to fire repeat notifications every sweep # until delivery succeeds. if ( state.auto_included_reason == "manager_of_foreign" and state.notified_at is None ): try: await self._enqueue_auto_include_notification( target=target, manager_person=person, ) state.notified_at = now summary.auto_included_notified += 1 except Exception: # noqa: BLE001 # Don't fail the whole sweep on a notification # row-write hiccup — the upsert itself # succeeded. Log and move on; next sweep will # retry the notification because notified_at # is still NULL. logger.exception( "auto_include notification enqueue failed " "for target=%s person=%s", target.slug, person.work_email, ) elif outcome.status == "skipped": state.last_status = "skipped" state.last_payload_hash = new_hash state.last_verified_at = now state.last_error = _safe_error(outcome.error) summary.skipped += 1 # Reaching this branch means the runner was CALLED and the # vendor answered — every runner's ``skipped`` return sits # downstream of a round-trip (google_schema after its GET, # zoom after finding the user active, scim after a 400/409 # the vendor itself produced). Unlike the payload-hash # short-circuit above (zero HTTP), that is proof the target # is up, so the run-status split may count it. See # ``_sweep``'s status computation. summary.verified_skipped += 1 elif outcome.status == "excluded": # The vendor refused this person by policy, not by error # (e.g. NetSuite's Administrator-record lock — those IT # admins are maintained manually, DECISIONS.md 2026-06-24). # Count as excluded, never as failed: a full sweep must not # go red over the handful of admins, and retry_count stays # 0 so the retry worker doesn't churn on an unreachable row. state.last_status = "excluded" state.last_error = None state.retry_count = 0 # Record the hash so the unchanged-payload short-circuit # re-affirms this row as excluded next sweep instead of # re-issuing the same PATCH (admin-locked records return # the identical 400 every time — pure churn). A genuine # payload change re-attempts, which is the right behaviour. state.last_payload_hash = new_hash state.last_verified_at = now if outcome.external_id and not state.external_id: state.external_id = outcome.external_id summary.excluded += 1 elif outcome.status == "blocked": # The vendor rejected this person for a NON-transient, # data-dependent reason (missing required field, unmapped # code, resigned vendor account, dept not in the active org # tree). Unlike ``failed`` (retry every sweep, burn a vendor # call, keep the run red), a blocked row records its hash so # the unchanged-payload short-circuit re-affirms it cheaply # WITHOUT a vendor call, retry_count stays 0 (no churn), and # it auto-retries the moment the rendered payload changes # (i.e. the data/config is fixed). The run stays GREEN — a # handful of people awaiting a data fix is not a sync # failure — while the backlog is surfaced (state = blocked + # actionable last_error) and alerted once, not every sweep. state.last_status = "blocked" state.last_error = _safe_error(outcome.error) state.retry_count = 0 state.last_payload_hash = new_hash state.last_verified_at = now if outcome.external_id and not state.external_id: state.external_id = outcome.external_id summary.blocked += 1 else: state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 # A failed outcome that still reports an external_id is # the runner saying "I know who this person is on the # vendor side" (e.g. huilianyi's resigned-row guard, # which addresses by 工号 but refuses to mutate). # Persist it when we have nothing — otherwise the # later out-of-scope deactivate path dead-ends on # external_id=NULL with a "run a full sweep" hint that # a full sweep can never satisfy (out-of-scope people # are never upserted again). if outcome.external_id and not state.external_id: state.external_id = outcome.external_id summary.failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{person.work_email}: {outcome.error or 'unknown'}" ) # Observability: prefer the ACTUAL wire body sent to the vendor # (final shape incl department_ids/orders/e164 mobile) and the # vendor's response/error over the pre-render payload + null that # made 41041-class failures undiagnosable. A runner that adopts # the capture protocol exposes ``request_body`` / ``response`` on # the outcome (or, transitionally, ``_last_write_*`` on itself); # fall back to the rendered payload + outcome.detail for no-write # paths (skips) and runners not yet migrated. _wire_req = outcome.request_body or getattr( runner, "_last_write_request", None, ) _wire_resp = outcome.response or getattr( runner, "_last_write_response", None, ) # Fold per-slot outcomes (resilient-write degradation: a dept / # mobile / field the vendor rejected and we dropped so the rest # could land) into the ledger's response so field-level truth is # visible instead of one opaque row status. if outcome.slot_outcomes: _base = dict(_wire_resp) if isinstance(_wire_resp, dict) else {} _base["slot_outcomes"] = outcome.slot_outcomes _wire_resp = _base self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome=( ("created" if did_create else "updated") if outcome.status == "success" else outcome.status ), subject_display=person.work_email, person_id=person.id, external_key=outcome.external_id or state.external_id, request_payload=_wire_req if _wire_req is not None else payload, response_detail=( _wire_resp if _wire_resp is not None else outcome.detail ), error=outcome.error, duration_ms=push_duration_ms, ) processed_in_batch += 1 await _maybe_commit_batch() # ── Out-of-scope sweep ──────────────────────────────────────── # People previously synced who no longer match the filter. # Apply on_offboard policy. # # Auto-included rows (e.g. CN managers pulled in because they # had a foreign report) are kept in-scope by design even after the # original reason lapses — "if HR moves the foreign report, we # still don't want to deactivate the manager's Navan account that's # been live for months." BUT that exemption holds ONLY WHILE THE # PERSON IS STILL EMPLOYED. A LEFT auto-included manager must still # be recovered: exempting them unconditionally meant a departed # manager's Navan / SCIM account (travel booking + corporate card) # could stay ACTIVE forever — the nightly reconcile never touched # them (auto-included) and the one-shot offboard event was their # ONLY teardown, so a single dropped/failed event = a permanent # open account with a green dashboard (2026-07 zero-trust review). # So: exempt only if the person is present AND active; a leaver (or # a person we can't resolve as active) falls through to the policy. # ``deactivated``/``deleted``/``excluded`` stay terminal — once the # account is shut the goal holds (re-counting historical leavers # here saturated the mass-action brake at ~10% cumulative attrition). def _auto_include_still_exempt(s: IdentitySyncState) -> bool: if not s.auto_included_reason: return False p = person_by_id.get(s.person_id) return p is not None and p.employment_status == "active" out_of_scope = [ s for s in state_rows if s.person_id not in in_scope_ids and s.last_status not in ("excluded", "deleted", "deactivated") and not _auto_include_still_exempt(s) ] # push-only mode: don't deactivate anyone merely outside THIS # run's scope. Emptying the list makes the brake + deactivation # loop below no-op, so a scoped manual/test sync only ever # creates/updates the in-scope people and touches nobody else. if push_only: out_of_scope = [] # Pre-launch P2 fix (Round 1, SEC-004): mass-action brake. # If a target's filter is restricted (or its mapping is # corrupted) such that nearly every previously-in-scope person # falls out, refuse to drive the destructive policy. Mirrors # the HRIS `confirm_batch` brake. Threshold: # * out_of_scope > max(25, 10% of (in_scope + out_of_scope)) # Skip the brake for ``policy == "keep"`` (the sweep only # re-marks state rows ``excluded`` and never calls vendor # APIs). For ``deactivate`` / ``delete`` the brake fires. policy_for_brake = (target.on_offboard or "deactivate").lower() # Brake math counts only FRESH fallout. Rows that already # failed ≥3 attempts are stuck on a surfaced, human-owned # condition (e.g. vendor-side block) — re-counting them every # sweep would saturate the brake and block legitimate new # offboards behind an old backlog. They still get retried. # # Auto-include-lapsed rows (``auto_included_reason`` set) are also # excluded from the brake TRIGGER — but NOT from the teardown loop # below. By construction (see ``_auto_include_still_exempt``) an # auto-included row only reaches ``out_of_scope`` once the person has # LEFT; deactivating a departed employee's account is always-safe and # is NOT the signal the brake exists to catch (a corrupted filter / # mapping dropping ACTIVE people). Counting them would let a one-time # historical backlog — every manager auto-included-then-departed while # their account stayed live, newly swept in when the exemption # narrowed to active-only (2026-07 zero-trust review) — trip the brake # on the FIRST post-deploy sweep. And because the brake path never # bumps ``retry_count``, that trip would recur every sweep, PERMANENTLY # blocking all new leaver teardown for the target until a manual # override. A genuine mass-leaver event still trips the brake via the # normal (non-auto-included) population. brake_population = [ s for s in out_of_scope if (s.retry_count or 0) < 3 and not s.auto_included_reason ] if policy_for_brake in ("deactivate", "delete") and out_of_scope: in_scope_count = len(in_scope_ids) total_known = in_scope_count + len(brake_population) threshold = max(25, int(total_known * 0.10)) # Round 2 P2 fix (LG2-004): also fire on a 100%-drop at # any non-trivial absolute count. The original # max(25, 10%) floor defeats the percentage check at the # boundary — for a 25-person target, threshold=25 and # ``out_of_scope > 25`` is False at exactly 25, so the # entire previously-known population disappears without # tripping the brake. Total drop is the strongest # possible signal of misconfiguration. full_drop = ( in_scope_count == 0 and len(brake_population) >= 5 ) if user_mass_action_brake_required( out_count=len(brake_population), in_scope_count=in_scope_count, ): guard = await DestructivePlanService(self.db).guard_or_record( target=target, scope="users", action=policy_for_brake, items=[ {"state_id": str(state.id), "subject_key": str(state.person_id), "external_id": state.external_id} for state in out_of_scope ], run_id=run_id, confirmation_token_hashes=confirmation_token_hashes or [], actor_user_id=actor_user_id, trace_id=trace_id, ) if not guard.authorized: logger.error( "identity_sync mass-action brake: target=%s policy=%s " "out_of_scope=%d threshold=%d in_scope=%d — refusing " "to apply on_offboard; plan_id=%s reason=%s", target.slug, policy_for_brake, len(out_of_scope), threshold, in_scope_count, guard.plan_id, guard.reason, ) # Persist in-scope mutations accumulated before bailing. if processed_in_batch > 0: await self.db.commit() processed_in_batch = 0 summary.errors.append( f"mass-action brake: plan_id={guard.plan_id} " f"fingerprint={guard.fingerprint} reason={guard.reason}; " f"out_of_scope={len(out_of_scope)} exceeds " f"threshold={threshold} for policy={policy_for_brake}." ) for state in out_of_scope: state.last_status = "failed" state.last_error = _safe_error( f"mass-action brake: plan={guard.plan_id} skipped " f"(out_of_scope={len(out_of_scope)}, " f"threshold={threshold}, in_scope={in_scope_count}, " f"full_drop={full_drop})" ) summary.failed += len(out_of_scope) if processed_in_batch == 0: await self.db.commit() return summary, "failed" logger.warning( "identity_sync destructive plan consumed: target=%s " "scope=users plan_id=%s fingerprint=%s", target.slug, guard.plan_id, guard.fingerprint, ) for state in out_of_scope: # Counter bumped at the top so the ``continue`` branches # below still trigger periodic commits — every iteration # mutates state in some way, so every iteration counts. processed_in_batch += 1 person = person_by_id.get(state.person_id) if person is None or not person.work_email: # Person left Helix's active population (hard-deleted, # soft-deleted, or HRIS-purged) — apply on_offboard using # whatever email we have on the state row would require # denormalising it; for now mark excluded so the sweep # stops trying. # # ``person_id=None`` on the ledger row is LOAD-BEARING, not # cosmetic: if the person was *physically* hard-deleted # from identity_persons (a manual prod cleanup that # disabled FK checks and so orphaned this ON DELETE CASCADE # state row instead of removing it), inserting the run item # with ``person_id=state.person_id`` fails the # ``fk_identity_sync_run_items_person`` FK (1452) and takes # down the entire sweep — and, via the dispatcher's # raise-on-failure, the whole ``identity_sync.dispatch`` # tick. The person is gone, so drop the FK link (exactly # what ON DELETE SET NULL would have done); the identity is # still recorded in ``subject_display`` + ``external_key``. # Record once (on the transition into ``excluded``) so a # steady-state orphan doesn't append a ledger row every # sweep — mirrors the keep / unsupported branches below. already_excluded = state.last_status == "excluded" state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) if not already_excluded: summary.excluded += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=str(state.person_id), person_id=None, external_key=state.external_id, response_detail={ "reason": "person no longer in Helix", }, ) await _maybe_commit_batch() continue # Non-employee gate is categorically different from offboarding. # A person rejected by ``employee_no_pattern`` (hand-created # service / admin row like TimeAdmin / 刘寅) was NEVER an HRIS # employee, so they must be ``excluded`` (kept, no vendor call) # regardless of on_offboard — routing them through the # deactivate/delete path would disable a live downstream account # we were merely asked to ignore (the SCIM targets are # on_offboard=deactivate and these rows carry real external_ids). if fails_employee_no_gate(person.employee_no, filter_cfg): if state.last_status != "excluded": state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) summary.excluded += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": ( "non-employee (employee_no gate); " "kept, not deactivated" ), }, ) await _maybe_commit_batch() continue policy = (target.on_offboard or "deactivate").lower() if policy == "keep": if state.last_status != "excluded": state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) summary.excluded += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": "left scope; on_offboard=keep", }, ) await _maybe_commit_batch() continue # Capability check — runners that don't support the # requested operation collapse to ``excluded`` rather # than raising NotImplementedError mid-sweep. This is # the production-grade contract: e.g. google_schema sets # ``supports_deactivate=False`` because Google account # suspension is a different domain (account_provisioner); # treating "deactivate the dept-schema projection" as # "stop projecting fields onto this person" is the # correct semantic and matches admin intent. requested_supported = ( (policy == "deactivate" and runner.supports_deactivate) or (policy == "delete" and runner.supports_delete) ) if not requested_supported: # Mirror the ``keep`` branch above: only count + stamp on # an actual status change (else a steady-state already- # excluded row re-inflates ``summary.excluded`` every # sweep), and join the commit cadence so a long run of # unsupported-offboard rows isn't deferred to the final # commit (crash-resilience parity). if state.last_status != "excluded": state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) summary.excluded += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": f"runner does not support {policy!r}", }, ) await _maybe_commit_batch() continue # Parity with the upsert short-circuit: a row that is ALREADY # ``excluded`` at this point was vendor-excluded by a prior # offboard attempt (the keep / unsupported-policy branches above # ``continue`` before here, so this can only be an NS # Administrator-record lock). Re-issuing the deactivate every # sweep just re-collects the same 400 — those admins are # deactivated manually. Stay excluded with no vendor call and no # re-count (steady-state), matching the keep/unsupported guards. if state.last_status == "excluded": await _maybe_commit_batch() continue # ── Leaver notice-period guard (#1735) ─────────────────── # HRIS-confirm sets ``leaver_pending`` immediately for a # future-dated leaver (hr/service.py), which drops them out of # the standard active/on_leave filter — but their last working # day has NOT arrived. Deactivating now locks a still-working # employee out of the vendor (Feishu freeze / Navan stop-号 / # SCIM deactivate) days-to-weeks early — the 2026-05-28 Thalie # Zhuang incident. Mirror offboarding's ``_leaver_revoke_not_ # before`` and ``scheduler_handler``'s account_backstop 'left'- # only invariant: defer until BOTH leaver_pending→left AND the # employee-local last working day has passed. ``left`` employees # are NOT deferred (their day is past by definition and # offboarding owns their teardown); only ``leaver_pending`` # within the notice window is skipped, to be picked up by the # next sweep after the status flip. if ( person.employment_status == "leaver_pending" and not _leaver_last_working_day_passed(person) ): await _maybe_commit_batch() continue offboard_t0 = time.monotonic() try: if policy == "delete": outcome = await runner.delete( person_email=person.work_email, prior_external_id=state.external_id, ) else: outcome = await runner.deactivate( person_email=person.work_email, prior_external_id=state.external_id, ) except AppException as e: from app.approval.feishu_client import ( is_systemic_feishu_failure, ) if is_systemic_feishu_failure(e): if processed_in_batch > 0: await self.db.commit() raise logger.exception( "identity_sync.%s %s for %s failed", policy, target.slug, person.work_email, ) outcome = PersonOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) except Exception as e: # noqa: BLE001 logger.exception( "identity_sync.%s %s for %s failed", policy, target.slug, person.work_email, ) outcome = PersonOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) now = datetime.now(UTC) state.last_synced_at = now # Captured BEFORE the deleted branch clears it — the audit # row for a destructive delete is exactly where the vendor # key must survive. prior_external_id = state.external_id if outcome.status == "deactivated": state.last_status = "deactivated" state.last_error = None summary.deactivated += 1 elif outcome.status == "deleted": state.last_status = "deleted" state.last_error = None state.external_id = None summary.deleted += 1 elif outcome.status == "excluded": # Vendor refused by policy (NS Administrator-record lock): # the IT admin is deactivated manually. Excluded, not # failed — no retry-churn, no red sweep. Surfaced as an # excluded run item so an operator can action it. state.last_status = "excluded" state.last_error = None state.retry_count = 0 summary.excluded += 1 else: state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{person.work_email} ({policy}): " f"{outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome=( outcome.status if outcome.status in ("deactivated", "deleted", "excluded") else "failed" ), subject_display=person.work_email, person_id=person.id, external_key=prior_external_id, response_detail=outcome.detail, error=outcome.error, duration_ms=int((time.monotonic() - offboard_t0) * 1000), ) # ``processed_in_batch`` was bumped at the top of this # iteration; only the maybe-commit call belongs at the # tail (matches the early-continue paths above which do # the same). await _maybe_commit_batch() # Final flush — commit any unpersisted state mutations from the # tail of either loop (count < BATCH_COMMIT_SIZE since the # last commit). Without this, a clean sweep that processed # 26 in-scope and 0 out-of-scope persons would commit at row # 25 and lose row 26 if anything in the post-loop summary # computation raised. if processed_in_batch > 0: await self.db.commit() processed_in_batch = 0 # Decide overall run status. # # ``partial`` means "the target is UP, but some people didn't make # it"; ``failed`` means "this target is DOWN". Only the latter # reaches the dispatcher's raise-on-failure gate (``ran_failed`` in # :meth:`run_due_targets`) and therefore the scheduled-job failure # notifier — so the split decides whether a dead target pages IT or # sits silent behind a yellow run row. # # The only honest evidence that the target is up is a vendor call # that GOT THROUGH. The raw ``skipped`` counter is not that # evidence: # the dominant skip is the payload-hash short-circuit, which never # touches the vendor at all (``blocked`` re-affirmations and # permanent domain-rejects are hash-persisted the same way). # Counting ALL skips as proof of life is what let a total credential # outage read as a healthy-ish ``partial``: # # 2026-08-14, target ``aws-ai`` — the AWS IAM Identity Center SCIM # token was revoked, so EVERY vendor call 401'd. The sweep landed # created=0 updated=0 deactivated=0 deleted=0, failed=127 … # skipped=282, and those 282 hash-skips (zero HTTP requests # between them) alone downgraded the run to ``partial``. The # dispatcher saw no ``failed`` run, reported an ok tick, and # nobody was paged — exactly the "a revoked vendor credential # could silently kill the nightly leaver-lockout backstop for # weeks" scenario that gate was built to prevent. The outage was # noticed only because the 03:00 vendor-snapshot cron happened to # fail on the same dead token an hour later. # # So: failures with not one vendor call that got through = target down. # # "Got through" is NOT the same as "wrote something", and reading it # that way produced the mirror-image false verdict: # # 2026-09-07, target ``google-dept`` — a steady-state day. 122 of # the in-scope people were vendor-re-verified (GET user → compare # customSchemas → already correct → ``skipped``), the other 288 # hash-skipped, one person's token refresh hit a transient # ``unauthorized_client``. created=updated=deactivated=deleted=0 # because a healthy read-then-diff runner in steady state writes # NOTHING, so a write-only proof of life read the run as # ``failed`` = "target down" and paged IT at P2 — for a target # that had just completed 122 successful Google API calls. The # next tick swept clean. A ``skipped`` the RUNNER returned is a # vendor round-trip the vendor answered; that is evidence, and # ``verified_skipped`` counts exactly those (never hash-skips — # which is why the aws-ai run above still lands ``failed``: its # 282 skips were all hash-skips and it recorded zero of these). # # Deliberately NOT extended to ``blocked`` / ``excluded``, though a # runner-returned ``blocked`` is also an answered round-trip: those # counters mix runner returns with the hash short-circuit's cheap # re-affirmation (which issues no HTTP), so counting them raw would # reopen the aws-ai hole and splitting them needs its own counter. # No production run in the 90 days to 2026-09-07 had the shape that # would require it (a ``failed`` run with blocked>0 and no writes), # so it stays unbuilt rather than untested. vendor_reachable = ( summary.created + summary.updated + summary.deactivated + summary.deleted + summary.verified_skipped ) if summary.failed == 0: status = "ok" elif vendor_reachable: status = "partial" else: status = "failed" return summary, status # ── feishu_contact Phase A — dept sweep ───────────────────────── @staticmethod def _topological_sort_depts( nodes: list[IdentityOrgNode], ) -> list[IdentityOrgNode]: """Return nodes in parent-before-child order. Falls back to original order for roots / nodes whose parent isn't in the snapshot (cycle-broken or detached subtrees) — better to push something than to spin forever on a dependency we can't resolve. """ by_code: dict[str, IdentityOrgNode] = { n.org_node_source_id: n for n in nodes } # Defensive log when HRIS hands us multiple rows under the # same dept_code (cycle in a migration restore, dual-write # race, etc.). The dict comprehension silently keeps the LAST # row; under topological pressure we want this to be loud so # the operator notices and reconciles HRIS. if len(by_code) != len(nodes): logger.warning( "feishu_dept topo-sort: duplicate org_node_source_id " "values in input (%d nodes → %d unique codes); only the " "last row per code will be pushed. Investigate " "identity_org_nodes for HRIS sync collisions.", len(nodes), len(by_code), ) ordered: list[IdentityOrgNode] = [] placed: set[str] = set() # Iterate breadth-first by ``org_level`` if available; falls # back to repeated passes for nodes with unset level. remaining = list(nodes) # Bound the loop — at most one iteration per node — so a # broken graph never spins forever. 1 + len(nodes) iterations # is the worst case for a strict chain. for _ in range(len(nodes) + 1): if not remaining: break progress = False still: list[IdentityOrgNode] = [] for n in remaining: parent = n.parent_org_node_source_id if not parent or parent not in by_code or parent in placed: ordered.append(n) placed.add(n.org_node_source_id) progress = True else: still.append(n) remaining = still if not progress: # Cycle or detached subtree — flush the rest as-is. ordered.extend(remaining) break return ordered @staticmethod def _extract_office_names_from_mapping( mapping_config: dict[str, Any] | None, ) -> frozenset[str]: """Scan ``mapping_config.rules`` for the ``office_dept_name`` rule and collect every value of its ``value_map`` — these are the curated Feishu office-dept names this target is configured to manage (e.g. "Tokyo Office", "SG Office", …). Returns an empty set if no office rule is configured, in which case the office dimension is disabled for this target. """ rules = (mapping_config or {}).get("rules") or [] out: set[str] = set() for r in rules: if not isinstance(r, dict): continue if r.get("target_field") != "office_dept_name": continue vmap = r.get("value_map") or {} if isinstance(vmap, dict): for v in vmap.values(): if isinstance(v, str) and v: out.add(v) return frozenset(out) async def _person_dept_chain( self, person_id: UUID, ) -> set[str]: """A person's primary-membership dept + its full ancestor chain (org_node source_ids), for the targeted single-person Phase A. Kind-agnostic (used by the feishu_contact AND huilianyi single-person ensure) — the org tree it walks is the shared HRIS one, not a per-vendor structure. Returns the leaf primary-membership dept code plus every parent up the active org tree. Empty set when the person has no primary membership (nothing to ensure — Phase B falls back to the office node / root as before). Ancestors that are inactive/absent stop the walk; Phase A intersects this set with the active node list anyway, so a stale code never forces a spurious create. """ membership = (await self.db.execute( select(IdentityOrgMembership).where( IdentityOrgMembership.person_id == person_id, IdentityOrgMembership.is_primary.is_(True), IdentityOrgMembership.deleted_at.is_(None), ) )).scalars().first() if membership is None or not membership.org_node_source_id: return set() parent_rows = (await self.db.execute( select( IdentityOrgNode.org_node_source_id, IdentityOrgNode.parent_org_node_source_id, ).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).all() parent_of: dict[str, str | None] = { sid: pid for sid, pid in parent_rows } chain: set[str] = set() cur: str | None = membership.org_node_source_id guard = 0 while cur and cur not in chain and guard < 100: chain.add(cur) cur = parent_of.get(cur) guard += 1 return chain async def _run_feishu_dept_phase( self, *, target: IdentitySyncTarget, run_id: UUID | None = None, push_only: bool = False, restrict_dept_codes: set[str] | None = None, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> RunSummary: """Phase A of ``feishu_contact`` — reconcile the HRIS org tree against the live Feishu dept tree. ``push_only`` (manual / test syncs): adopt/create/update the in-scope depts ONLY; NEVER soft-delete a dept merely because it's outside THIS run's scope (e.g. another group's dept adopted by an earlier test sync). Retiring a genuinely-removed dept is the scheduled full-reconcile's job. ``restrict_dept_codes`` (single-person event-hook path): adopt/ create ONLY these dept codes (a new hire's primary-membership dept + its ancestor chain) instead of the full in-scope set. Implies ``push_only`` (a targeted per-person ensure must NEVER soft-delete anything). This is what guarantees a new hire's business department exists + is adopted on Feishu BEFORE Phase B pushes the user, so the user is placed in their real department instead of only the appended office node. Walks active :class:`IdentityOrgNode` rows in topological order (parent-before-child), passes each to :meth:`FeishuDepartmentRunner.upsert_dept` (which adopts an existing same-name Feishu dept by name → PATCH, or POST creates a new one and captures the auto-assigned ``od-XXX``). Out-of-scope adopted depts get soft-deleted, capped by a mass-action brake at 10 depts/sweep. Returns a :class:`RunSummary` with ONLY the ``dept_*`` counters populated (the orchestrator merges them into the run row's full summary). Phase A NEVER raises into the caller — any unhandled exception is logged + recorded in ``summary.errors`` + ``dept_failed`` is bumped, so a bad dept run never blocks Phase B from running on the up-to-date dept_state map. """ from app.approval.feishu_client import FeishuApprovalClient from app.domains.identity_sync.runners.feishu_department import ( FeishuDepartmentRunner, ) summary = RunSummary() # Resolve the Feishu integration once (shared with Phase B). if target.integration_id is None: summary.errors.append( "feishu_contact dept-phase: integration_id is null; " "configure the feishu integration on the target.", ) summary.dept_failed = 1 return summary integ = await self.db.get(Integration, target.integration_id) if integ is None: summary.errors.append( f"feishu_contact dept-phase: integration " f"{target.integration_id} not found.", ) summary.dept_failed = 1 return summary try: resolved = await _resolve_integration(self.db, integ) except Exception as e: # noqa: BLE001 summary.errors.append( f"feishu_contact dept-phase: integration resolve " f"failed: {type(e).__name__}: {e!s:.200}", ) summary.dept_failed = 1 return summary app_id = resolved.config.get("app_id") app_secret = resolved.config.get("app_secret") if not app_id or not app_secret: summary.errors.append( "feishu_contact dept-phase: app_id/app_secret missing.", ) summary.dept_failed = 1 return summary client = FeishuApprovalClient(app_id=app_id, app_secret=app_secret) helix_dept_to_external = await self._load_helix_dept_index( target_id=target.id, ) runner = FeishuDepartmentRunner( client=client, helix_dept_to_external=helix_dept_to_external, ) try: await runner.prime_catalog() except Exception as e: # noqa: BLE001 # FAIL-LOUD, do NOT continue POST-only. prime_catalog builds the # adopt-match index (existing Feishu depts by name/parent); a # partial/failed catalog means every not-yet-adopted dept misses # the adopt-match and POSTs a DUPLICATE parallel dept whose od is # then permanently adopted (the #1757 split-dept class). The # runner's prime_catalog is deliberately fail-loud (raises on a # truncated/cyclic walk) precisely so this can't happen silently; # honour that contract here — abort Phase A (record + count), and # let Phase B still run on the EXISTING dept_state map (the # already-designed degradation), rather than creating duplicates. logger.exception( "feishu_contact dept-phase: prime_catalog failed — ABORTING " "Phase A (NOT running POST-only, which would create duplicate " "departments); Phase B proceeds on the existing adoption map.", ) summary.errors.append( "dept-phase aborted: prime_catalog failed " f"({type(e).__name__}: {e!s:.200}) — skipped dept " "create/update this run to avoid duplicate departments; " "existing adoptions unaffected.", ) summary.dept_failed = 1 return summary # Load HRIS dept tree. async with best_effort( self.db, label="identity_sync: dept node scan", log=logger, ) as nodes3_step: nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() if not nodes3_step.ok: summary.errors.append( f"feishu_contact dept-phase: load IdentityOrgNode " f"failed: {nodes3_step.reason(max_len=200)}", ) summary.dept_failed = 1 return summary # ── Scope to depts that actually hold an in-scope member ───── # Skip hollow shells + filtered-out BUs so Phase A never POSTs # empty / cross-language-duplicate departments to Feishu (the # 2026-06-08 prod audit found 10 such empty English nodes that # collide semantically with existing Chinese Feishu depts). if restrict_dept_codes is not None: # Targeted per-person ensure: scope is exactly the caller's # dept chain (leaf + ancestors), and we NEVER soft-delete. scoped_ids = set(restrict_dept_codes) push_only = True else: try: scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(nodes), ) except Exception as e: # noqa: BLE001 # Fail safe-OPEN would re-introduce the empty-shell problem; # fail safe-CLOSED (skip nothing) would too. Surface the # error + skip Phase A creates entirely this run rather than # guess — Phase B still runs on the existing dept_state map. summary.errors.append( f"feishu_contact dept-phase: scope computation failed: " f"{type(e).__name__}: {e!s:.200}", ) summary.dept_failed = 1 return summary total_active = len(nodes) # Capture the full active-HRIS dept code set BEFORE narrowing to # member-bearing depts. Soft-delete (below) must key off "this dept is # gone / inactive in HRIS", NOT "this dept transiently has no in-scope # member" — a team whose last member just offboarded (or all went on # leave) is still a real, active department, and we must never # destructively delete it on the downstream system for that. active_node_codes: set[str] = {n.org_node_source_id for n in nodes} nodes = [n for n in nodes if n.org_node_source_id in scoped_ids] summary.dept_skipped = total_active - len(nodes) nodes = sorted( nodes, key=lambda n: (n.org_level or 0, n.org_node_source_id), ) sorted_nodes = self._topological_sort_depts(list(nodes)) # Existing state rows for this target. state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() state_by_code: dict[str, IdentitySyncDepartmentState] = { s.dept_code: s for s in state_rows } summary.dept_in_scope = len(sorted_nodes) batch_commit_size = 25 processed_in_batch = 0 async def _maybe_commit_batch() -> None: nonlocal processed_in_batch if processed_in_batch >= batch_commit_size: await self.db.commit() processed_in_batch = 0 # ── In-scope: adopt/PATCH or POST-create each HRIS dept. ──── for node in sorted_nodes: dept_code = node.org_node_source_id state = state_by_code.get(dept_code) if state is None: state = IdentitySyncDepartmentState( id=uuid.uuid4(), target_id=target.id, dept_code=dept_code, last_status="pending", ) self.db.add(state) state_by_code[dept_code] = state # Skip via payload hash (cheap dedup). Hash inputs are the # raw IdentityOrgNode fields the runner actually pushes. phase_a_payload = { "dept_code": dept_code, "dept_name": node.org_node_name or "", "parent_dept_code": node.parent_org_node_source_id or "", "dept_order": node.org_level if node.org_level is not None else None, } new_hash = hash_payload(phase_a_payload) # Drift re-verification (mirrors the user phase): a dept not # vendor-verified within _DRIFT_REVERIFY_INTERVAL is re-pushed # even on an unchanged hash, so a rename / re-parent done on the # vendor side outside Helix is re-asserted (idempotent upsert). _dept_verify_stale = force_reverify or _drift_reverify_stale( state.last_verified_at, jitter_key=dept_code, ) if ( state.last_payload_hash == new_hash and state.last_status in ("success", "skipped") and not _dept_verify_stale ): state.last_status = "skipped" state.last_synced_at = datetime.now(UTC) state.last_error = None processed_in_batch += 1 await _maybe_commit_batch() continue # SAVEPOINT-isolate (2026-08-06): a DB-level failure inside the # runner used to deactivate the transaction, so the per-dept # ``state.*`` writes below, EVERY later dept in this sweep, and # the systemic branch's own ``self.db.commit()`` all raised. The # savepoint also makes that deliberate mid-sweep commit reliable. async with best_effort( self.db, label=f"identity_sync: feishu dept upsert {dept_code}", log=logger, ) as dept4_step: outcome: DeptOutcome = await runner.upsert_dept( dept_code=dept_code, dept_name=node.org_node_name or "", parent_dept_code=node.parent_org_node_source_id or None, leader_email=None, # HRIS doesn't carry dept leader today dept_order=node.org_level, prior_external_id=state.external_id, ) if not dept4_step.ok: # NB: not named ``e`` — Python deletes except-bound names at # block exit and this function binds ``e`` in a later loop, # which mypy reports as reading a deleted variable. dept_err = dept4_step.error if isinstance(dept_err, AppException): from app.approval.feishu_client import ( is_systemic_feishu_failure, ) if is_systemic_feishu_failure(dept_err): # Deliberate whole-sweep abort: persist what landed, # then propagate. Reached with a usable session now # that the savepoint rolled the failure back. if processed_in_batch > 0: await self.db.commit() raise dept_err logger.error( "feishu_contact dept-phase: upsert_dept %s failed: %s", dept_code, dept4_step.reason(), ) # Carry the prior external_id through so the absent-clear # logic below ONLY fires on the runner's deliberate # ``external_id=None`` signal (dept-absent codes). Bare # transient failures (5xx, network, DB blip) keep the # adoption mapping — otherwise a flaky Feishu response # would wipe the od and force a duplicate POST next sweep. outcome = DeptOutcome( status="failed", external_id=state.external_id, error=dept4_step.reason() or "unknown", ) now = datetime.now(UTC) state.last_synced_at = now if outcome.status == "success": was_new = state.last_payload_hash is None state.last_status = "success" state.last_payload_hash = new_hash state.last_verified_at = now if outcome.external_id: state.external_id = outcome.external_id state.last_error = _safe_error(outcome.error) state.retry_count = 0 if was_new: summary.dept_created += 1 else: summary.dept_updated += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="created" if was_new else "updated", subject_display=node.org_node_name or dept_code, external_key=state.external_id or dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) else: self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="failed", subject_display=node.org_node_name or dept_code, external_key=state.external_id or dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 # Round-55: when the runner explicitly clears # ``outcome.external_id`` on a failed PATCH (the dept # disappeared on the Feishu side — _DEPT_ABSENT_OR_DONE # codes), we MUST clear ``state.external_id`` too. # Otherwise the next sweep reads the same dead od via # ``prior_external_id``, retries the same absent-PATCH, # same failure, forever. The runner's own docstring # promises "next sweep will re-adopt or POST" — that # promise only holds if the orchestrator drops the # stale external_id here. # # The orchestrator's own failure-outcome construction # (above) carries ``state.external_id`` through, so the # ``is None`` check here ONLY matches the runner's # deliberate clear, not a transient 5xx / DB blip / # generic AppException default. if outcome.external_id is None: state.external_id = None summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{dept_code}: {outcome.error or 'unknown'}" ) processed_in_batch += 1 await _maybe_commit_batch() # ── Out-of-scope: soft-delete previously adopted depts that # no longer have a matching HRIS row. Mass-action brake at 10. in_scope_codes: set[str] = {n.org_node_source_id for n in sorted_nodes} out_of_scope = [ s for s in state_rows if s.dept_code not in active_node_codes and s.last_status not in ("soft_deleted", "excluded") ] # push-only mode: never soft-delete a dept merely outside THIS # run's scope (e.g. another group's dept adopted by an earlier # test sync). Emptying the list no-ops the soft-delete loop below. if push_only: out_of_scope = [] dept_brake_threshold = 10 dept_guard = None if department_mass_action_brake_required(len(out_of_scope)): dept_guard = await DestructivePlanService(self.db).guard_or_record( target=target, scope="feishu_departments", action="soft_delete", items=[{"state_id": str(s.id), "subject_key": s.dept_code, "external_id": s.external_id} for s in out_of_scope], run_id=run_id, confirmation_token_hashes=confirmation_token_hashes or [], actor_user_id=actor_user_id, trace_id=trace_id, ) if dept_guard is not None and not dept_guard.authorized: logger.error( "feishu_contact dept-phase mass-action brake: target=%s " "out_of_scope=%d threshold=%d in_scope=%d — refusing " "to soft-delete.", target.slug, len(out_of_scope), dept_brake_threshold, len(in_scope_codes), ) for s in out_of_scope: s.last_status = "failed" s.last_error = _safe_error( "mass-action brake (dept): skipped", ) summary.dept_failed += len(out_of_scope) if len(summary.errors) < 50: summary.errors.append( f"mass-action brake (dept): plan_id={dept_guard.plan_id} " f"fingerprint={dept_guard.fingerprint} reason={dept_guard.reason}; " f"out_of_scope=" f"{len(out_of_scope)} exceeds threshold=" f"{dept_brake_threshold}", ) if processed_in_batch > 0: await self.db.commit() try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "feishu_contact dept-phase: runner.aclose() raised", exc_info=True, ) return summary for state in out_of_scope: processed_in_batch += 1 try: outcome = await runner.soft_delete( dept_code=state.dept_code, prior_external_id=state.external_id, ) except AppException as e: from app.approval.feishu_client import ( is_systemic_feishu_failure, ) if is_systemic_feishu_failure(e): if processed_in_batch > 0: await self.db.commit() raise logger.exception( "feishu_contact dept-phase: soft_delete %s failed", state.dept_code, ) outcome = DeptOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) except Exception as e: # noqa: BLE001 logger.exception( "feishu_contact dept-phase: soft_delete %s failed", state.dept_code, ) outcome = DeptOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) state.last_synced_at = datetime.now(UTC) if outcome.status == "soft_deleted": state.last_status = "soft_deleted" state.last_error = None summary.dept_soft_deleted += 1 else: state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{state.dept_code} (soft_delete): " f"{outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome=( "soft_deleted" if outcome.status == "soft_deleted" else "failed" ), subject_display=state.dept_code, external_key=state.external_id or state.dept_code, response_detail=outcome.detail, error=outcome.error, ) await _maybe_commit_batch() if processed_in_batch > 0: await self.db.commit() try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "feishu_contact dept-phase: runner.aclose() raised " "on success-path teardown", exc_info=True, ) return summary # ── scim group phase — dept tree → SCIM Groups (AWS IdC) ──────── async def _run_scim_group_phase( self, *, target: IdentitySyncTarget, runner: SyncRunner, run_id: UUID | None = None, ) -> RunSummary: """Mirror Authing's downstream group sync: one SCIM Group per in-scope HRIS dept, members = that dept's DIRECT in-scope members' SCIM user ids (from this target's own ``identity_sync_state`` rows — which is why this phase runs AFTER the user sweep). State lives in ``identity_sync_department_state`` keyed on the HRIS dept code; ``external_id`` stores the vendor's group id. Same contracts as the other dept-shaped phases: never raises into the caller, hash-skip on unchanged member sets, batch commits, consecutive-failure breaker. Out-of-scope groups are NEVER deleted (AWS permission-set assignments hang off them); their state rows flip to ``excluded`` and the group is left for the admin to retire manually. """ from app.domains.identity_sync.runners.scim import ScimRunner summary = RunSummary() if not isinstance(runner, ScimRunner): summary.errors.append( "scim group-phase: runner is not a ScimRunner " f"({type(runner).__name__}) — phase skipped.", ) summary.dept_failed = 1 return summary async with best_effort( self.db, label="identity_sync: dept node scan", log=logger, ) as nodes4_step: nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(nodes), ) if not nodes4_step.ok: summary.errors.append( f"scim group-phase: scope computation failed: " f"{nodes4_step.reason(max_len=200)}", ) summary.dept_failed = 1 return summary total_active = len(nodes) scoped_nodes = [ n for n in nodes if n.org_node_source_id in scoped_ids ] summary.dept_skipped = total_active - len(scoped_nodes) summary.dept_in_scope = len(scoped_nodes) # Disambiguated displayName per in-scope dept (single source of # truth, shared with the single-person event-path ensure so both # resolve to the SAME group name). resolved_group_name = self._resolve_scim_group_names( all_nodes=list(nodes), scoped_nodes=scoped_nodes, ) # Member map: dept code → [scim user ids] from this target's # own user-state rows (success/skipped = present downstream). async with best_effort( self.db, label="identity_sync: membership candidate scan", log=logger, ) as cand_step: candidates, _person_by_id, membership_by_person = ( await self._load_candidates() ) user_states = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() if not cand_step.ok: summary.errors.append( f"scim group-phase: member map load failed: " f"{cand_step.reason(max_len=200)}", ) summary.dept_failed = 1 return summary # Membership mirrors ACCOUNT EXISTENCE, not the latest push # attempt: a person whose update transiently failed still has # a live downstream account — stripping them from the group # (replace-all!) would revoke their permission-set access over # a vendor 500. Excluded rows (on_offboard=keep / unsupported # policy) are likewise untouched accounts → membership kept. # Only deactivated/deleted rows — accounts we ourselves shut — # leave the member map. scim_id_by_person: dict[UUID, str] = { st.person_id: st.external_id for st in user_states if st.external_id and st.last_status not in ("deactivated", "deleted") } members_by_dept: dict[str, list[str]] = {} for cand in candidates: sid = scim_id_by_person.get(cand.person_id) if not sid: continue dept_code = cand.org_node_source_id if not dept_code: continue members_by_dept.setdefault(dept_code, []).append(sid) state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() state_by_code: dict[str, IdentitySyncDepartmentState] = { st.dept_code: st for st in state_rows } batch_commit_size = 25 processed_in_batch = 0 consecutive_failures = 0 breaker_threshold = 5 async def _maybe_commit_batch() -> None: nonlocal processed_in_batch if processed_in_batch >= batch_commit_size: await self.db.commit() processed_in_batch = 0 def _fail( state: IdentitySyncDepartmentState, err: str, *, counts_for_breaker: bool = True, ) -> None: # ``counts_for_breaker=False`` for DATA-shaped failures # (duplicate displayName) — the breaker exists to stop a # dead-gateway grind, and several adjacent same-named # depts would otherwise abort the healthy remainder. nonlocal consecutive_failures if counts_for_breaker: consecutive_failures += 1 state.last_status = "failed" state.last_error = _safe_error(err) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append(err[:300]) for node_idx, node in enumerate( sorted(scoped_nodes, key=lambda n: (n.org_level or 0, n.org_node_source_id)) ): if consecutive_failures >= breaker_threshold: remaining = len(scoped_nodes) - node_idx summary.errors.append( f"scim group-phase circuit breaker: " f"{breaker_threshold} consecutive failures — " f"aborting; {remaining} group(s) NOT attempted " "this sweep." ) break dept_code = node.org_node_source_id # Disambiguated displayName (qualified with parent when the raw # name collides across business groups). Used everywhere below: # hash, find/create/replace, run-item label. dept_name = resolved_group_name.get( dept_code, node.org_node_name or dept_code, ) state = state_by_code.get(dept_code) if state is None: state = IdentitySyncDepartmentState( id=uuid.uuid4(), target_id=target.id, dept_code=dept_code, last_status="pending", ) self.db.add(state) state_by_code[dept_code] = state now = datetime.now(UTC) member_ids = sorted(set(members_by_dept.get(dept_code, []))) payload_hash = hash_payload({ "displayName": dept_name, "members": member_ids, }) if ( state.last_payload_hash == payload_hash and state.last_status in ("success", "skipped") and state.external_id ): state.last_status = "skipped" state.last_synced_at = now state.last_error = None continue group_item_payload = { "displayName": dept_name, "externalId": dept_code, "member_count": len(member_ids), } group_t0 = time.monotonic() try: group_id = state.external_id if not group_id: group_id = await runner.find_group_id( display_name=dept_name, external_code=dept_code, ) created = False if not group_id: group_id, err = await runner.create_group( display_name=dept_name, external_code=dept_code, ) if not group_id: _fail(state, f"group {dept_name!r}: {err}") self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="failed", subject_display=dept_name, external_key=dept_code, request_payload=group_item_payload, error=err, duration_ms=int((time.monotonic() - group_t0) * 1000), ) state.last_synced_at = now processed_in_batch += 1 await _maybe_commit_batch() continue created = True err = await runner.replace_group_members( group_id=group_id, member_ids=member_ids, display_name=dept_name, ) if err and err.startswith("name_patch_failed"): # Membership landed; only the label didn't. Record # without failing — but do NOT store the hash, so # the rename retries next sweep. consecutive_failures = 0 state.external_id = group_id state.last_status = "success" state.last_error = _safe_error(err) summary.dept_updated += 1 if len(summary.errors) < 50: summary.errors.append( f"group {dept_name!r}: {err[:250]}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="updated", subject_display=dept_name, external_key=group_id, request_payload=group_item_payload, error=err, duration_ms=int((time.monotonic() - group_t0) * 1000), ) elif err == "group_absent": # Group vanished server-side between lookup and # PATCH — drop the stale id; next sweep recreates. state.external_id = None _fail(state, ( f"group {dept_name!r} ({group_id}) absent on " "PATCH — stale id cleared, next sweep recreates." )) self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="failed", subject_display=dept_name, external_key=group_id, request_payload=group_item_payload, error=state.last_error, duration_ms=int((time.monotonic() - group_t0) * 1000), ) elif err: state.external_id = group_id _fail(state, f"group {dept_name!r}: {err}") self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="failed", subject_display=dept_name, external_key=group_id, request_payload=group_item_payload, error=err, duration_ms=int((time.monotonic() - group_t0) * 1000), ) else: consecutive_failures = 0 was_new = state.last_payload_hash is None state.external_id = group_id state.last_status = "success" state.last_payload_hash = payload_hash state.last_error = None state.retry_count = 0 if created or was_new: summary.dept_created += 1 else: summary.dept_updated += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="created" if (created or was_new) else "updated", subject_display=dept_name, external_key=group_id, request_payload=group_item_payload, duration_ms=int((time.monotonic() - group_t0) * 1000), ) except Exception as e: # noqa: BLE001 logger.exception( "scim group-phase: group %s failed", dept_code, ) _fail(state, ( f"group {dept_name!r}: {type(e).__name__}: {e!s:.300}" )) self._add_run_item( run_id=run_id, target_id=target.id, item_type="group", outcome="failed", subject_display=dept_name, external_key=dept_code, request_payload=group_item_payload, error=f"{type(e).__name__}: {e!s:.300}", duration_ms=int((time.monotonic() - group_t0) * 1000), ) state.last_synced_at = now processed_in_batch += 1 await _maybe_commit_batch() # Out-of-scope groups: NEVER deleted (permission-set # assignments hang off them on AWS) — mark excluded once. for st in state_rows: if st.dept_code in scoped_ids: continue if st.last_status in ("excluded", "soft_deleted"): continue st.last_status = "excluded" st.last_synced_at = datetime.now(UTC) st.last_error = _safe_error( "dept left Helix scope; group kept on the vendor " "(permission-set assignments may reference it) — " "retire manually if unwanted.", ) processed_in_batch += 1 if processed_in_batch > 0: await self.db.commit() return summary @staticmethod def _resolve_scim_group_names( *, all_nodes: list[Any], scoped_nodes: list[Any], ) -> dict[str, str]: """HRIS dept code → SCIM Group displayName for the in-scope depts, with duplicate-leaf-name disambiguation. SCIM group adoption matches by displayName (AWS IdC group names are a FLAT unique namespace; it can't hold two "Sales"). Real org trees legitimately repeat a leaf name across business groups (Sales / PreSales / Professional Services each live under several BGs). Colliding names are qualified with their immediate parent — "Sales (Japan Business Group)" vs "Sales (EMEA)"; a second pass appends the HRIS dept code if that still collides, so every pushed displayName is globally unique. Single source of truth for BOTH the full group phase (:meth:`_run_scim_group_phase`) and the single-person event-path ensure (:meth:`_ensure_scim_dept_group`), so a group created on the event path and the one the nightly sweep reconciles carry the SAME displayName (never a split-brain duplicate). """ name_by_code: dict[str, str] = { n.org_node_source_id: (n.org_node_name or "") for n in all_nodes } name_counts: dict[str, int] = {} for n in scoped_nodes: name_counts[n.org_node_name or ""] = ( name_counts.get(n.org_node_name or "", 0) + 1 ) dup_names = {nm for nm, c in name_counts.items() if c > 1 and nm} resolved: dict[str, str] = {} for n in scoped_nodes: raw = n.org_node_name or n.org_node_source_id if raw in dup_names: parent_name = name_by_code.get( n.parent_org_node_source_id or "", "", ).strip() resolved[n.org_node_source_id] = ( f"{raw} ({parent_name})" if parent_name else f"{raw} [{n.org_node_source_id}]" ) else: resolved[n.org_node_source_id] = raw resolved_counts: dict[str, int] = {} for nm in resolved.values(): resolved_counts[nm] = resolved_counts.get(nm, 0) + 1 for code, nm in list(resolved.items()): if resolved_counts[nm] > 1: resolved[code] = f"{nm} [{code}]" return resolved async def _ensure_scim_dept_group( self, *, target: IdentitySyncTarget, runner: SyncRunner, dept_code: str, ) -> str | None: """Find-or-create the SCIM Group for ONE dept on the event path and persist its group id to ``identity_sync_department_state``. Mirrors the full group phase's adopt/create (externalId=dept_code, displayName from :meth:`_resolve_scim_group_names`) so the event-created group and the nightly-swept group are identical. Returns the group id, or None when the dept is out of scope / can't resolve a name / the vendor create fails (caller then logs + self-heals at the nightly sweep). Never raises. """ from app.domains.identity_sync.runners.scim import ScimRunner if not isinstance(runner, ScimRunner): return None nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(nodes), ) if dept_code not in scoped_ids: # Out of scope for this target — no group should exist. return None scoped_nodes = [n for n in nodes if n.org_node_source_id in scoped_ids] display = self._resolve_scim_group_names( all_nodes=list(nodes), scoped_nodes=scoped_nodes, ).get(dept_code) if not display: return None group_id = await runner.find_group_id( display_name=display, external_code=dept_code, ) if not group_id: group_id, err = await runner.create_group( display_name=display, external_code=dept_code, ) if not group_id: logger.warning( "scim event-path group create failed for dept %s on " "target %s (self-heals at nightly sweep): %s", dept_code, target.slug, err, ) return None # Persist so the nightly sweep + subsequent event adds reuse it. st = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, IdentitySyncDepartmentState.dept_code == dept_code, ) )).scalar_one_or_none() if st is None: st = IdentitySyncDepartmentState( id=uuid.uuid4(), target_id=target.id, dept_code=dept_code, last_status="pending", ) self.db.add(st) st.external_id = group_id # Leave last_status/hash for the full sweep to own (member set # is authored there) — we only pin the adopted group id. return group_id async def _add_member_to_dept_group( self, *, target: IdentitySyncTarget, runner: SyncRunner, dept_code: str, scim_user_id: str, ) -> None: """Best-effort single-member group add for the event path. Creates the dept's SCIM Group first if it doesn't exist yet (:meth:`_ensure_scim_dept_group`) so a new hire whose dept has never been swept still lands in their group immediately, then adds the member. Never raises and logs instead of failing the person push (the nightly replace-all membership sweep is the reconciler of record for the full member set). """ from app.domains.identity_sync.runners.scim import ScimRunner if not isinstance(runner, ScimRunner): return async with best_effort( self.db, label="identity_sync: sync-state lookup", log=logger, ) as state_step: st = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, IdentitySyncDepartmentState.dept_code == dept_code, ) )).scalar_one_or_none() group_id = st.external_id if st is not None else None if not group_id: # No group yet (new hire in a never-swept dept) — create # it now instead of skipping, so the member add can land. group_id = await self._ensure_scim_dept_group( target=target, runner=runner, dept_code=dept_code, ) if not group_id: logger.info( "scim event-path group add skipped: dept %s has no " "group on target %s and could not be created " "(nightly sweep will fix)", dept_code, target.slug, ) return err = await runner.add_group_member( group_id=group_id, member_id=scim_user_id, ) if err: logger.warning( "scim event-path group add failed (self-heals at " "the nightly sweep): target=%s dept=%s: %s", target.slug, dept_code, err, ) if not state_step.ok: logger.warning( "scim event-path group add raised (self-heals at the " "nightly sweep): target=%s dept=%s", target.slug, dept_code, ) # ── huilianyi Phase A — dept sweep ─────────────────────────────── async def _run_huilianyi_dept_phase( self, *, target: IdentitySyncTarget, run_id: UUID | None = None, push_only: bool = False, restrict_dept_codes: set[str] | None = None, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> RunSummary: """Phase A of ``huilianyi`` — reconcile the HRIS org tree against 汇联易 departments. ``push_only`` (manual / test syncs): never soft-delete a dept merely outside THIS run's scope — see :meth:`_run_feishu_dept_phase`. ``restrict_dept_codes`` (single-person event-hook path): create/ update ONLY these dept codes (a new hire's dept + ancestor chain) so the vendor dept exists BEFORE Phase B references it via ``custDeptNumber`` — otherwise the user create hard-fails (``120103 部门不存在``) and only self-heals at the next full sweep. Implies ``push_only``. Structurally mirrors :meth:`_run_feishu_dept_phase` but is deliberately simpler — 汇联易 keys departments by ``custDeptNumber`` (= the HRIS dept code), so there is no name-adoption catalog to prime and no opaque-id mapping to lose. Parents that fell outside the sweep scope are None'd out so a pushed dept never references a dept we didn't push. Same contracts as the Feishu phase: dept_skipped counts hollow/out-of-scope shells, soft-deletes are brake-limited at 10 per sweep, and the phase NEVER raises into the caller — Phase B always gets to run. One extra guard the Feishu phase doesn't need: a consecutive- failure circuit breaker. Feishu aborts systemic failures via ``is_systemic_feishu_failure``; the 汇联易 runner converts all vendor errors into failed outcomes, so a dead gateway / revoked 企业密钥 would otherwise grind through every dept at one timeout each. """ from app.domains.identity_sync.runners.huilianyi import ( HuilianyiDepartmentRunner, ) summary = RunSummary() try: client = await self._build_huilianyi_client(target) except (AppException, NotFoundError, ValueError) as e: # ValueError = HuilianyiClient rejecting a malformed config # field (e.g. token_auth_style typo) — same "record + let # Phase B surface its own error" contract as a missing # integration, NOT a run-crashing exception. summary.errors.append( f"huilianyi dept-phase: {e!s:.300}", ) summary.dept_failed = 1 return summary runner = HuilianyiDepartmentRunner( client=client, dept_code_map=(target.runner_options or {}).get( "huilianyi_dept_code_map", ) or {}, ) try: result = await self._huilianyi_dept_phase_body( target=target, runner=runner, summary=summary, run_id=run_id, push_only=push_only, restrict_dept_codes=restrict_dept_codes, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=trace_id, ) except Exception as e: # noqa: BLE001 # Contract: Phase A NEVER raises into the caller — Phase B # always gets to run. A mid-loop DB error (deadlock on a # batch commit) lands here. logger.exception( "huilianyi dept-phase: unhandled error (Phase B " "continues)", ) # #1755: recover the SHARED work_session FIRST. A failed # flush/commit (the TiDB 1205/1213 deadlock this branch # explicitly expects) leaves the session in pending-rollback; # without this rollback the very first SQL Phase B issues # (_build_runner / _sweep._load_candidates) raises # PendingRollbackError and the whole run finalises failed with # an error UNRELATED to the real deadlock — Phase B never # actually ran. Roll back so the session is usable again. try: await self.db.rollback() except Exception: # noqa: BLE001 logger.exception( "huilianyi dept-phase: session rollback after error " "ALSO failed; Phase B may be unable to proceed on this " "poisoned session", ) summary.errors.append( f"huilianyi dept-phase: {type(e).__name__}: {e!s:.300}", ) summary.dept_failed += 1 result = summary finally: try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "huilianyi dept-phase: runner.aclose() raised", exc_info=True, ) # Non-blocking audit: alert IT if any active org node has NO # huilianyi_dept_code_map entry — its code would passthrough # untranslated and risk a mismatched parallel dept (the # recurring split bug). Deliberately does NOT touch run status: # the sync stays green, this is an actionable heads-up so a # human maps the new dept the day it appears. Fire-and-forget in # its own session (mirrors _finalise_run's failure notifier). try: from app.core.background import spawn from app.domains.identity_sync.alert_notifier import ( maybe_notify_unmapped_depts, ) spawn(maybe_notify_unmapped_depts(target_id=target.id)) except Exception: # noqa: BLE001 logger.warning( "huilianyi dept-phase: unmapped-dept audit spawn failed", exc_info=True, ) return result async def _huilianyi_dept_phase_body( self, *, target: IdentitySyncTarget, runner: Any, summary: RunSummary, run_id: UUID | None = None, push_only: bool = False, restrict_dept_codes: set[str] | None = None, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> RunSummary: """Inner sweep for :meth:`_run_huilianyi_dept_phase` — split out so the shell can guarantee aclose + never-raise around every exit path (the body keeps its early returns).""" async with best_effort( self.db, label="identity_sync: dept node scan", log=logger, ) as nodes5_step: nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() if not nodes5_step.ok: summary.errors.append( f"huilianyi dept-phase: load IdentityOrgNode failed: " f"{nodes5_step.reason(max_len=200)}", ) summary.dept_failed = 1 return summary if restrict_dept_codes is not None: # Targeted per-person ensure: scope is exactly the caller's # dept chain, and we NEVER soft-delete. scoped_ids = set(restrict_dept_codes) push_only = True else: try: scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(nodes), ) except Exception as e: # noqa: BLE001 summary.errors.append( f"huilianyi dept-phase: scope computation failed: " f"{type(e).__name__}: {e!s:.200}", ) summary.dept_failed = 1 return summary total_active = len(nodes) # Capture the full active-HRIS dept code set BEFORE narrowing to # member-bearing depts. Soft-delete (below) must key off "this dept is # gone / inactive in HRIS", NOT "this dept transiently has no in-scope # member" — a team whose last member just offboarded (or all went on # leave) is still a real, active department, and we must never # destructively delete it on the downstream system for that. active_node_codes: set[str] = {n.org_node_source_id for n in nodes} nodes = [n for n in nodes if n.org_node_source_id in scoped_ids] summary.dept_skipped = total_active - len(nodes) nodes = sorted( nodes, key=lambda n: (n.org_level or 0, n.org_node_source_id), ) sorted_nodes = self._topological_sort_depts(list(nodes)) in_scope_codes: set[str] = { n.org_node_source_id for n in sorted_nodes } # Re-parent Helix-root BUs under the 汇联易 PingCAP root so the vendor # tree stays single-rooted (see ``HUILIANYI_DEFAULT_ROOT_DEPT_CODE``). # Empty string disables the synthetic root (BUs stay top-level). root_dept_code = ( (target.runner_options or {}).get("huilianyi_root_dept_code") if (target.runner_options or {}).get("huilianyi_root_dept_code") is not None else HUILIANYI_DEFAULT_ROOT_DEPT_CODE ) or None # Per-dept 部门负责人 (managerEmployeeId) + 成本中心 (customFormValues # CC) — the only two enrichable fields the 汇联易 open-API dept record # exposes. Best-effort: a failure here degrades to code/name/parent # only, never blocks the dept sweep. try: dept_attrs = await self._huilianyi_dept_attributes( target=target, in_scope_codes=in_scope_codes, ) except Exception as e: # noqa: BLE001 logger.warning( "huilianyi dept-phase: attribute computation failed (%s); " "pushing code/name/parent only this sweep", e, ) dept_attrs = {} state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() state_by_code: dict[str, IdentitySyncDepartmentState] = { s.dept_code: s for s in state_rows } summary.dept_in_scope = len(sorted_nodes) batch_commit_size = 25 processed_in_batch = 0 consecutive_failures = 0 breaker_threshold = 5 async def _maybe_commit_batch() -> None: nonlocal processed_in_batch if processed_in_batch >= batch_commit_size: await self.db.commit() processed_in_batch = 0 for node_idx, node in enumerate(sorted_nodes): if consecutive_failures >= breaker_threshold: remaining = len(sorted_nodes) - node_idx summary.errors.append( f"huilianyi dept-phase circuit breaker: " f"{breaker_threshold} consecutive failures — " f"aborting; {remaining} in-scope dept(s) NOT " "attempted this sweep (check credentials / " "gateway, then re-run)." ) break dept_code = node.org_node_source_id state = state_by_code.get(dept_code) if state is None: state = IdentitySyncDepartmentState( id=uuid.uuid4(), target_id=target.id, dept_code=dept_code, last_status="pending", ) self.db.add(state) state_by_code[dept_code] = state # Out-of-scope parents are dropped from the payload so the # vendor never sees a reference to a dept we didn't push. parent_code = node.parent_org_node_source_id or None if parent_code is not None and parent_code not in in_scope_codes: parent_code = None # A Helix-root BU (no in-scope parent) nests under the 汇联易 # PingCAP root rather than floating at the vendor's top level. # Guard against the root code itself being a synced node (would # self-parent) — Helix has no such node today, but be defensive. if parent_code is None and root_dept_code and dept_code != root_dept_code: parent_code = root_dept_code da = dept_attrs.get( dept_code, HuilianyiDeptAttrs( leader_no=None, cost_center_label=None, hrbp_no=None, budget_owner_no=None, ), ) phase_a_payload = { "dept_code": dept_code, "dept_name": node.org_node_name or "", "parent_dept_code": parent_code or "", # Folded into the hash so any role/cost-center change # re-pushes instead of being skipped as unchanged. "leader": da.leader_no or "", "costCenter": da.cost_center_label or "", "hrbp": da.hrbp_no or "", "budgetOwner": da.budget_owner_no or "", } new_hash = hash_payload(phase_a_payload) # Drift re-verification (mirrors the user phase): a dept not # vendor-verified within _DRIFT_REVERIFY_INTERVAL is re-pushed # even on an unchanged hash, so a rename / re-parent done on the # vendor side outside Helix is re-asserted (idempotent upsert). _dept_verify_stale = force_reverify or _drift_reverify_stale( state.last_verified_at, jitter_key=dept_code, ) if ( state.last_payload_hash == new_hash and state.last_status in ("success", "skipped") and not _dept_verify_stale ): state.last_status = "skipped" state.last_synced_at = datetime.now(UTC) state.last_error = None processed_in_batch += 1 await _maybe_commit_batch() continue async with best_effort( self.db, label="identity_sync: dept upsert", log=logger, ) as dept5_step: outcome: DeptOutcome = await runner.upsert_dept( dept_code=dept_code, dept_name=node.org_node_name or "", parent_dept_code=parent_code, leader_email=None, dept_order=None, prior_external_id=state.external_id, leader_employee_no=da.leader_no, cost_center_label=da.cost_center_label, hrbp_employee_no=da.hrbp_no, budget_owner_employee_no=da.budget_owner_no, ) if not dept5_step.ok: logger.error( "huilianyi dept-phase: upsert_dept %s failed", dept_code, ) outcome = DeptOutcome( status="failed", external_id=state.external_id, error=dept5_step.reason(max_len=300) or "unknown", ) now = datetime.now(UTC) state.last_synced_at = now if outcome.status == "success": consecutive_failures = 0 was_new = state.last_payload_hash is None state.last_status = "success" state.last_payload_hash = new_hash state.last_verified_at = now if outcome.external_id: state.external_id = outcome.external_id state.last_error = None state.retry_count = 0 if was_new: summary.dept_created += 1 else: summary.dept_updated += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="created" if was_new else "updated", subject_display=node.org_node_name or dept_code, external_key=dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) else: consecutive_failures += 1 state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{dept_code}: {outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="failed", subject_display=node.org_node_name or dept_code, external_key=dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) processed_in_batch += 1 await _maybe_commit_batch() # ── Out-of-scope: 失效 previously pushed depts that no longer # have a matching HRIS row. Mass-action brake at 10. out_of_scope = [ s for s in state_rows if s.dept_code not in active_node_codes and s.last_status not in ("soft_deleted", "excluded") ] # push-only mode: never soft-delete a dept merely outside THIS # run's scope (e.g. another group's dept adopted by an earlier # test sync). Emptying the list no-ops the soft-delete loop below. if push_only: out_of_scope = [] dept_brake_threshold = 10 # Same stuck-row carve-out as the user sweep: depts whose 失效 # permanently fails (120102 / 120125 — vendor-side child depts or # accounts still resident, a surfaced human task) must not saturate # the brake and block fresh legitimate disables behind the backlog. dept_brake_population = [ s for s in out_of_scope if (s.retry_count or 0) < 3 ] dept_guard = None if department_mass_action_brake_required(len(dept_brake_population)): dept_guard = await DestructivePlanService(self.db).guard_or_record( target=target, scope="huilianyi_departments", action="disable", items=[{"state_id": str(s.id), "subject_key": s.dept_code, "external_id": s.external_id} for s in out_of_scope], run_id=run_id, confirmation_token_hashes=confirmation_token_hashes or [], actor_user_id=actor_user_id, trace_id=trace_id, ) if dept_guard is not None and not dept_guard.authorized: logger.error( "huilianyi dept-phase mass-action brake: target=%s " "out_of_scope=%d threshold=%d in_scope=%d — refusing " "to disable.", target.slug, len(out_of_scope), dept_brake_threshold, len(in_scope_codes), ) for s in out_of_scope: s.last_status = "failed" s.last_error = _safe_error( "mass-action brake (dept): skipped", ) summary.dept_failed += len(out_of_scope) if len(summary.errors) < 50: summary.errors.append( f"mass-action brake (dept): plan_id={dept_guard.plan_id} " f"fingerprint={dept_guard.fingerprint} reason={dept_guard.reason}; " f"out_of_scope=" f"{len(out_of_scope)} exceeds threshold=" f"{dept_brake_threshold}" ) if processed_in_batch > 0: await self.db.commit() return summary for state in out_of_scope: if consecutive_failures >= breaker_threshold: # Same breaker as the upsert loop — don't grind through # disables against a dead gateway. break processed_in_batch += 1 try: outcome = await runner.soft_delete( dept_code=state.dept_code, prior_external_id=state.external_id, ) except Exception as e: # noqa: BLE001 logger.exception( "huilianyi dept-phase: soft_delete %s failed", state.dept_code, ) outcome = DeptOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) state.last_synced_at = datetime.now(UTC) if outcome.status == "soft_deleted": consecutive_failures = 0 state.last_status = "soft_deleted" state.last_error = None summary.dept_soft_deleted += 1 else: consecutive_failures += 1 state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{state.dept_code} (失效): " f"{outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome=( "soft_deleted" if outcome.status == "soft_deleted" else "failed" ), subject_display=state.dept_code, external_key=state.dept_code, response_detail=outcome.detail, error=outcome.error, ) await _maybe_commit_batch() if processed_in_batch > 0: await self.db.commit() return summary async def _run_netsuite_dept_phase( self, *, target: IdentitySyncTarget, run_id: UUID | None = None, push_only: bool = False, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> RunSummary: """Phase A of ``netsuite`` — reconcile the HRIS org tree against NetSuite departments. ``push_only`` (manual / test syncs): never soft-delete a dept merely outside THIS run's scope — see :meth:`_run_feishu_dept_phase`. Structurally mirrors :meth:`_run_huilianyi_dept_phase`: NetSuite keys departments by ``externalId`` (= the HRIS org-node code), so the HRIS dept code IS the vendor key — no name-adoption catalog, no opaque-id mapping. Parents that fell outside the sweep scope are None'd out so a pushed dept never references a dept we didn't push. Same contracts as the other dept phases: ``dept_skipped`` counts hollow/out-of-scope shells, soft-deletes are brake-limited at 10 per sweep, and the phase NEVER raises into the caller — Phase B always gets to run. This phase is GATED OFF by default — ``run_target`` only invokes it when ``runner_options.netsuite_sync_departments`` is True. Enabling the NetSuite target / running the employee sweep does NOT touch departments unless that flag is explicitly set, because NetSuite departments are still Authing-era reference data until the cutover is adopted (see DECISIONS.md). """ from app.domains.identity_sync.runners.netsuite import ( NetSuiteDepartmentRunner, ) summary = RunSummary() try: client = await self._build_netsuite_client(target) except (AppException, NotFoundError, ValueError) as e: # Same "record + let Phase B surface its own error" contract # as a missing integration, NOT a run-crashing exception. summary.errors.append( f"netsuite dept-phase: {e!s:.300}", ) summary.dept_failed = 1 return summary _dept_opts = target.runner_options or {} runner = NetSuiteDepartmentRunner( client=client, create_missing=bool( _dept_opts.get("netsuite_create_missing", False) ), ) try: result = await self._netsuite_dept_phase_body( target=target, runner=runner, summary=summary, run_id=run_id, push_only=push_only, force_reverify=force_reverify, confirmation_token_hashes=confirmation_token_hashes, actor_user_id=actor_user_id, trace_id=trace_id, ) except Exception as e: # noqa: BLE001 # Contract: Phase A NEVER raises into the caller — Phase B # always gets to run. logger.exception( "netsuite dept-phase: unhandled error (Phase B continues)", ) # #1755: recover the SHARED work_session FIRST — see the # huilianyi twin above. A poisoned (pending-rollback) session # makes Phase B fail with a misleading PendingRollbackError # and Phase B never runs. try: await self.db.rollback() except Exception: # noqa: BLE001 logger.exception( "netsuite dept-phase: session rollback after error " "ALSO failed; Phase B may be unable to proceed on this " "poisoned session", ) summary.errors.append( f"netsuite dept-phase: {type(e).__name__}: {e!s:.300}", ) summary.dept_failed += 1 result = summary finally: try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "netsuite dept-phase: runner.aclose() raised", exc_info=True, ) # Non-blocking audit: alert IT to org nodes NOT yet adopted in # NetSuite (no externalId match) — the split-risk seed once # create_missing is flipped on. Does NOT touch run status (sync # stays green; unadopted depts are already skipped in update-only). # Fires only when the dept phase runs (already gated behind # netsuite_sync_departments). Fire-and-forget in its own session. try: from app.core.background import spawn from app.domains.identity_sync.alert_notifier import ( maybe_notify_unadopted_ns_depts, ) spawn(maybe_notify_unadopted_ns_depts(target_id=target.id)) except Exception: # noqa: BLE001 logger.warning( "netsuite dept-phase: unadopted-dept audit spawn failed", exc_info=True, ) return result async def _netsuite_dept_phase_body( self, *, target: IdentitySyncTarget, runner: Any, summary: RunSummary, run_id: UUID | None = None, push_only: bool = False, force_reverify: bool = False, confirmation_token_hashes: list[str] | None = None, actor_user_id: UUID | None = None, trace_id: str = "identity-sync-destructive", ) -> RunSummary: """Inner sweep for :meth:`_run_netsuite_dept_phase` — split out so the shell can guarantee aclose + never-raise around every exit path (the body keeps its early returns). Mirrors :meth:`_huilianyi_dept_phase_body`.""" async with best_effort( self.db, label="identity_sync: dept node scan", log=logger, ) as nodes6_step: nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() if not nodes6_step.ok: summary.errors.append( f"netsuite dept-phase: load IdentityOrgNode failed: " f"{nodes6_step.reason(max_len=200)}", ) summary.dept_failed = 1 return summary try: scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(nodes), ) except Exception as e: # noqa: BLE001 summary.errors.append( f"netsuite dept-phase: scope computation failed: " f"{type(e).__name__}: {e!s:.200}", ) summary.dept_failed = 1 return summary total_active = len(nodes) # Capture the full active-HRIS dept code set BEFORE narrowing to # member-bearing depts. Soft-delete (below) must key off "this dept is # gone / inactive in HRIS", NOT "this dept transiently has no in-scope # member" — a team whose last member just offboarded (or all went on # leave) is still a real, active department, and we must never # destructively delete it on the downstream system for that. active_node_codes: set[str] = {n.org_node_source_id for n in nodes} nodes = [n for n in nodes if n.org_node_source_id in scoped_ids] summary.dept_skipped = total_active - len(nodes) nodes = sorted( nodes, key=lambda n: (n.org_level or 0, n.org_node_source_id), ) sorted_nodes = self._topological_sort_depts(list(nodes)) in_scope_codes: set[str] = { n.org_node_source_id for n in sorted_nodes } state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() state_by_code: dict[str, IdentitySyncDepartmentState] = { s.dept_code: s for s in state_rows } summary.dept_in_scope = len(sorted_nodes) batch_commit_size = 25 processed_in_batch = 0 consecutive_failures = 0 breaker_threshold = 5 async def _maybe_commit_batch() -> None: nonlocal processed_in_batch if processed_in_batch >= batch_commit_size: await self.db.commit() processed_in_batch = 0 for node_idx, node in enumerate(sorted_nodes): if consecutive_failures >= breaker_threshold: remaining = len(sorted_nodes) - node_idx summary.errors.append( f"netsuite dept-phase circuit breaker: " f"{breaker_threshold} consecutive failures — " f"aborting; {remaining} in-scope dept(s) NOT " "attempted this sweep (check credentials / host, " "then re-run)." ) break dept_code = node.org_node_source_id state = state_by_code.get(dept_code) if state is None: state = IdentitySyncDepartmentState( id=uuid.uuid4(), target_id=target.id, dept_code=dept_code, last_status="pending", ) self.db.add(state) state_by_code[dept_code] = state # Drop the parent reference unless the parent is BOTH in scope # AND already adopted in NetSuite. NetSuite addresses the parent # by ``externalId`` (= the parent's org_node_source_id); sending # one for a parent whose NS dept has no externalId yet (un-adopted # collision/gap, or a parent that 404-skips this sweep) is a # dangling reference → INVALID_KEY_OR_REF → the CHILD fails. The # parent's adoption signal is ``state.external_id``: the dept # phase only stores it on a successful upsert (the skipped/404 # branch leaves it None), and topo-sort guarantees the parent is # processed before the child, so by here a truthy external_id # means "adopted & pushed". When the parent isn't adopted we omit # ``parent`` entirely — PATCH-merge then PRESERVES the dept's # existing (SWC-era) parent rather than clobbering it with a # dangling ref, and the link self-heals on a later sweep once the # parent adopts (the payload hash changes, the child re-syncs). parent_code = node.parent_org_node_source_id or None if parent_code is not None: _parent_state = state_by_code.get(parent_code) _parent_adopted = ( _parent_state is not None and bool(_parent_state.external_id) ) if parent_code not in in_scope_codes or not _parent_adopted: parent_code = None phase_a_payload = { "dept_code": dept_code, "dept_name": node.org_node_name or "", "parent_dept_code": parent_code or "", } new_hash = hash_payload(phase_a_payload) # Drift re-verification (mirrors the user phase): a dept not # vendor-verified within _DRIFT_REVERIFY_INTERVAL is re-pushed # even on an unchanged hash, so a rename / re-parent done on the # vendor side outside Helix is re-asserted (idempotent upsert). _dept_verify_stale = force_reverify or _drift_reverify_stale( state.last_verified_at, jitter_key=dept_code, ) if ( state.last_payload_hash == new_hash and state.last_status in ("success", "skipped") and not _dept_verify_stale ): state.last_status = "skipped" state.last_synced_at = datetime.now(UTC) state.last_error = None processed_in_batch += 1 await _maybe_commit_batch() continue async with best_effort( self.db, label="identity_sync: dept upsert", log=logger, ) as dept6_step: outcome: DeptOutcome = await runner.upsert_dept( dept_code=dept_code, dept_name=node.org_node_name or "", parent_dept_code=parent_code, leader_email=None, dept_order=None, prior_external_id=state.external_id, ) if not dept6_step.ok: logger.error( "netsuite dept-phase: upsert_dept %s failed", dept_code, ) outcome = DeptOutcome( status="failed", external_id=state.external_id, error=dept6_step.reason(max_len=300) or "unknown", ) now = datetime.now(UTC) state.last_synced_at = now if outcome.status == "success": consecutive_failures = 0 was_new = state.last_payload_hash is None state.last_status = "success" state.last_payload_hash = new_hash state.last_verified_at = now if outcome.external_id: state.external_id = outcome.external_id state.last_error = None state.retry_count = 0 # Real vendor op over was_new — an adopted dept's first sweep # is an UPDATE (PATCH eid:), not a create (see employee note). _op = outcome.detail.get("op") if isinstance(outcome.detail, dict) else None did_create = (_op == "create") if _op in ("create", "update") else was_new if did_create: summary.dept_created += 1 else: summary.dept_updated += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="created" if did_create else "updated", subject_display=node.org_node_name or dept_code, external_key=dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) elif outcome.status == "skipped": # Update-only mode: this dept isn't adopted in NetSuite yet # (404, empty externalId on the existing record). Refusing # to create avoids a parallel dept tree — count as skipped, # not failed, and don't trip the breaker. consecutive_failures = 0 state.last_status = "skipped" state.last_error = None state.retry_count = 0 summary.dept_skipped += 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="skipped", subject_display=node.org_node_name or dept_code, external_key=dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) else: consecutive_failures += 1 state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{dept_code}: {outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome="failed", subject_display=node.org_node_name or dept_code, external_key=dept_code, request_payload=phase_a_payload, response_detail=outcome.detail, error=outcome.error, ) processed_in_batch += 1 await _maybe_commit_batch() # ── Out-of-scope: soft-delete previously pushed depts that no # longer have a matching HRIS row. Mass-action brake at 10. out_of_scope = [ s for s in state_rows if s.dept_code not in active_node_codes and s.last_status not in ("soft_deleted", "excluded") ] # push-only mode: never soft-delete a dept merely outside THIS # run's scope (e.g. another group's dept adopted by an earlier # test sync). Emptying the list no-ops the soft-delete loop below. if push_only: out_of_scope = [] dept_brake_threshold = 10 # Same stuck-row carve-out as huilianyi: depts whose soft-delete # permanently fails must not saturate the brake and block fresh # legitimate disables behind the backlog. dept_brake_population = [ s for s in out_of_scope if (s.retry_count or 0) < 3 ] dept_guard = None if department_mass_action_brake_required(len(dept_brake_population)): dept_guard = await DestructivePlanService(self.db).guard_or_record( target=target, scope="netsuite_departments", action="soft_delete", items=[{"state_id": str(s.id), "subject_key": s.dept_code, "external_id": s.external_id} for s in out_of_scope], run_id=run_id, confirmation_token_hashes=confirmation_token_hashes or [], actor_user_id=actor_user_id, trace_id=trace_id, ) if dept_guard is not None and not dept_guard.authorized: logger.error( "netsuite dept-phase mass-action brake: target=%s " "out_of_scope=%d threshold=%d in_scope=%d — refusing " "to disable.", target.slug, len(out_of_scope), dept_brake_threshold, len(in_scope_codes), ) for s in out_of_scope: s.last_status = "failed" s.last_error = _safe_error( "mass-action brake (dept): skipped", ) summary.dept_failed += len(out_of_scope) if len(summary.errors) < 50: summary.errors.append( f"mass-action brake (dept): plan_id={dept_guard.plan_id} " f"fingerprint={dept_guard.fingerprint} reason={dept_guard.reason}; " f"out_of_scope=" f"{len(out_of_scope)} exceeds threshold=" f"{dept_brake_threshold}" ) if processed_in_batch > 0: await self.db.commit() return summary for state in out_of_scope: if consecutive_failures >= breaker_threshold: break processed_in_batch += 1 try: outcome = await runner.soft_delete( dept_code=state.dept_code, prior_external_id=state.external_id, ) except Exception as e: # noqa: BLE001 logger.exception( "netsuite dept-phase: soft_delete %s failed", state.dept_code, ) outcome = DeptOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) state.last_synced_at = datetime.now(UTC) if outcome.status == "soft_deleted": consecutive_failures = 0 state.last_status = "soft_deleted" state.last_error = None summary.dept_soft_deleted += 1 else: consecutive_failures += 1 state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 summary.dept_failed += 1 if len(summary.errors) < 50: summary.errors.append( f"{state.dept_code} (soft_delete): " f"{outcome.error or 'unknown'}" ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="department", outcome=( "soft_deleted" if outcome.status == "soft_deleted" else "failed" ), subject_display=state.dept_code, external_key=state.dept_code, response_detail=outcome.detail, error=outcome.error, ) await _maybe_commit_batch() if processed_in_batch > 0: await self.db.commit() return summary async def _enqueue_auto_include_notification( self, *, target: IdentitySyncTarget, manager_person: IdentityPerson, ) -> None: """Write a NotificationOutbox row telling the auto-included manager why we just created their account on the downstream system. Sent in fire-and-forget mode: we write a ``status='pending'`` row and let the standard ``notifications.retry_pending_outbox`` scheduled job deliver it on the next tick. Doing this rather than calling ``NotificationService.dispatch()`` synchronously means a single slow Feishu round-trip per CN manager can't stall the whole sweep (worst case: 500 managers × 2s each = 17 min added to a 30s sweep). ``notified_at`` is set by the caller on enqueue, so a transient Feishu outage doesn't trigger a new enqueue every sweep. """ from app.notifications.models import STATUS_PENDING, NotificationOutbox # Title/body below are the pre-template fallback only. The real # copy lives in the ``identity_sync.auto_included_manager`` # template (app/notifications/templates/platform.py) and is # picked per recipient language at delivery time. # # This used to be hardcoded Chinese with a hardcoded "Navan", # rationalised as "the auto-included manager is by definition # CN-based". Neither half held: ``_compute_auto_included_manager_ # ids`` dropped its ``is_china_employee`` gate on 2026-07-07, and # auto-include is a property of ANY identity-sync target, not of # Navan. The vendor name now comes from ``target.name``. manager_name = ( manager_person.name_pinyin or manager_person.display_name or manager_person.work_email ) title = f"你的 {target.name} 账号已自动创建" body_lines = [ f"Hi {manager_name},", "", f"由于你的直接下属中包含外籍员工,Helix 已经在 {target.name} " f"为你自动创建了账号,以便系统中能够正确关联汇报关系。", "", f" · 同步目标:{target.name}", f" · 你的工作邮箱:{manager_person.work_email}", "", "首次登录请通过公司 SSO。如有疑问请联系 IT 团队。", "", "— Helix IT 平台", ] body_text = "\n".join(body_lines) outbox = NotificationOutbox( recipient_person_id=manager_person.id, recipient_email=manager_person.work_email, template_key="identity_sync.auto_included_manager", title=title, body_text=body_text, channels=["feishu", "email"], status=STATUS_PENDING, attempts=0, max_attempts=5, related_resource_type="identity_sync_target", related_resource_id=target.id, context={ "target_slug": target.slug, "target_name": target.name, "reason": "manager_of_foreign", # Template placeholders. The greeting name and the work # email are baked into ``body_text`` above too, but the # template re-renders from context alone (both on # delivery and on every bell re-render), so they have to # be here or the recipient sees a literal # ``{manager_name}``. "manager_name": manager_name, "work_email": manager_person.work_email, }, ) self.db.add(outbox) await self.db.flush() async def _finalise_run( self, run_id: UUID, *, status: str, error_message: str | None, summary: RunSummary | None, ) -> None: """Final write-back in a fresh session — runs even if the sweep raised, to prevent stuck 'running' rows.""" async with async_session_factory() as session: run_row = await session.get(IdentitySyncRun, run_id) if run_row is None: return finished = datetime.now(UTC) run_row.finished_at = finished # MySQL/TiDB DATETIME columns drop the tzinfo on read, so # the freshly re-fetched ``started_at`` comes back naive # even though we wrote it as aware. Re-attach UTC before # subtracting from the aware ``finished`` value, otherwise # Python raises "can't subtract offset-naive and # offset-aware datetimes" and the entire run row stays at # status='running' forever (the FastAPI handler also # 500s, which is what the user sees as the generic # "An unexpected error occurred"). started = run_row.started_at if started is not None and started.tzinfo is None: started = started.replace(tzinfo=UTC) # ``max(0, …)``: ``started_at`` is persisted into a MySQL/TiDB # DATETIME column with no fractional-second precision, so a # value like 13:16:45.6 is ROUNDED UP to 13:16:46 on write. # A sub-second run (e.g. a LockBusy skip that finalises ~100ms # later at 13:16:45.7) then computes a NEGATIVE delta and the # UI renders a nonsensical "-0.1s". Clamp to zero. run_row.duration_ms = max(0, int( (finished - started).total_seconds() * 1000, )) if started is not None else 0 run_row.status = status run_row.error_message = _safe_error(error_message) # Scrub the per-person error list inside ``result_summary`` # too — it lands in ``run_row.result_summary`` and the # mirror on the target row, both of which the admin UI # tooltip-renders without further redaction. scrubbed_summary_dict: dict[str, Any] | None = None if summary is not None: run_row.persons_in_scope = summary.persons_in_scope run_row.created_count = summary.created run_row.updated_count = summary.updated run_row.skipped_count = summary.skipped run_row.deactivated_count = summary.deactivated + summary.deleted run_row.failed_count = summary.failed run_row.blocked_count = summary.blocked scrubbed_summary_dict = summary.as_dict() if scrubbed_summary_dict.get("errors"): scrubbed_summary_dict["errors"] = [ _safe_error(e) or "" for e in scrubbed_summary_dict["errors"] ] run_row.result_summary = scrubbed_summary_dict # ── Item-level failures must reach ``error_message`` ── # A sweep that fails per-PERSON never raises: it returns a # summary and finalises here with ``error_message=None``, # because the only caller that passes text is the # except-path. Everything downstream reads that column and # nothing reads ``result_summary.errors``: # # * ``run_due_targets`` builds ``ran_failed`` from # ``error_message or 'failed'`` — so the dispatcher's # RuntimeError, and therefore the P2 card IT receives, # literally read "google-dept: failed" (2026-09-07); # * ``alert_notifier`` renders it as the card's error line # — blank; # * ``compute_failure_signature`` hashes it, so EVERY # distinct per-person cause on a target collapsed to one # signature and the second, different failure was # suppressed as a duplicate of the first. # # The diagnosis was already in ``summary.errors`` the whole # time (``": "``). Lift the first few # into the column every consumer actually reads. Mirrors the # dept-phase synthesis below, which has done this since # 2026-07 for Phase A only. _synth: list[str] = [] if summary.failed > 0 and not run_row.error_message: # Flatten to ONE line and cap each head: consumers keep # only the first line of this column, so an embedded # newline (a vendor error carrying a traceback) would # silently truncate the message back to nothing useful. _heads = [ " ".join(str(e).split())[:160] for e in (summary.errors or []) if e ][:3] _lead = ( f"{summary.failed} of {summary.persons_in_scope} " f"person(s) failed" ) _synth.append( f"{_lead}: {' | '.join(_heads)}" if _heads else f"{_lead}; see result_summary.errors for details." ) # When Phase A had failures BUT no top-level error # message was passed in (Phase B succeeded fully), # synthesise an error_message that points the admin to # the dept-phase counters / errors in result_summary. # Without this, the runs list view shows status=partial # + dept_failed=N + error_message=None and admin can't # tell from the list view that Phase A had trouble. if summary.dept_failed > 0 and not run_row.error_message: _synth.append( f"dept phase had {summary.dept_failed} failure(s) " f"(in_scope={summary.dept_in_scope}, " f"created={summary.dept_created}, " f"updated={summary.dept_updated}, " f"soft_deleted={summary.dept_soft_deleted}); " "see result_summary.errors for details." ) if _synth: # Single line: every consumer (signature, card body, # dispatcher preview) keeps only the first line. run_row.error_message = _safe_error("; ".join(_synth)) target_row = await session.get(IdentitySyncTarget, run_row.target_id) if target_row is not None: target_row.last_run_at = finished target_row.last_run_status = status target_row.last_run_summary = scrubbed_summary_dict # The cron cursor (next_run_at) moves ONLY on the # dispatcher's SCHEDULED run. Per-person / event / manual # runs finalise through this same path — often many times # per hour on a heavily-provisioned target like # feishu_contact (onboarding fires an # ``onboarding_feishu_provision`` push at the top of every # hour, including the cron minute). If those moved # next_run_at too, each one shoves the daily slot into the # future BEFORE the 60s dispatch tick at the cron minute can # see the target as due — starving the scheduled full sweep. # The observable symptom (2026-07-14): a "0 4 * * *" daily # cron fired only ~7×/30d, so existing-employee dept / # manager changes never propagated to Feishu until a manual # sweep. Google targets (no hourly per-person traffic) were # unaffected, which pinpointed the cause. # # HOW it moves depends on whether the run left UNFINISHED # WORK behind (see the ``_SCHED_*`` re-arm policy): a # ``cancelled`` run (deploy/restart) re-runs in ~60s so the # day's reconcile isn't lost; a run that ended with anyone # still un-synced retries the same day on an escalating # backoff before falling back to the daily slot; a clean run # advances straight to the next cron slot. # # The retry key is ``_RUN_STATUSES_WITH_FAILURES`` — i.e. # "did anything fail", which includes ``partial``. It used # to be "was the TARGET down", which treated ``partial`` # exactly like a clean run and pushed the cursor to # tomorrow: a single transient per-person failure on a live # target cost that person a full day of staleness, the # exact 24h cost this whole policy was written to remove. is_cron = ( target_row.enabled and target_row.schedule_config.get("trigger") == "cron" ) if is_cron and run_row.trigger_source == "scheduler": # Outcome-aware cursor re-arm — the fix for "one # transient failure / one deploy costs a whole day of # staleness". See ``_SCHED_*`` constants for the policy. if status == "cancelled": # Deploy / restart killed the sweep. Re-run almost # immediately; do NOT advance to tomorrow and do NOT # count it as a data failure (streak untouched). target_row.next_run_at = ( finished + _SCHED_CANCELLED_RETRY_DELAY ) elif status not in _RUN_STATUSES_WITH_FAILURES: target_row.next_run_at = _compute_next_run( target_row.schedule_config, after=finished, ) target_row.sched_failure_streak = 0 else: streak = (target_row.sched_failure_streak or 0) + 1 if streak <= _SCHED_FAILURE_MAX_SAMEDAY_RETRIES: # Same-day retry with escalating backoff. target_row.next_run_at = ( finished + _sched_failure_backoff(streak) ) else: # Exhausted same-day retries — fall back to the # next cron slot; staleness watchdog + failure # notifier keep the persistently-broken target # visible without hammering the vendor. target_row.next_run_at = _compute_next_run( target_row.schedule_config, after=finished, ) # Cap the stored streak so it can't grow unbounded # over months of a chronically-failing target. target_row.sched_failure_streak = min( streak, _SCHED_FAILURE_MAX_SAMEDAY_RETRIES + 1, ) elif is_cron and target_row.next_run_at is None: # ``next_run_at is None`` self-heal: an enabled cron # target whose cursor drifted to NULL (manual SQL / # migration) would otherwise never become due again now # that only the scheduled run advances it — re-seed on # any finalise (event / manual runs reach here too). target_row.next_run_at = _compute_next_run( target_row.schedule_config, after=finished, ) await session.commit() # ── Failure alert ──────────────────────────────────────────── # Fire AFTER the run row is committed (so the notifier's own # session reads the persisted status / error / target) and ONLY # for runs that actually went wrong. ``partial`` carries # per-person or dept-phase failures the admin should still see. # This is the trigger-source-agnostic path: cron, manual AND # event runs all funnel through ``_finalise_run``, which closes # the original gap (an event:hire_confirm 403 that never reached # the cron dispatcher's raise-on-failure notifier). Best-effort, # fire-and-forget via the retained-task helper so a slow Feishu # never blocks the run path or the HTTP handler. if status in ("failed", "partial"): from app.core.background import spawn from app.domains.identity_sync.alert_notifier import ( maybe_notify_run_failure, ) spawn( maybe_notify_run_failure(run_id=run_id), name=f"identity_sync.alert:{run_id}", ) # STRICT value_map misses can occur on ANY run outcome (a green run # still silently drops an unmapped code's field) — fire the # dedup'd "new unmapped code" heads-up whenever the summary recorded # misses, independent of the run status. if summary is not None and ( scrubbed_summary_dict or {} ).get("value_map_misses"): from app.core.background import spawn from app.domains.identity_sync.alert_notifier import ( maybe_notify_value_map_misses, ) spawn( maybe_notify_value_map_misses(run_id=run_id), name=f"identity_sync.value_map_miss:{run_id}", ) # A ``blocked`` person keeps the run GREEN by design (non-transient # vendor refusal, not retry-worthy) — which meant the backlog had # NO signal at all: the state row said "blocked" and nothing told # anyone. That is how a new hire went a day without a Zoom account # while every run reported ok. Fire the dedup'd backlog heads-up on # any outcome, for the same reason as the value_map audit above. if summary is not None and (summary.blocked or 0) > 0: from app.core.background import spawn from app.domains.identity_sync.alert_notifier import ( maybe_notify_blocked_backlog, ) spawn( maybe_notify_blocked_backlog(run_id=run_id), name=f"identity_sync.blocked_backlog:{run_id}", ) # ── Per-person on-demand push ──────────────────────────────────── async def sync_person_to_target( self, *, target_id: UUID, person_id: UUID, trigger_source: str, force_in_scope: bool = False, lockbusy_retries_left: int = _EVENT_PUSH_LOCKBUSY_RETRIES, ) -> PersonOutcome: """Push one person to one target — the SINGLE chokepoint every single-person push funnels through: * Lifecycle event hooks (hire / leave / role-change) via ``events.enqueue_person_sync`` — ``force_in_scope=False``. * Admin "Sync this person" button — ``force_in_scope=True``. * MCP ``sync_person_to_target`` tool — ``force_in_scope=True``. * Onboarding Feishu provisioning coordinator. **Platform invariant enforced here: at most ONE in-flight push for the same (person, target) at a time.** A single lifecycle event can double-fire (an offboard is spawned by BOTH ``advance_service.process_advance`` and the leaver-blocked completion path), an admin can double-click, an MCP client can retry, and a scheduler can re-dispatch — any of which otherwise starts a second concurrent push that races the per-target ``identity_sync:target:`` lock acquired below, so the loser lands a scary green-but-"locked by another sync — this push was NOT executed" run. Rather than patch each caller (and re-learn the same lesson for the next one), the guard lives at this chokepoint so EVERY caller — current and future — is covered uniformly: the duplicate collapses cheaply with NO target-lock collision, recorded as a small ``skipped`` run row so the drop is observable (never a silent return with no trace). One drop is NOT cheap: an offboard/deactivate-class event. The in-flight push it collapsed into may have snapshotted the person while still active (e.g. an overlapping update), so the collapsed offboard may be the ONLY deactivation this target will ever see (``event_only`` targets have no sweep at all; cron targets would wait a full day). Those ride the event-push must-deliver machinery: bounded delayed re-enqueue, recorded as a skipped run row; only when retries exhaust on an ``event_only`` target does the collapse finalise as a FAILED run ("dropped: another push in flight — needs re-push") so the failure notifier pages IT. See :meth:`_record_inflight_collapse`. Keyed per-(person, target), so distinct people/targets still push in parallel, and it only collapses *overlapping* duplicates — a later, non-overlapping push for the same person+target (e.g. a role-change minutes after an offboard) finds the guard free and proceeds. A concurrent full CRON sweep takes a DIFFERENT lock and is deliberately NOT collapsed here: that legitimate event-vs-sweep case still flows through the push body's own ``LockBusy``→skipped ("next sweep reconciles") path. (This generalises the earlier point-fix in ``hr_sync_coordinator._provision_feishu_account_bg``, which added the same in-flight guard for the onboarding Feishu double-fire alone; that guard is now belt-and-suspenders.) """ from app.core.distributed_lock import LockBusy, distributed_lock # ttl comfortably exceeds a single push (feishu / huilianyi run an # extra dept phase) and auto-expires if the worker dies mid-push, # so a crash can never wedge this person out of future pushes. guard_key = f"identity_sync:push_inflight:{target_id}:{person_id}" try: async with distributed_lock(guard_key, ttl=1800): return await self._push_person_to_target( target_id=target_id, person_id=person_id, trigger_source=trigger_source, force_in_scope=force_in_scope, lockbusy_retries_left=lockbusy_retries_left, ) except LockBusy: return await self._record_inflight_collapse( target_id=target_id, person_id=person_id, trigger_source=trigger_source, force_in_scope=force_in_scope, lockbusy_retries_left=lockbusy_retries_left, ) async def _record_inflight_collapse( self, *, target_id: UUID, person_id: UUID, trigger_source: str, force_in_scope: bool = False, lockbusy_retries_left: int = 0, ) -> PersonOutcome: """Finalise the outcome of a push collapsed by the per-(person, target) in-flight guard in :meth:`sync_person_to_target`. The original #1955 guard returned a silent ``skipped`` with NO run row, which had two holes: * **Observability** — the drop left no trace anywhere, so an admin diffing "events fired" vs "runs recorded" re-derived the collapse from logs every time. Every collapse now records a one-person run row (mirroring the in-body target-lock ``LockBusy`` path) so the runs page shows the drop + reason. * **Lost deactivations** — an offboard/deactivate-class event may collapse into an in-flight push that snapshotted the person while still ACTIVE (an overlapping update / hire-day push), so the collapse can silently drop the only deactivation this target would see (``event_only`` targets have no sweep to reconcile it at all; cron targets would wait until the next daily sweep). Those ride the event-push must-deliver machinery (PR4/#1974): re-enqueue via :func:`_retry_person_push_after_delay` — bounded by ``lockbusy_retries_left`` — and record a *skipped* run row noting the re-enqueue (one signal, not a page + retry both). Only when retries are exhausted AND the target is ``event_only`` (nothing will ever reconcile it) does the collapse finalise as FAILED so ``_finalise_run``'s failure notifier pages IT to re-trigger by hand. Ordinary duplicate collapses (hire double-fire, admin double-click, MCP retry, any non-offboard trigger) stay cheap and are NOT re-enqueued — deduping them is this guard's whole point: no vendor call, no target-lock collision — just a skipped run row with the reason. """ target = await self.get_target(target_id) _sched = target.schedule_config or {} _reconciled_by_cron = (_sched.get("trigger") or "manual") == "cron" _event_reason = ( trigger_source.split(":", 1)[1] if trigger_source.startswith("event:") else "" ) _offboard_class = _event_reason in _OFFBOARD_EVENT_REASONS reenqueued = False if _offboard_class and lockbusy_retries_left > 0: # Must-deliver: the deactivation must land on its own once the # in-flight push finishes — the retry re-enters the public # chokepoint (so a still-running original collapses it again, # burning one retry, and a freed guard lets it through). from app.core.background import spawn spawn( _retry_person_push_after_delay( target_id=target_id, person_id=person_id, trigger_source=trigger_source, force_in_scope=force_in_scope, retries_left=lockbusy_retries_left - 1, delay_s=_EVENT_PUSH_LOCKBUSY_RETRY_DELAY_S, ), name=( f"identity_sync.inflight_collapse_retry:" f"{target_id}:{person_id}" ), ) reenqueued = True dropped_forever = ( _offboard_class and not _reconciled_by_cron and not reenqueued ) if dropped_forever: outcome = PersonOutcome( status="failed", error=( "dropped: another push in flight — needs re-push. " f"This {trigger_source!r} deactivation collapsed into a " "concurrent push for the same person+target (re-enqueue " "retries exhausted), and this target has NO scheduled " "sweep (event_only), so nothing will reconcile it — " "re-trigger the push." ), ) elif reenqueued: outcome = PersonOutcome( status="skipped", error=( "a push for this person+target was already in flight; " f"this {trigger_source!r} deactivation was re-enqueued " f"(retry in {_EVENT_PUSH_LOCKBUSY_RETRY_DELAY_S}s, " f"{lockbusy_retries_left} attempt(s) left) so it lands " "once the in-flight push finishes." ), ) else: outcome = PersonOutcome( status="skipped", error=( "a push for this person+target was already in flight; " "collapsed to that concurrent run (no-op)" + ( ". The person is covered by the next scheduled " "sweep." if _reconciled_by_cron else "" ) ), ) logger.log( logging.WARNING if dropped_forever else logging.INFO, "sync_person_to_target: push for person=%s target=%s (%s) " "collapsed — a push for this person+target is already in " "flight (duplicate event fan-out / double-click / retry); %s", person_id, target_id, trigger_source, "event_only offboard DROPPED after retries — recording FAILED " "run, alerting" if dropped_forever else ( "offboard-class — re-enqueued via must-deliver, recording " "skipped run row" if reenqueued else "recording skipped run row" ), ) # Record a one-person run row so the drop is observable on the # runs page (and, for the dropped-forever case, so the failure # notifier fires off the finalised FAILED run). run_row = IdentitySyncRun( id=uuid.uuid4(), target_id=target.id, started_at=datetime.now(UTC), status="running", trigger_source=trigger_source, persons_in_scope=1, ) self.db.add(run_row) await self.db.flush() await self.db.commit() summary = RunSummary(persons_in_scope=1) if outcome.status == "failed": summary.failed = 1 if outcome.error: summary.errors.append(outcome.error) else: summary.skipped = 1 await self._finalise_run( run_row.id, status="failed" if outcome.status == "failed" else "ok", error_message=outcome.error, summary=summary, ) return outcome async def _push_person_to_target( self, *, target_id: UUID, person_id: UUID, trigger_source: str, force_in_scope: bool = False, lockbusy_retries_left: int = _EVENT_PUSH_LOCKBUSY_RETRIES, ) -> PersonOutcome: """Inner body of :meth:`sync_person_to_target`, run under the per-(person, target) in-flight guard the public wrapper holds. ``force_in_scope=True`` bypasses the filter membership check entirely: the runner always upserts (never deactivates), which matches the admin / API caller's explicit intent. Without this override, picking a person who isn't in the target's filter would silently route to the deactivate path — confusing UX because the admin asked to "push" someone and got a deactivation. The result is recorded as a single-person IdentitySyncRun (persons_in_scope=1) so the runs page reflects every push. """ target = await self.get_target(target_id) if not target.enabled: raise AppException( code="IDENTITY_SYNC_DISABLED", message=f"target {target.slug!r} is disabled", status_code=409, ) person = await self.db.get(IdentityPerson, person_id) if person is None or person.deleted_at is not None or not person.work_email: raise NotFoundError("IdentityPerson", str(person_id)) run_row = IdentitySyncRun( id=uuid.uuid4(), target_id=target.id, started_at=datetime.now(UTC), status="running", trigger_source=trigger_source, persons_in_scope=1, ) self.db.add(run_row) await self.db.flush() await self.db.commit() # Acquire the SAME per-target distributed lock the dispatcher # uses (``identity_sync:target:``) so a concurrent full # sweep cannot race this single-person path on the same # ``identity_sync_state`` row. Without this guard, the sweep # could be writing ``last_payload_hash`` for person P while # this code commits a different hash for P → lost update + # potential duplicate downstream API call. ``LockBusy`` on # acquire is graceful: we record the run as skipped (the # sweep is covering this person anyway). from app.core.distributed_lock import LockBusy, distributed_lock runner: SyncRunner | None = None outcome = PersonOutcome(status="failed", error="not started") error_message: str | None = None lock_key = f"identity_sync:target:{target.slug}" try: # auto_renew (#1754): keep the per-target lock alive for the # whole push (feishu/huilianyi also run a dept phase here) so a # concurrent full sweep can't slip in on lock expiry and race # this person's identity_sync_state row. async with distributed_lock( lock_key, ttl=900, auto_renew=True, ): async with async_session_factory() as work_session: svc = IdentitySyncService(work_session) work_target = await work_session.get(IdentitySyncTarget, target.id) if work_target is None: raise NotFoundError("IdentitySyncTarget", str(target.id)) # feishu_contact / huilianyi: the full dept sweep # (Phase A) does NOT run on this single-person path, so a # new hire's department may not exist on the vendor yet. # Feishu would then silently DROP the business dept (user # lands only in the appended office node); huilianyi would # HARD-FAIL the user create (`120103 部门不存在`). Ensure the # person's primary-membership dept + ancestor chain # exists/is adopted on the vendor FIRST, then commit so # _build_runner's dept index sees it before Phase B. if work_target.kind in ("feishu_contact", "huilianyi"): chain = await svc._person_dept_chain(person_id) # Skip the ensure when every dept in the chain is # ALREADY adopted — the common case for an existing # employee whose dept was synced by a prior full # sweep. Only a genuinely-unadopted dept (new hire / # never-synced BU) pays the cost (Feishu: ~2s catalog # prime). already = await svc._load_helix_dept_index( target_id=work_target.id, ) if chain and (chain - set(already)): try: if work_target.kind == "feishu_contact": await svc._run_feishu_dept_phase( target=work_target, run_id=run_row.id, restrict_dept_codes=chain, ) else: await svc._run_huilianyi_dept_phase( target=work_target, run_id=run_row.id, restrict_dept_codes=chain, ) await work_session.commit() except Exception: # noqa: BLE001 # Dept ensure is best-effort: a failure here # must not block the user push. Phase B still # runs on whatever's already adopted (and the # scheduled full sweep reconciles the rest). logger.exception( "sync_person_to_target: dept-chain " "ensure failed for person=%s target=%s; " "proceeding with Phase B on existing map", person_id, work_target.slug, ) await work_session.rollback() runner = await svc._build_runner(work_target) outcome = await svc._push_one_person( target=work_target, runner=runner, person_id=person_id, force_in_scope=force_in_scope, run_id=run_row.id, ) await work_session.commit() except LockBusy: # Another execution against the same target is in flight # (a scheduled sweep, manual full Sync Now, or another # per-person push). The in-flight run does NOT necessarily # cover this person (a concurrent single-person push for # someone else, or a sweep that snapshotted candidates # before this change committed) — the push was DROPPED. # # Whether that's benign depends on the target having a CRON # schedule to reconcile it: a cron target's next sweep re-pushes # this person, so a dropped push is a harmless "skipped" (green # run). But an ``event_only`` target has NO scheduled sweep — the # dropped push is lost forever, and finalising as skipped→ok # buries it behind a green run with no alert. For those, treat # the drop as a FAILURE so the run's failure notifier fires and # IT knows to re-trigger. (The failure notifier only fires on # failed/partial, never on ok.) _sched = target.schedule_config or {} _reconciled_by_cron = (_sched.get("trigger") or "manual") == "cron" # MUST-DELIVER (PR4): a dropped push is no longer just "skipped # and hope the next sweep catches it". Schedule a bounded delayed # RE-ENQUEUE so the push lands on its own once the in-flight run # releases the target lock (~seconds-to-minutes) — critical for # ``event_only`` targets (no sweep to reconcile them) and it # collapses offboard-revocation latency on cron targets from # "next daily sweep" to "~a minute". Bounded by # ``lockbusy_retries_left`` so a permanently-contended target # can't spawn an unbounded retry chain; when retries exhaust we # fall back to the old behaviour (cron → skipped/green; # event_only → failed so the notifier pages IT to re-trigger). if lockbusy_retries_left > 0: from app.core.background import spawn spawn( _retry_person_push_after_delay( target_id=target_id, person_id=person_id, trigger_source=trigger_source, force_in_scope=force_in_scope, retries_left=lockbusy_retries_left - 1, delay_s=_EVENT_PUSH_LOCKBUSY_RETRY_DELAY_S, ), name=f"identity_sync.lockbusy_retry:{target_id}:{person_id}", ) outcome = PersonOutcome( status="skipped", error=( f"target {target.slug!r} was locked by another sync " f"— push re-enqueued (retry in " f"{_EVENT_PUSH_LOCKBUSY_RETRY_DELAY_S}s, " f"{lockbusy_retries_left} attempt(s) left)." ), ) error_message = outcome.error logger.info( "sync_person_to_target LockBusy: target=%s person=%s — " "re-enqueued (%d retries left)", target.slug, person_id, lockbusy_retries_left - 1, ) else: outcome = PersonOutcome( status="skipped" if _reconciled_by_cron else "failed", error=( f"target {target.slug!r} is locked by another sync " "— this push was NOT executed after retries; " + ( "the person is covered by the next scheduled " "sweep (re-trigger manually if it can't wait)." if _reconciled_by_cron else "this target has NO scheduled sweep " "(event_only), so nothing will reconcile it — " "re-trigger the push." ) ), ) error_message = outcome.error logger.warning( "sync_person_to_target dropped on LockBusy after retries: " "target=%s person=%s (%s)", target.slug, person_id, "next sweep reconciles" if _reconciled_by_cron else "event_only — NOT reconciled, alerting", ) except AppException as e: error_message = f"{e.code}: {e.message}" outcome = PersonOutcome(status="failed", error=error_message) except asyncio.CancelledError: # Cooperative cancel — finalize the run row before # propagating so the row doesn't leak at status='running'. error_message = "cancelled (worker shutdown or task cancel)" outcome = PersonOutcome(status="failed", error=error_message) await self._finalise_run( run_row.id, status="failed", error_message=error_message, summary=RunSummary(persons_in_scope=1, failed=1, errors=[error_message]), ) if runner is not None: try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning("runner.aclose() raised post-cancel", exc_info=True) raise except Exception as e: # noqa: BLE001 logger.exception("sync_person_to_target failed") error_message = f"{type(e).__name__}: {e!s:.500}" outcome = PersonOutcome(status="failed", error=error_message) finally: if runner is not None: try: await runner.aclose() except Exception: # noqa: BLE001 logger.warning( "runner.aclose() raised", exc_info=True, ) # Build a one-person summary for the run row. # ``excluded`` (#1752) is a GREEN outcome — the runner refused the # person by policy (NS admin-lock, unadopted-404, create-skip # list, SF no-push / HRIS-purged design gate), not by error. # Aligning it here with _sweep stops every such refusal from # landing a false-red run + failure-alert card. # ``blocked`` is a GREEN outcome (data-dependent vendor refusal that # auto-clears on a data fix) — like ``excluded``, it must not land a # red run + failure alert. ``op`` distinguishes create vs update so a # single-person account creation isn't mislabelled "updated". _ok_outcomes = ( "success", "skipped", "deactivated", "deleted", "excluded", "blocked", ) summary = RunSummary(persons_in_scope=1) if outcome.status == "success": _op = outcome.detail.get("op") if isinstance(outcome.detail, dict) else None if _op == "create": summary.created = 1 else: summary.updated = 1 elif outcome.status == "skipped": summary.skipped = 1 elif outcome.status == "deactivated": summary.deactivated = 1 elif outcome.status == "deleted": summary.deleted = 1 elif outcome.status == "excluded": summary.excluded = 1 elif outcome.status == "blocked": summary.blocked = 1 else: summary.failed = 1 if outcome.error: summary.errors.append(outcome.error) await self._finalise_run( run_row.id, status="ok" if outcome.status in _ok_outcomes else "failed", error_message=error_message, summary=summary, ) # Audit row — IdentitySyncRun records the operational outcome # but the audit_logs table is the canonical "who did what" # surface (queryable by actor, joinable with PII reveal / # role-mutation gates). A single-person push is a real # downstream API call on real production user data; the audit # row makes the actor + target + person discoverable in a # single SELECT next to other admin actions. actor_uid: UUID | None = None if trigger_source.startswith("manual:") and not trigger_source.startswith("manual:mcp:"): try: actor_uid = UUID(trigger_source[len("manual:"):]) except ValueError: actor_uid = None elif trigger_source.startswith("manual:mcp:"): try: actor_uid = UUID(trigger_source[len("manual:mcp:"):]) except ValueError: actor_uid = None await self.audit.log( trace_id=uuid.uuid4(), actor_type="user" if actor_uid else "system", actor_user_id=actor_uid, action="identity_sync.person.push", status="success" if outcome.status in _ok_outcomes else "failed", resource_type="identity_sync_target", resource_id=str(target.id), detail={ "person_id": str(person_id), "outcome": outcome.status, "external_id": outcome.external_id, "trigger_source": trigger_source, "force_in_scope": force_in_scope, "error": _safe_error(outcome.error), }, ) await self.db.commit() return outcome async def _push_one_person( self, *, target: IdentitySyncTarget, runner: SyncRunner, person_id: UUID, force_in_scope: bool = False, run_id: UUID | None = None, ) -> PersonOutcome: """Render + push a single person, updating its state row. Used by sync_person_to_target. By default checks filter membership; with ``force_in_scope=True`` the filter check is skipped (admin explicit override). With ``run_id`` set, every vendor-touching path records a ledger row, and the returned outcome's ``detail`` carries the rendered ``request_payload`` so the caller (admin UI / MCP) can show the full round-trip without a second query. """ candidates, person_by_id, membership_by_person = await self._load_candidates() children_idx = await self._build_dept_children_index() filter_cfg = FilterConfig.from_dict(target.filter_config or {}) mapping_cfg = MappingConfig.from_dict(target.mapping_config or {}) dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) cand_match = next( (c for c in candidates if c.person_id == person_id), None, ) if cand_match is None: # #1756: cand_match None means either (a) the person is # genuinely absent, or (b) they EXIST but _load_candidates # excluded them by a DESIGN gate — SF ``customString24=2061`` # (不推送) or ``hris_record_state='purged'``. Case (b) reaches # here for every lifecycle event (offboard/hire) on every # event-aware target, because sync_person_to_target's first # lookup only checks deleted_at/work_email. Raising NotFound # here finalises the run ``failed`` with a misleading # "IdentityPerson not found" + a false-red alert card, once per # target. Instead return an ``excluded`` outcome citing the # gate — run stays green (parity with _sweep's excluded # semantics). Only a genuinely-absent person keeps NotFound. gated = await self.db.get(IdentityPerson, person_id) if gated is not None and gated.deleted_at is None: reason = _person_downstream_gate_reason(gated) if reason is not None: self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=gated.work_email or str(person_id), person_id=person_id, response_detail={"reason": reason}, ) return PersonOutcome(status="excluded", error=reason) raise NotFoundError("IdentityPerson", str(person_id)) natural_in_scope = matches( cand_match, filter_cfg, department_source_ids=dept_set, ) in_scope = force_in_scope or natural_in_scope # Leaver guard (2026-07-02; 2026-07-05 also covers leaver_pending): # a forced per-person push must never upsert a departing/departed # person the filter no longer matches — on vendors whose upsert falls # back to create (Feishu, SCIM), that re-creates a downstream account # offboarding has already revoked (or is mid-revoking). ``left`` AND # ``leaver_pending`` both qualify: offboarding is already in flight # for leaver_pending and its downstream accounts are being torn down, # so a force-push would race the teardown and rebuild the account. # Targets that legitimately carry leavers (include_offboarded / # explicit status lists) still match naturally and are unaffected; # the automatic offboard path (force_in_scope=False → deactivate/ # delete) is untouched. if ( force_in_scope and not natural_in_scope and cand_match.employment_status in ("left", "leaver_pending") ): self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=cand_match.work_email, person_id=cand_match.person_id, response_detail={ "reason": ( "leaver guard: person has left and is outside " "the target filter; forced push refused" ), }, ) return PersonOutcome( status="skipped", error=( "leaver guard: person has employment_status='left' and " "is outside this target's filter — refusing forced push " "(it would re-create an offboarded downstream account). " "If this target must carry leavers, set " "include_offboarded in its filter instead." ), ) # Load state row (don't create yet — see the no-op guard below). state = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, IdentitySyncState.person_id == person_id, ) )).scalars().first() person = person_by_id[person_id] # Event-fan-out no-op guard (PR4 must-deliver): a lifecycle event # (hire/role-change) fans out to EVERY event-aware target regardless # of each target's filter. For an ACTIVE person who is OUT of this # target's scope AND was NEVER linked here (no state row, or a bare # pending row with no external_id), there is nothing to do — routing # them into the offboard/deactivate path would (a) fabricate a state # row + a "deactivated" ledger entry for a person this target never # synced, and (b) on email-addressed vendors (SCIM/Navan) even # deactivate a same-email account belonging to a DIFFERENT channel # (a hire event revoking someone's unrelated account — finding #2). # So: a non-forced, out-of-scope, never-linked, STILL-ACTIVE person # is a clean no-op excluded. # # LEAVERS are deliberately EXCLUDED from this guard: an offboard # event for a left/leaver_pending person MUST reach the deactivate # path even with no prior link, because that path resolves the vendor # account BY EMAIL — catching an account created out-of-band (never # linked in Helix) that offboarding still has to shut. A forced admin # push, an in-scope person, or a previously-linked person all fall # through unchanged too. never_linked = state is None or ( state.external_id is None and state.last_status == "pending" ) if ( not in_scope and not force_in_scope and never_linked and person.employment_status == "active" ): self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, response_detail={ "reason": ( "event fan-out no-op: person is out of this target's " "scope and was never linked here — nothing to sync" ), }, ) return PersonOutcome( status="excluded", error="out of scope and never linked; event no-op", ) if state is None: state = IdentitySyncState( id=uuid.uuid4(), target_id=target.id, person_id=person_id, last_status="pending", ) self.db.add(state) if not in_scope: policy = (target.on_offboard or "deactivate").lower() if policy == "keep": state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": "out of scope; on_offboard=keep", }, ) return PersonOutcome(status="skipped", external_id=state.external_id) # Capability gate (mirrors the sweep's logic) — see the # comment in :meth:`_sweep` for why we collapse to # ``excluded`` rather than raising on unsupported ops. requested_supported = ( (policy == "deactivate" and runner.supports_deactivate) or (policy == "delete" and runner.supports_delete) ) if not requested_supported: state.last_status = "excluded" state.last_synced_at = datetime.now(UTC) self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="excluded", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": f"runner does not support {policy!r}", }, ) return PersonOutcome( status="skipped", external_id=state.external_id, error=( f"runner does not support {policy!r}; " "treated as excluded." ), ) # Leaver notice-period guard (parity with _sweep's out-of-scope # loop, #1735): a ``leaver_pending`` person whose local last # working day has NOT passed must NOT be deactivated yet — that # locks a still-working employee out days-to-weeks early (the # 2026-05-28 Thalie Zhuang class). ``left`` is past by definition # and flows through; only the notice window is deferred. Latent # today (event reasons fire for active/left only) but closed here # so any future event reason (dept-move / status-change) or a # force_in_scope=False caller can't re-open it. if ( person.employment_status == "leaver_pending" and not _leaver_last_working_day_passed(person) ): self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="skipped", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, response_detail={ "reason": ( "leaver notice-period guard: last working day " "not yet passed; deferring deactivate" ), }, ) return PersonOutcome( status="skipped", external_id=state.external_id, error=( "leaver notice-period guard: deferring deactivate " "until the local last working day passes" ), ) offboard_t0 = time.monotonic() if policy == "delete": outcome = await runner.delete( person_email=person.work_email, prior_external_id=state.external_id, ) else: outcome = await runner.deactivate( person_email=person.work_email, prior_external_id=state.external_id, ) state.last_synced_at = datetime.now(UTC) prior_external_id = state.external_id if outcome.status in ("deactivated", "deleted"): state.last_status = outcome.status state.last_error = None if outcome.status == "deleted": state.external_id = None elif outcome.status == "excluded": # #1752 — vendor refused this offboard by policy (e.g. # NetSuite Administrator-record lock), not by error. Parity # with _sweep's out-of-scope excluded branch: excluded, not # failed; retry_count reset so the retry worker doesn't # churn on an unreachable row; run stays green. state.last_status = "excluded" state.last_error = None state.retry_count = 0 else: state.last_status = "failed" state.last_error = _safe_error(outcome.error) # Parity with the sweep's offboard failed branch: count # the retry so the retry worker / UI see the same # escalation signal a sweep-failed deactivate produces. state.retry_count = (state.retry_count or 0) + 1 self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome=( outcome.status if outcome.status in ("deactivated", "deleted", "excluded") else "failed" ), subject_display=person.work_email, person_id=person.id, external_key=prior_external_id, response_detail=outcome.detail, error=outcome.error, duration_ms=int((time.monotonic() - offboard_t0) * 1000), ) return outcome # In scope — render + push. membership = membership_by_person.get(person_id) managers = await self._resolve_manager_emails([person_id]) mgr_email, mgr_name = managers.get(person_id, (None, None)) hrbps = await self._resolve_hrbp_emails([person_id]) hrbp_email, hrbp_name = hrbps.get(person_id, (None, None)) ctx = build_person_context( person, org_path=membership.org_path if membership else None, manager_email=mgr_email, manager_name=mgr_name, hrbp_email=hrbp_email, hrbp_name=hrbp_name, org_path_drop_root=(target.runner_options or {}).get("drop_root_segments") or [], org_node_source_id=( membership.org_node_source_id if membership else None ), ) # 汇联易 主岗直属领导 — parity with _sweep, so a single-person push # (hire event / manual "sync this person") renders the SAME payload # the nightly sweep would. Without this the two paths would disagree # and ping-pong the field every night. if target.kind == "huilianyi": mgr_no_hly = ( await self._resolve_pushable_manager_nos( target.filter_config, [person_id], ) ).get(person_id) if mgr_no_hly: ctx["manager_employee_no"] = mgr_no_hly # UPDATE shape for already-linked persons — parity with _sweep: # ``create_only`` rules are omitted so merge-semantics vendors # preserve their existing value. payload = render_payload( mapping_cfg, ctx, exclude_create_only=bool(state.external_id), ) if target.kind == "feishu_contact": payload["_helix_residence_country"] = ( ctx.get("residence_country") or "" ) # Write-mode gate (#1736) — parity with _sweep (the sync-template # matrix's operation dimension). The single-person path (lifecycle # event hooks, admin "Sync this person", MCP) previously had NO # write_mode check and called ``runner.upsert`` directly, so an # event push would CREATE on an ``update_only`` target (configured # "never build accounts") or re-touch a linked record on a # ``create_only`` target ("build once, never re-touch"). Evaluated # on Helix's per-(target,person) link state, before any vendor # call, exactly as the sweep does. ``force_in_scope`` overrides the # FILTER, never the write_mode — the target-level "never # create/never re-touch" constraint still holds for an explicit # push, matching the sweep. _wm_skip = None if target.write_mode == "create_only" and state.external_id: _wm_skip = "write_mode=create_only (already linked)" elif target.write_mode == "update_only" and not state.external_id: _wm_skip = "write_mode=update_only (not yet linked)" if _wm_skip: state.last_status = "skipped" _wm_now = datetime.now(UTC) state.last_synced_at = _wm_now # Parity with the sweep's write-mode gate: a write-mode skip # is a deliberate no-touch of an already-decided link state, # so bump the drift-reverify clock too (see _sweep). state.last_verified_at = _wm_now state.last_error = None self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome="skipped", subject_display=person.work_email, person_id=person.id, external_key=state.external_id, request_payload=payload, response_detail={"reason": _wm_skip}, ) return PersonOutcome( status="skipped", external_id=state.external_id, error=_wm_skip, ) # Required-field preflight parity with the sweep: a mapping field # marked ``required`` that renders to nothing is a hard BLOCK, not # a burned vendor call that 400s. Auto-clears when the data is fixed. # Mirrors the render_payload shape above — create_only rules aren't # in a linked person's UPDATE payload, so they can't drop. _, _dropped_required = collect_render_diagnostics( mapping_cfg, ctx, exclude_create_only=bool(state.external_id), ) # Hash-skip parity with the sweep is intentionally absent: an # explicit single-person push means "talk to the vendor NOW" — # the admin is verifying the round-trip, not saving API calls. was_new = state.last_payload_hash is None push_t0 = time.monotonic() if _dropped_required: _reason = ( "required field(s) not rendered: " + "; ".join(f"{tf} ({why})" for tf, why in _dropped_required) + " — fix the person's HRIS data or the target's mapping " "value_map, then re-sync." ) outcome = PersonOutcome( status="blocked", error=_reason, detail={ "blocked_reason": "required_field_missing", "fields": [tf for tf, _ in _dropped_required], }, ) else: outcome = await runner.upsert( person_email=person.work_email, payload=payload, prior_external_id=state.external_id, ) push_duration_ms = int((time.monotonic() - push_t0) * 1000) # State bookkeeping mirrors the sweep's in-scope branches # (success/skipped/excluded bump ``last_verified_at`` — a real # vendor touch resets the drift re-verify clock exactly as a # sweep push does; without it every single-person-pushed row # aged into a "stale, re-verify" full vendor round-trip on the # next sweep even though it was verified minutes ago). _push_now = datetime.now(UTC) state.last_synced_at = _push_now if outcome.status == "success": state.last_status = "success" state.last_payload_hash = hash_payload(payload) state.last_verified_at = _push_now if outcome.external_id: state.external_id = outcome.external_id state.retry_count = 0 # Slot-degrade parity with the sweep: a partial write clears # last_verified_at (force re-verify next sweep) + keeps the # note; a clean success bumps the drift clock + clears error. _dropped_slots = [ s for s in (outcome.slot_outcomes or []) if isinstance(s, dict) and s.get("status") in ("dropped", "drift", "failed") ] if _dropped_slots: state.last_verified_at = None state.last_error = _safe_error(outcome.error or "partial write") else: state.last_verified_at = datetime.now(UTC) state.last_error = None # Hire-day group membership — the event path runs without # the full group phase, so a new hire would otherwise sit # outside their dept's SCIM group (= no AWS permission # sets) until the nightly sweep. Best-effort: the nightly # replace-all self-heals anything missed here. if ( target.kind in ("scim", "navan") and (target.runner_options or {}).get("push_groups") and state.external_id and membership is not None and membership.org_node_source_id ): await self._add_member_to_dept_group( target=target, runner=runner, dept_code=membership.org_node_source_id, scim_user_id=state.external_id, ) elif outcome.status == "skipped": state.last_status = "skipped" state.last_payload_hash = hash_payload(payload) state.last_verified_at = _push_now state.last_error = _safe_error(outcome.error) elif outcome.status == "excluded": # #1752 — the runner refused this person by policy, not by # error (NetSuite admin-lock / unadopted-404 / create-skip # list). Parity with _sweep's in-scope excluded branch: never # failed, retry_count=0, run stays green, and the hash is # recorded so a re-run with an unchanged payload re-affirms # excluded rather than re-issuing the same refused call. state.last_status = "excluded" state.last_payload_hash = hash_payload(payload) state.last_verified_at = _push_now state.last_error = None state.retry_count = 0 if outcome.external_id and not state.external_id: state.external_id = outcome.external_id elif outcome.status == "blocked": # Data-dependent vendor refusal — parity with _sweep: record the # hash so the unchanged-payload re-affirms cheaply, retry_count # stays 0 (no churn), keep the actionable reason, run stays green. state.last_status = "blocked" state.last_payload_hash = hash_payload(payload) state.last_verified_at = _push_now state.last_error = _safe_error(outcome.error) state.retry_count = 0 if outcome.external_id and not state.external_id: state.external_id = outcome.external_id else: state.last_status = "failed" state.last_error = _safe_error(outcome.error) state.retry_count = (state.retry_count or 0) + 1 # Observability parity with the sweep path: log the ACTUAL wire body # + vendor response (+ per-slot degradation outcomes) the runner # captured, not the pre-render payload with a null response — so the # admin "Sync this person" / MCP / onboarding-provisioning path is as # diagnosable as the cron sweep. Falls back to the rendered payload + # outcome.detail for runners not yet on the capture protocol. _wire_req = outcome.request_body or getattr( runner, "_last_write_request", None, ) _wire_resp = outcome.response or getattr( runner, "_last_write_response", None, ) if outcome.slot_outcomes: _base = dict(_wire_resp) if isinstance(_wire_resp, dict) else {} _base["slot_outcomes"] = outcome.slot_outcomes _wire_resp = _base # Prefer the runner's REAL vendor op over ``was_new`` — same rule the # sweep's ledger row uses, and the same rule THIS function's run # summary already used (they disagreed until 2026-08-27: a mapping # config change NULLs the payload hash, so ``was_new`` alone labelled # an ordinary PUT update 「已创建」 and read as "a duplicate employee # was made on the vendor"). _item_op = ( outcome.detail.get("op") if isinstance(outcome.detail, dict) else None ) _item_created = ( (_item_op == "create") if _item_op in ("create", "update") else was_new ) self._add_run_item( run_id=run_id, target_id=target.id, item_type="person", outcome=( ("created" if _item_created else "updated") if outcome.status == "success" else outcome.status ), subject_display=person.work_email, person_id=person.id, external_key=outcome.external_id or state.external_id, request_payload=_wire_req if _wire_req is not None else payload, response_detail=( _wire_resp if _wire_resp is not None else outcome.detail ), error=outcome.error, duration_ms=push_duration_ms, ) # Echo the rendered payload back so the caller can render the # full round-trip inline (admin push modal / MCP result). outcome.detail = { **(outcome.detail or {}), "request_payload": payload, "duration_ms": push_duration_ms, } return outcome # ── Dispatch tick ──────────────────────────────────────────────── async def list_due_targets(self, *, now: datetime) -> list[IdentitySyncTarget]: """Targets whose schedule says fire-now AND that aren't already running (best-effort — the per-target Redis lock is the authoritative single-flight gate).""" rows = (await self.db.execute( select(IdentitySyncTarget).where( IdentitySyncTarget.deleted_at.is_(None), IdentitySyncTarget.enabled.is_(True), IdentitySyncTarget.next_run_at.is_not(None), IdentitySyncTarget.next_run_at <= now, ) )).scalars().all() # Filter to only cron-trigger targets (event_only / manual # shouldn't have next_run_at set, but defense in depth). return [ r for r in rows if (r.schedule_config or {}).get("trigger") == "cron" ] async def run_due_targets( self, ) -> dict[str, Any]: """Master-tick entry point. Walks due targets and runs each one under a per-target distributed lock. Designed to be called every minute by ``identity_sync.dispatch``. Targets are processed STRICTLY SEQUENTIALLY within a tick by design — do not switch this loop to ``asyncio.gather``. The single-flight guarantee depends on the framework's outer ``sched_job:identity_sync.dispatch`` lock being held across the entire tick; parallel target execution would also blow through downstream rate limits when multiple targets share an SCIM endpoint family. Slow targets simply push subsequent targets to the next tick — the framework's 60s interval is deliberately short enough to make this drift acceptable. """ from app.core.distributed_lock import LockBusy, distributed_lock now = datetime.now(UTC) due = await self.list_due_targets(now=now) # Sort by kind priority so dept targets land before user # targets in the same tick (the user runner needs depts to # exist for its ``department_ids`` push). Within the same # kind, fall back to ``next_run_at`` so earlier-due targets # still come first. due = sorted( due, key=lambda t: ( _KIND_DISPATCH_PRIORITY.get(t.kind, 100), t.next_run_at or now, t.slug, ), ) ran: list[str] = [] ran_failed: list[str] = [] skipped: list[str] = [] failed: list[str] = [] for target in due: lock_key = f"identity_sync:target:{target.slug}" try: # auto_renew (#1754): a full sweep can exceed the 900s TTL # (a mapping change nulls every last_payload_hash → full # re-push; SCIM worst case POST→409→GET→PUT ×N at 60/min). # Renew the lock so the next dispatch tick keeps hitting # LockBusy instead of starting a second concurrent sweep. async with distributed_lock( lock_key, ttl=900, auto_renew=True, ): try: run_row = await self.run_target( target, trigger_source="scheduler", ) # A run reaches the failure notifier when it # finalised WITHOUT raising but left something # broken behind — either the whole target is down # (``failed``: breaker open / mass-action brake / # config error / not one vendor call got through) # or the target is up and specific people did not # sync (``partial``). Without this the job run # lands status='ok' and nobody hears: a revoked # vendor credential could silently kill the nightly # leaver-lockout backstop for weeks. # # ``partial`` deliberately counts too, and the # per-target ``alert_notifier`` is NOT the reason # it is safe to: that path sends to # ``alert_recipient_ids`` only, with no admin # default by design (DECISIONS 2026-06-19), and # every production target has that list empty — so # for ``partial`` it currently reaches nobody. The # automation-exception ledger does record the # failed run ITEM and route it to an owner, but # that is a review queue on a collector cadence, # not a page: it is the follow-up, not the alarm. if ( run_row is not None and run_row.status in _RUN_STATUSES_WITH_FAILURES ): ran_failed.append( f"{target.slug}: " f"{(run_row.error_message or 'failed')[:200]}" ) else: ran.append(target.slug) except Exception as e: # noqa: BLE001 logger.exception( "dispatch run_target %s raised", target.slug, ) failed.append(f"{target.slug}: {e!s:.200}") except LockBusy: skipped.append(f"{target.slug} (locked)") continue return { "due": len(due), "ran": ran, "ran_failed": ran_failed, "skipped": skipped, "failed": failed, } # ── Dry-run (impact preview before "Sync now") ─────────────────── async def dry_run( self, target_id: UUID, *, sample_size: int = 20, full_reconcile: bool = False, ) -> dict[str, Any]: """Compute what a full sweep WOULD do, without calling the runner. Returns the in-scope / out-of-scope split + a sample of each group + the on_offboard policy that would apply. This is the safety net behind the "Sync now" confirmation dialog. It costs zero downstream API calls — purely local candidate selection + filter evaluation against the same snapshot a real run would see. ``full_reconcile`` mirrors the run-endpoint flag: a manual run is push-only by default (never deactivates out-of-scope people), so the preview must not claim otherwise. When the admin asks for a full reconcile the out-of-scope section shows the real ``on_offboard`` consequence again. """ target = await self.get_target(target_id) try: filter_cfg = FilterConfig.from_dict(target.filter_config or {}) except FilterError as e: raise AppException( code="IDENTITY_SYNC_INVALID_FILTER", message=str(e), status_code=400, ) from e candidates, person_by_id, membership_by_person = await self._load_candidates() children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) # Existing state rows for this target — needed BEFORE the # in-scope split so the preview can mirror _sweep()'s # future-hire deferral (first-time creation only; a person # with a state row is never deferred). state_rows = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() state_person_ids = {s.person_id for s in state_rows} _override_ids = set(filter_cfg.person_ids) _lead_hours = await get_provision_lead_hours(self.db) in_scope_persons: list[Any] = [] deferred_future_hires: list[dict[str, Any]] = [] for cand in candidates: if not matches(cand, filter_cfg, department_source_ids=dept_set): continue _p = person_by_id.get(cand.person_id) if ( cand.person_id not in state_person_ids and str(cand.person_id) not in _override_ids and _p is not None and is_future_hire(_p, lead_hours=_lead_hours) ): deferred_future_hires.append({ "person_id": str(cand.person_id), "email": _p.work_email, "name": _p.display_name or _p.work_email or str(cand.person_id), "hire_date": _p.hire_date.isoformat() if _p.hire_date else None, }) continue in_scope_persons.append(cand) in_scope_ids = {c.person_id for c in in_scope_persons} # Approver auto-include (#1753) — the real sweep pulls in active # managers of in-scope reports (+ rehydrates prior auto-included # rows) and upserts them. The preview MUST simulate the same set, # or the "Sync now" confirm dialog understates real impact (an # admin approves 150 and the sweep quietly creates 5 more manager # accounts + fires their "account auto-created" notifications). # Reuse the exact helper the sweep uses so the two never drift. auto_included_manager_ids = self._compute_auto_included_manager_ids( in_scope_persons=in_scope_persons, in_scope_ids=in_scope_ids, person_by_id=person_by_id, state_rows=state_rows, filter_cfg=filter_cfg, ) auto_included_persons: list[Any] = [] for mid in auto_included_manager_ids: mgr = person_by_id.get(mid) if mgr is None: continue mgr_membership = membership_by_person.get(mid) auto_included_persons.append(CandidatePerson( person_id=mid, work_email=mgr.work_email, employment_status=mgr.employment_status, country=mgr.country, job_title=mgr.job_title, employee_no=mgr.employee_no, company_code=mgr.company_code, employee_type_code=mgr.employee_type_code, hire_date=mgr.hire_date, hris_person_id=mgr.hris_person_id, org_path=mgr_membership.org_path if mgr_membership else None, org_node_source_id=( mgr_membership.org_node_source_id if mgr_membership else None ), )) # Add to in_scope_ids so an auto-included manager who happens # to carry a prior (non-auto) state row is not double-counted # as out-of-scope below — parity with _sweep. in_scope_ids.add(mid) out_of_scope_states = [ s for s in state_rows if s.person_id not in in_scope_ids # mirror _sweep() exactly — incl. terminal ``deactivated`` and s.last_status not in ("excluded", "deleted", "deactivated") and not s.auto_included_reason ] def _name(pid: UUID) -> str: p = person_by_id.get(pid) if p is None: return str(pid) # Human label = the golden-record display_name ("孟凡生 Jason # Meng"), NOT name_pinyin ("mengfansheng jason meng") and NOT # the email-derived ctx.full_name ("Fansheng Meng"). The # latter two are romanisations the runner synthesises for # SCIM given/family splitting — misleading as a person label. return p.display_name or p.name_pinyin or p.work_email or str(pid) # Build a person_id → last_status dict once. The previous # implementation linear-scanned ``state_rows`` per sample, so # at sample_size=200 + state_rows=500 we did 100k iterations # per dry-run; the dict makes it O(n + samples). status_by_person: dict[UUID, str] = { s.person_id: s.last_status for s in state_rows } # Linked-vendor-record map so the payload preview mirrors the real # push shape: a linked person is pushed with the UPDATE shape, so # create_only fields must be excluded from the preview too. external_by_person: dict[UUID, str] = { s.person_id: s.external_id for s in state_rows if s.external_id } def _status_for(pid: UUID) -> str | None: return status_by_person.get(pid) # Render the actual outbound payload for each sample person so # admins can see the full JSON they're about to push, not just # name + email + country. This is the difference between "I # think it'll work" and "I see exactly what Navan / Google # will receive." Render is pure-data — zero API calls. try: mapping_cfg = MappingConfig.from_dict(target.mapping_config or {}) except MappingError as e: raise AppException( code="IDENTITY_SYNC_INVALID_MAPPING", message=str(e), status_code=400, ) from e # Resolve managers in one batch for the in-scope sample so the # rendered payload includes manager_email / manager_name. sample_in_scope = in_scope_persons[:sample_size] sample_person_ids = [c.person_id for c in sample_in_scope] # Out-of-scope sample also gets rendered context (no payload — # they'd be deactivated, not upserted) but we still surface the # current snapshot so the admin sees who they're about to lose. sample_oos_ids = [s.person_id for s in out_of_scope_states[:sample_size]] # Auto-included managers are UPSERTED (with a rendered payload), # so their sample surfaces the same rich context as in-scope rows. sample_auto = auto_included_persons[:sample_size] sample_auto_ids = [c.person_id for c in sample_auto] _resolve_ids = list( set(sample_person_ids + sample_oos_ids + sample_auto_ids) ) managers_by_person = await self._resolve_manager_emails(_resolve_ids) hrbp_by_person = await self._resolve_hrbp_emails(_resolve_ids) manager_identity_by_person = await self._resolve_manager_identity( _resolve_ids ) # 汇联易 主岗直属领导 — the preview must render the same payload the # sweep will send, or the admin reviews a payload that doesn't exist. hly_manager_nos: dict[UUID, str] = ( await self._resolve_pushable_manager_nos( target.filter_config, _resolve_ids, ) if target.kind == "huilianyi" else {} ) runner_drop_root = (target.runner_options or {}).get( "drop_root_segments", ) or [] def _person_summary(pid: UUID, *, with_payload: bool) -> dict[str, Any]: """Build a rich preview row for one person. ``with_payload=True`` renders the outbound JSON via the target's mapping rules so the admin sees exactly what's about to land in the downstream system. ``False`` (out- of-scope) skips render — those people receive a deactivate / delete / keep, not a payload. """ p = person_by_id.get(pid) if p is None: return { "person_id": str(pid), "email": None, "name": str(pid), "current_status": _status_for(pid), } membership = membership_by_person.get(pid) mgr_email, mgr_name = managers_by_person.get(pid, (None, None)) hrbp_email, hrbp_name = hrbp_by_person.get(pid, (None, None)) mgr_employee_no, mgr_employment_status = ( manager_identity_by_person.get(pid, (None, None)) ) ctx = build_person_context( p, org_path=membership.org_path if membership else None, manager_email=mgr_email, manager_name=mgr_name, hrbp_email=hrbp_email, hrbp_name=hrbp_name, org_path_drop_root=runner_drop_root, org_node_source_id=( membership.org_node_source_id if membership else None ), ) if hly_manager_nos.get(pid): ctx["manager_employee_no"] = hly_manager_nos[pid] row: dict[str, Any] = { "person_id": str(pid), "email": p.work_email, # Golden-record display_name (via _name) — the authoritative # human label. ctx.full_name is the email-derived romanisation # ("Fansheng Meng") the runner uses only for SCIM name-splitting; # first_name / last_name below still carry it for debugging. "name": _name(pid), "first_name": ctx.get("first_name"), "last_name": ctx.get("last_name"), "job_title": p.job_title, "country": p.country, "city": p.city, "location": p.work_location_name, "company_name": p.company_name, "company_code": p.company_code, "cost_center": p.cost_center, "business_unit": ctx.get("business_unit"), "department_l1": ctx.get("department_l1"), "department_l2": ctx.get("department_l2"), "department_l3": ctx.get("department_l3"), "department_l4": ctx.get("department_l4"), "manager_email": mgr_email, "manager_name": mgr_name, "manager_employee_no": mgr_employee_no, "manager_employment_status": mgr_employment_status, "employment_status": p.employment_status, "hire_date": p.hire_date.isoformat() if p.hire_date else None, "termination_date": ( p.termination_date.isoformat() if p.termination_date else None ), "current_status": _status_for(pid), } if with_payload: try: row["rendered_payload"] = render_payload( mapping_cfg, ctx, exclude_create_only=bool(external_by_person.get(pid)), ) except MappingError as e: row["rendered_payload"] = None row["render_error"] = str(e) return row in_scope_sample = [ _person_summary(c.person_id, with_payload=True) for c in sample_in_scope ] auto_included_sample = [ { **_person_summary(c.person_id, with_payload=True), "auto_include_reason": "manager_of_foreign", } for c in sample_auto ] out_of_scope_sample = [] for s in out_of_scope_states[:sample_size]: row = _person_summary(s.person_id, with_payload=False) row["current_status"] = s.last_status row["external_id"] = s.external_id out_of_scope_sample.append(row) groups_plan: dict[str, Any] | None = None if ( target.kind in ("scim", "navan") and (target.runner_options or {}).get("push_groups") ): # Local-only group impact (zero vendor calls — the group # phase's own member source IS local state, and AWS IdC # doesn't return members anyway, so this preview matches # what a real run would compute). all_nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(all_nodes), ) scoped_nodes = [ n for n in all_nodes if n.org_node_source_id in scoped_ids ] name_counts: dict[str, int] = {} for n in scoped_nodes: name_counts[n.org_node_name or ""] = ( name_counts.get(n.org_node_name or "", 0) + 1 ) dup_names = sorted( nm for nm, c in name_counts.items() if c > 1 and nm ) dept_states = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() dept_state_by_code = {d.dept_code: d for d in dept_states} scim_ids = { st.person_id: st.external_id for st in state_rows if st.external_id and st.last_status not in ("deactivated", "deleted") } member_counts: dict[str, int] = {} for cand in candidates: if cand.person_id in scim_ids and cand.org_node_source_id: member_counts[cand.org_node_source_id] = ( member_counts.get(cand.org_node_source_id, 0) + 1 ) group_rows = [] would_create_groups = 0 for n in scoped_nodes: code = n.org_node_source_id st = dept_state_by_code.get(code) is_new = st is None or not st.external_id if is_new: would_create_groups += 1 group_rows.append({ "name": n.org_node_name, "src_id": code, "would_create": is_new, "member_count": member_counts.get(code, 0), }) groups_plan = { "in_scope_count": len(scoped_nodes), "would_create_count": would_create_groups, "duplicate_display_names": dup_names, "groups_sample": group_rows[:sample_size], "note": ( "组阶段是全量替换式成员写入。成员数基于当前已同步" "的用户状态 —— 全新目标此处显示 0,因为真实运行会" "先跑用户阶段再建组,首跑的实际成员数以用户阶段" "结果为准(IdC 不回读成员,无法显示供应商现状)。" ), } return { "target_id": str(target.id), "target_slug": target.slug, "on_offboard": target.on_offboard, # Total real upsert impact = filter matches + auto-included # managers (parity with _sweep.summary.persons_in_scope, which # counts both). #1753: previously omitted the managers, so the # confirm dialog understated the real create count. "in_scope_count": len(in_scope_persons) + len(auto_included_persons), "in_scope_sample": in_scope_sample, # Approver auto-include set (#1753) surfaced separately so the # admin sees exactly which out-of-filter managers the run will # ALSO create/update (and notify). "auto_included_count": len(auto_included_persons), "auto_included_sample": auto_included_sample, "out_of_scope_count": len(out_of_scope_states), "out_of_scope_sample": out_of_scope_sample, # What the manual run this dialog gates will ACTUALLY do to # the out-of-scope set: push-only runs skip them entirely. "manual_run_push_only": not full_reconcile, "out_of_scope_action": ( target.on_offboard if full_reconcile else "keep_push_only" ), "deferred_future_hire_count": len(deferred_future_hires), "deferred_future_hire_sample": deferred_future_hires[:sample_size], "existing_state_count": len(state_rows), "candidates_total": len(candidates), "filter": filter_cfg.to_dict(), "groups": groups_plan, } async def dry_run_feishu( self, target_id: UUID, *, sample_size: int = 50, ) -> dict[str, Any]: """Full read-only sync plan for a ``feishu_contact`` target. Computes EXACTLY what a real run would do — same filter (incl. the employee_no gate), same dept scoping, same adopt-by-name — but issues only read GETs (dept tree walk + user batch_get_id). No writes to Feishu or TiDB. Backs the "同步前预览" panel so an admin sees, before flipping the switch, precisely which users / departments get created vs updated vs excluded vs skipped. Returns separate ``users`` and ``departments`` plans + a ``warnings`` list (e.g. soft-delete mass-action brake tripped). """ target = await self.get_target(target_id) if target.kind != "feishu_contact": raise AppException( code="IDENTITY_SYNC_NOT_FEISHU", message=( f"dry_run_feishu only applies to feishu_contact " f"targets; {target.slug!r} is kind={target.kind!r}" ), status_code=400, ) try: filter_cfg = FilterConfig.from_dict(target.filter_config or {}) except FilterError as e: raise AppException( code="IDENTITY_SYNC_INVALID_FILTER", message=str(e), status_code=400, ) from e warnings: list[str] = [] # ── Resolve Feishu client (read-only use). ────────────────── from app.approval.feishu_client import FeishuApprovalClient from app.domains.identity_sync.runners.feishu_department import ( FeishuDepartmentRunner, ) if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message="feishu_contact target has no integration_id set.", status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=f"integration {target.integration_id} not found.", status_code=400, ) resolved = await _resolve_integration(self.db, integ) app_id = resolved.config.get("app_id") app_secret = resolved.config.get("app_secret") if not app_id or not app_secret: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message="feishu integration missing app_id/app_secret.", status_code=400, ) client = FeishuApprovalClient(app_id=app_id, app_secret=app_secret) try: # ── USERS ─────────────────────────────────────────────── candidates, person_by_id, _membership = await self._load_candidates() children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) def _disp(pid: UUID) -> str: p = person_by_id.get(pid) if p is None: return str(pid) # display_name first so the preview shows the authoritative # name ("孟凡生 Jason Meng"), not the pinyin romanisation # ("mengfansheng jason meng"). Mirrors _name in dry_run. return p.display_name or p.name_pinyin or p.work_email or str(pid) # Mirror _sweep()'s future-hire deferral so the preview does # not claim "would create" for someone the run will defer to # their start date (first-time creation only; a person with a # state row for this target is never deferred). _fh_state_ids = { # ``select(...person_id)`` + ``.scalars()`` yields the # UUID column values directly — iterate them, do NOT # re-access ``.person_id`` (that AttributeError'd the # whole dry-run 500). pid for pid in (await self.db.execute( select(IdentitySyncState.person_id).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() } _fh_override_ids = set(filter_cfg.person_ids) _lead_hours = await get_provision_lead_hours(self.db) deferred_future_hires: list[dict[str, Any]] = [] in_scope = [] for c in candidates: if not matches(c, filter_cfg, department_source_ids=dept_set): continue _p = person_by_id.get(c.person_id) if ( c.person_id not in _fh_state_ids and str(c.person_id) not in _fh_override_ids and _p is not None and is_future_hire(_p, lead_hours=_lead_hours) ): deferred_future_hires.append({ "person_id": str(c.person_id), "email": _p.work_email, "name": _disp(c.person_id), "hire_date": ( _p.hire_date.isoformat() if _p.hire_date else None ), }) continue in_scope.append(c) # Accounts the employee_no gate specifically removes — the # "these are filtered out as non-employees" list. excluded_nonemployee: list[dict[str, Any]] = [] if filter_cfg.employee_no_pattern: pat = re.compile(filter_cfg.employee_no_pattern) for c in candidates: emp = (c.employee_no or "").strip() if not emp or not pat.fullmatch(emp): excluded_nonemployee.append({ "person_id": str(c.person_id), "email": c.work_email, "employee_no": c.employee_no, "name": _disp(c.person_id), }) # Live read-only lookup: who already exists on Feishu (by the # SAME email path the runner's dedup uses) → update vs create. # Mirror the runner's .com/.com.cn corporate-sibling tolerance # (FeishuApprovalClient.resolve_user_id_by_email): query both # sibling addresses and match by local-part within the pingcap # pair — otherwise a person whose Feishu account is @pingcap.com.cn # (while Helix stores @pingcap.com) is wrongly reported as "would # create" here even though the runner now finds + updates them. from app.approval.feishu_client import _PINGCAP_EMAIL_SIBLING emails: set[str] = set() for c in in_scope: if not c.work_email: continue e = c.work_email.strip().lower() emails.add(e) loc, _, dom = e.partition("@") sib = _PINGCAP_EMAIL_SIBLING.get(dom) if sib and loc: emails.add(f"{loc}@{sib}") found_emails: set[str] = set() found_pingcap_locals: set[str] = set() email_list = sorted(emails) for i in range(0, len(email_list), 50): chunk = email_list[i:i + 50] try: r = await client.request( "POST", "/contact/v3/users/batch_get_id?user_id_type=user_id", json_body={"emails": chunk}, ) except AppException as e: warnings.append( f"user lookup batch failed ({e.code}); create/update " "split for that batch is approximate" ) continue for u in (r.get("user_list") or []): em = (u.get("email") or "").lower() if em and u.get("user_id"): found_emails.add(em) eloc, _, edom = em.partition("@") if edom in _PINGCAP_EMAIL_SIBLING and eloc: found_pingcap_locals.add(eloc) # Provisioning policy (Authing parity): the admin skip-list # suppresses CREATE/REOPEN for named person_ids, and the # reopen_frozen toggle asserts active for in-scope people. # Reflect both in the plan so the preview shows exactly which # "would create" rows are held back by the skip-list. _ropts = target.runner_options or {} _reopen_frozen = bool(_ropts.get("reopen_frozen", False)) _skip_pids = { str(x) for x in (_ropts.get("skip_provision_person_ids") or []) } would_update: list[dict[str, Any]] = [] would_create: list[dict[str, Any]] = [] would_skip_create: list[dict[str, Any]] = [] for c in in_scope: wl = (c.work_email or "").strip().lower() row = { "person_id": str(c.person_id), "email": c.work_email, "employee_no": c.employee_no, "name": _disp(c.person_id), } _wloc, _, _wdom = wl.partition("@") _found = wl in found_emails or ( _wdom in _PINGCAP_EMAIL_SIBLING and _wloc in found_pingcap_locals ) if _found: would_update.append(row) elif str(c.person_id) in _skip_pids: # On the skip-list → not created (would be a no-op # 'skipped' in a real run). Surfaced separately so the # admin sees the suppression is deliberate. would_skip_create.append(row) else: would_create.append(row) users_plan = { "in_scope_count": len(in_scope), "would_update_count": len(would_update), "would_create_count": len(would_create), # The create list is the eyeball set — people not yet on # Feishu by email. Sample capped; full set via CSV later. "would_create_sample": would_create[:sample_size], # Skip-listed people that WOULD have been created but are # suppressed by runner_options.skip_provision_person_ids. "would_skip_create_count": len(would_skip_create), "would_skip_create_sample": would_skip_create[:sample_size], "excluded_nonemployee_count": len(excluded_nonemployee), "excluded_nonemployee": excluded_nonemployee[:sample_size], "deferred_future_hire_count": len(deferred_future_hires), "deferred_future_hire_sample": ( deferred_future_hires[:sample_size] ), "employee_no_pattern": filter_cfg.employee_no_pattern, # reopen_frozen: when true, in-scope HRIS-active people # who are frozen on Feishu get reopened (打开) — except any # on the skip-list. Surfaced so the admin knows the toggle # state before running. "reopen_frozen": _reopen_frozen, } # ── DEPARTMENTS ───────────────────────────────────────── all_nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(all_nodes), ) scoped_nodes = [ n for n in all_nodes if n.org_node_source_id in scoped_ids ] skipped_nodes = [ n for n in all_nodes if n.org_node_source_id not in scoped_ids ] dept_state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() state_by_code = {s.dept_code: s for s in dept_state_rows} # Prime the live Feishu catalog (read-only BFS) so adopt-by- # name classification matches the real runner exactly. dept_runner = FeishuDepartmentRunner(client=client) await dept_runner.prime_catalog() would_create_dept: list[dict[str, Any]] = [] would_adopt_dept: list[dict[str, Any]] = [] # Classify exactly as the real runner would: sibling-scoped # (parent_od, name) match + globally-unique flat-name # fallback (see FeishuDepartmentRunner.match_existing). # Processed parent-before-child (topo sort) with a running # HRIS dept_code → Feishu od map — seeded from persisted # adoptions, extended as each dept is classified — so a child # resolves its sibling key against the parent's od, mirroring # how the live sweep grows _helix_dept_to_external. running_od: dict[str, str] = { s.dept_code: s.external_id for s in dept_state_rows if s.external_id } for n in self._topological_sort_depts(list(scoped_nodes)): st = state_by_code.get(n.org_node_source_id) if st is not None and st.external_id: would_adopt_dept.append({ "name": n.org_node_name, "src_id": n.org_node_source_id, "feishu_id": st.external_id, "via": "state", }) running_od[n.org_node_source_id] = st.external_id continue parent_od = running_od.get( n.parent_org_node_source_id or "", ) or "0" od, match_kind = dept_runner.match_existing( dept_name=n.org_node_name, parent_od=parent_od, ) if od: would_adopt_dept.append({ "name": n.org_node_name, "src_id": n.org_node_source_id, "feishu_id": od, "via": match_kind, }) running_od[n.org_node_source_id] = od else: would_create_dept.append({ "name": n.org_node_name, "src_id": n.org_node_source_id, "level": n.org_level, }) # Out-of-scope adopted depts → soft-delete (with brake). soft_delete = [ {"name": s.dept_code, "src_id": s.dept_code, "feishu_id": s.external_id} for s in dept_state_rows if s.dept_code not in scoped_ids and s.last_status not in ("soft_deleted", "excluded") ] brake_tripped = len(soft_delete) > 10 if brake_tripped: warnings.append( f"部门停用 mass-action brake: {len(soft_delete)} depts would " "be soft-deleted (>10) — a real run REFUSES and marks them " "failed instead. Review before enabling." ) depts_plan = { "synced_count": len(scoped_nodes), "would_create_count": len(would_create_dept), "would_adopt_count": len(would_adopt_dept), "would_create": would_create_dept[:sample_size], "would_adopt_sample": would_adopt_dept[:sample_size], "skipped_empty_count": len(skipped_nodes), "skipped_sample": [ {"name": n.org_node_name, "src_id": n.org_node_source_id, "level": n.org_level} for n in skipped_nodes[:sample_size] ], "soft_delete_count": len(soft_delete), "soft_delete_sample": soft_delete[:sample_size], "soft_delete_brake_tripped": brake_tripped, "feishu_catalog_size": len(dept_runner._name_to_external), } finally: # FeishuApprovalClient opens a fresh httpx.AsyncClient per # request (auto-closed via async-with), so there is no # persistent connection to release here. pass return { "target_id": str(target.id), "target_slug": target.slug, "kind": target.kind, "enabled": target.enabled, "users": users_plan, "departments": depts_plan, "warnings": warnings, "filter": filter_cfg.to_dict(), } async def dry_run_huilianyi( self, target_id: UUID, *, sample_size: int = 50, ) -> dict[str, Any]: """Full read-only cutover plan for a ``huilianyi`` target. Pulls the COMPLETE vendor roster (员工 + 部门, 增量查询接口 with a full-history window) and reconciles it against Helix's golden record using the same filter / scope / mapping a real run uses. Zero writes to 汇联易 or TiDB. Backs the 切换前对账 panel and the person-by-person manual verification workflow. Beyond the feishu-plan shape, this surfaces the cutover- specific risk classes: * ``email_conflicts`` — 工号+email are BOTH tenant-unique on the vendor; a Helix email already attached to a DIFFERENT 工号 makes the push collide (create) or hijack (update). * ``resigned_on_vendor`` — rows the runner will refuse to auto-rehire (HR must 离职恢复 first). * ``unmapped_company`` — people the runner's required-field guard will fail until the companyCode value_map is extended. * ``vendor_only_active`` — vendor actives with a gate-shaped 工号 missing from Helix scope: Helix can never disable them on offboard — the reconciliation gap to resolve BEFORE turning off the legacy Authing pull. * ``untouched_vendor_depts`` — enabled vendor depts whose code Helix doesn't manage (the pre-recode Beisen-coded tree). This count dropping to ~0 is the live gauge that the 部门 编码原地对齐 step is complete. """ target = await self.get_target(target_id) if target.kind != "huilianyi": raise AppException( code="IDENTITY_SYNC_NOT_HUILIANYI", message=( f"dry_run_huilianyi only applies to huilianyi " f"targets; {target.slug!r} is kind={target.kind!r}" ), status_code=400, ) try: filter_cfg = FilterConfig.from_dict(target.filter_config or {}) except FilterError as e: raise AppException( code="IDENTITY_SYNC_INVALID_FILTER", message=str(e), status_code=400, ) from e from app.domains.identity_sync.runners.huilianyi import ( HuilianyiError, fetch_all_departments, fetch_all_employees, ) warnings: list[str] = [] client = await self._build_huilianyi_client(target) try: # ── Vendor rosters (read-only, full-history window) ────── try: vendor_users = await fetch_all_employees( client, include_leavers=True, ) vendor_depts = await fetch_all_departments(client) except HuilianyiError as e: # Vendor error bodies can echo Authorization material # and this message goes straight to the admin API # response — scrub before embedding, never include the # raw body_excerpt. raise AppException( code="IDENTITY_SYNC_HUILIANYI_READ_FAILED", message=( f"汇联易全量读取失败: {_safe_error(str(e)[:300])} " "—— 确认该权限ID 已开通「员工 → 增量查询员工」和" "「部门 → 增量查询部门」接口。" ), status_code=502, ) from e def _v_status(row: dict[str, Any]) -> str: return str(row.get("status") or "") vendor_by_empno: dict[str, dict[str, Any]] = {} vendor_email_to_empno: dict[str, str] = {} for row in vendor_users: emp = str(row.get("employeeID") or "").strip() if not emp: continue vendor_by_empno[emp] = row em = str(row.get("email") or "").strip().lower() if not em: continue # Emails are tenant-unique across LIVE rows, but the # full-history pull includes resigned rows — prefer an # ACTIVE holder so a leaver's parked email doesn't # mask (or misattribute) the real conflict. prev = vendor_email_to_empno.get(em) if prev is not None: prev_active = _v_status( vendor_by_empno.get(prev) or {}, ) in ("1001", "1002") cur_active = _v_status(row) in ("1001", "1002") if prev_active and not cur_active: continue vendor_email_to_empno[em] = emp # ── Helix side: same filter evaluation as a real sweep ─── candidates, person_by_id, _membership = await self._load_candidates() children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) def _disp(pid: UUID) -> str: p = person_by_id.get(pid) if p is None: return str(pid) # display_name first so the preview shows the authoritative # name ("孟凡生 Jason Meng"), not the pinyin romanisation # ("mengfansheng jason meng"). Mirrors _name in dry_run. return p.display_name or p.name_pinyin or p.work_email or str(pid) # Mirror _sweep()'s future-hire deferral (see dry_run / # dry_run_feishu — same first-creation-only semantics). _fh_state_ids = { # ``select(...person_id)`` + ``.scalars()`` yields the # UUID column values directly — iterate them, do NOT # re-access ``.person_id`` (that AttributeError'd the # whole dry-run 500). pid for pid in (await self.db.execute( select(IdentitySyncState.person_id).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() } _fh_override_ids = set(filter_cfg.person_ids) _lead_hours = await get_provision_lead_hours(self.db) deferred_future_hires: list[dict[str, Any]] = [] in_scope = [] for c in candidates: if not matches(c, filter_cfg, department_source_ids=dept_set): continue _p = person_by_id.get(c.person_id) if ( c.person_id not in _fh_state_ids and str(c.person_id) not in _fh_override_ids and _p is not None and is_future_hire(_p, lead_hours=_lead_hours) ): deferred_future_hires.append({ "person_id": str(c.person_id), "email": _p.work_email, "name": _disp(c.person_id), "hire_date": ( _p.hire_date.isoformat() if _p.hire_date else None ), }) continue in_scope.append(c) excluded_nonemployee: list[dict[str, Any]] = [] gate_pat = None if filter_cfg.employee_no_pattern: gate_pat = re.compile(filter_cfg.employee_no_pattern) for c in candidates: emp = (c.employee_no or "").strip() if not emp or not gate_pat.fullmatch(emp): excluded_nonemployee.append({ "person_id": str(c.person_id), "email": c.work_email, "employee_no": c.employee_no, "name": _disp(c.person_id), }) # companyCode value_map coverage — same rule the runner's # required-field guard enforces at push time. company_value_map: dict[str, str] | None = None for rule in (target.mapping_config or {}).get("rules") or []: if ( isinstance(rule, dict) and rule.get("target_field") == "companyCode" and isinstance(rule.get("value_map"), dict) ): company_value_map = rule["value_map"] break would_update: list[dict[str, Any]] = [] would_create: list[dict[str, Any]] = [] email_conflicts: list[dict[str, Any]] = [] email_changes: list[dict[str, Any]] = [] resigned_on_vendor: list[dict[str, Any]] = [] unmapped_company: list[dict[str, Any]] = [] missing_employee_no: list[dict[str, Any]] = [] in_scope_empnos: set[str] = set() def _holder_note(holder: str) -> tuple[str, str]: """(holder_status, conflict reason suffix) — a RESIGNED holder needs 离职恢复/释放邮箱, an active one needs an actual email cleanup; conflating them sends the admin chasing the wrong fix.""" hstat = _v_status(vendor_by_empno.get(holder) or {}) if hstat == "1003": return hstat, ( ";占用方在汇联易侧是离职状态 —— 处理方式是释放" "该离职行的邮箱(或离职恢复后改邮箱),不是清理" "在职数据" ) return hstat, "" for c in in_scope: emp = (c.employee_no or "").strip() helix_email = (c.work_email or "").strip().lower() if emp: in_scope_empnos.add(emp) row = { "person_id": str(c.person_id), "email": c.work_email, "employee_no": c.employee_no, "name": _disp(c.person_id), } if not emp: # The real run hard-fails these at the runner's # missing-employeeID guard — the plan must say the # same thing, not "would create". missing_employee_no.append(row) continue p = person_by_id.get(c.person_id) company_code = ( (p.company_code or "").strip() if p is not None else "" ) if company_value_map is not None and ( not company_code or company_code not in company_value_map ): unmapped_company.append({ **row, "company_code": company_code or None, }) vrow = vendor_by_empno.get(emp) if emp else None if vrow is not None: vendor_email = str(vrow.get("email") or "").strip().lower() if _v_status(vrow) == "1003": resigned_on_vendor.append({ **row, "vendor_email": vendor_email or None, }) continue if helix_email and vendor_email != helix_email: holder = vendor_email_to_empno.get(helix_email) if holder and holder != emp: hstat, note = _holder_note(holder) email_conflicts.append({ **row, "vendor_email": vendor_email or None, "conflict_with_employee_no": holder, "holder_status": hstat, "reason": ( "目标邮箱已被另一工号占用,更新会撞" f"邮箱唯一约束{note}" ), }) continue email_changes.append({ **row, "vendor_email": vendor_email or None, }) would_update.append(row) else: holder = vendor_email_to_empno.get(helix_email) if holder: hstat, note = _holder_note(holder) email_conflicts.append({ **row, "conflict_with_employee_no": holder, "holder_status": hstat, "reason": ( "工号在汇联易不存在,但邮箱已挂在另一工号上," f"创建会撞邮箱唯一约束{note}" ), }) continue would_create.append(row) # Vendor actives that Helix will NEVER manage. Split by the # employee_no gate: gate-shaped 工号 missing from Helix = # real reconciliation gap; non-gate rows (admin / C-prefix # external) are by-design out of Helix's jurisdiction. vendor_only_active: list[dict[str, Any]] = [] vendor_nongate_active_count = 0 for emp, vrow in vendor_by_empno.items(): if _v_status(vrow) not in ("1001", "1002"): continue if emp in in_scope_empnos: continue if gate_pat is not None and not gate_pat.fullmatch(emp): vendor_nongate_active_count += 1 continue vendor_only_active.append({ "employee_no": emp, "vendor_email": vrow.get("email"), "vendor_name": vrow.get("fullName"), "vendor_status": _v_status(vrow), }) users_plan = { "in_scope_count": len(in_scope), "vendor_total": len(vendor_by_empno), "would_update_count": len(would_update), "would_create_count": len(would_create), "would_create_sample": would_create[:sample_size], "email_conflict_count": len(email_conflicts), "email_conflicts": email_conflicts[:sample_size], "email_change_count": len(email_changes), "email_changes_sample": email_changes[:sample_size], "resigned_on_vendor_count": len(resigned_on_vendor), "resigned_on_vendor": resigned_on_vendor[:sample_size], "unmapped_company_count": len(unmapped_company), "unmapped_company": unmapped_company[:sample_size], "missing_employee_no_count": len(missing_employee_no), "missing_employee_no": missing_employee_no[:sample_size], "vendor_only_active_count": len(vendor_only_active), "vendor_only_active": vendor_only_active[:sample_size], "vendor_nongate_active_count": vendor_nongate_active_count, "excluded_nonemployee_count": len(excluded_nonemployee), "excluded_nonemployee": excluded_nonemployee[:sample_size], "deferred_future_hire_count": len(deferred_future_hires), "deferred_future_hire_sample": ( deferred_future_hires[:sample_size] ), "employee_no_pattern": filter_cfg.employee_no_pattern, } if email_conflicts: warnings.append( f"{len(email_conflicts)} 个邮箱唯一约束冲突 —— 推送前先在" "汇联易侧清理(改邮箱或办离职),否则这些人会持续失败。" ) if unmapped_company: warnings.append( f"{len(unmapped_company)} 人的 company_code 不在 " "companyCode value_map 里,推送会被必填校验拦下 —— " "先补映射。" ) if missing_employee_no: warnings.append( f"{len(missing_employee_no)} 个范围内员工没有工号 —— " "真实运行会在必填校验上失败;先补 HRIS 工号。" ) if vendor_only_active: shape_note = ( "(工号符合门禁形状)" if gate_pat is not None else "(该目标未配置工号门禁,此名单包含全部不在范围" "内的在职账号)" ) warnings.append( f"{len(vendor_only_active)} 个汇联易在职账号" f"{shape_note}不在 Helix 同步范围内 —— Helix 永远" "不会为他们办停用,关闭 Authing 拉取前先核对这份名单。" ) # ── DEPARTMENTS ───────────────────────────────────────── all_nodes = (await self.db.execute( select(IdentityOrgNode).where( IdentityOrgNode.deleted_at.is_(None), IdentityOrgNode.node_status == "active", ) )).scalars().all() scoped_ids = await self._compute_inscope_dept_ids( target=target, nodes=list(all_nodes), ) scoped_nodes = [ n for n in all_nodes if n.org_node_source_id in scoped_ids ] skipped_nodes = [ n for n in all_nodes if n.org_node_source_id not in scoped_ids ] vendor_dept_by_code: dict[str, dict[str, Any]] = {} for d in vendor_depts: code = str(d.get("custDeptNumber") or "").strip() if code: vendor_dept_by_code[code] = d would_create_dept: list[dict[str, Any]] = [] would_update_dept: list[dict[str, Any]] = [] disabled_on_vendor: list[dict[str, Any]] = [] for n in scoped_nodes: code = n.org_node_source_id vd = vendor_dept_by_code.get(code) entry = { "name": n.org_node_name, "src_id": code, "level": n.org_level, } if vd is None: would_create_dept.append(entry) continue v_dept_status = str(vd.get("status") or "") if v_dept_status == "101": would_update_dept.append({ **entry, "vendor_name": vd.get("departmentName"), }) else: # 102 禁用 / 103 删除 — a PUT against it is vendor- # semantics-dependent; surface for manual 恢复. disabled_on_vendor.append({ **entry, "vendor_name": vd.get("departmentName"), "vendor_status": v_dept_status, }) dept_state_rows = (await self.db.execute( select(IdentitySyncDepartmentState).where( IdentitySyncDepartmentState.target_id == target.id, ) )).scalars().all() would_disable = [ {"src_id": s.dept_code, "external_id": s.external_id} for s in dept_state_rows if s.dept_code not in scoped_ids and s.last_status not in ("soft_deleted", "excluded") ] brake_tripped = len(would_disable) > 10 if brake_tripped: warnings.append( f"部门失效 mass-action brake: {len(would_disable)} 个部门" "将被失效(>10)—— 真实运行会拒绝执行并标记 failed。" ) # Enabled vendor depts Helix doesn't manage = the legacy # Beisen-coded tree. Should trend to ~0 after 原地改码. scoped_code_set = {n.org_node_source_id for n in scoped_nodes} untouched_vendor = [ { "cust_dept_number": code, "vendor_name": d.get("departmentName"), } for code, d in vendor_dept_by_code.items() if str(d.get("status") or "") == "101" and code not in scoped_code_set ] if disabled_on_vendor: warnings.append( f"{len(disabled_on_vendor)} 个 Helix 范围内部门在汇联易侧" "处于禁用/删除状态 —— 推送前先在汇联易后台「恢复禁用" "部门」。" ) depts_plan = { "synced_count": len(scoped_nodes), "vendor_enabled_total": sum( 1 for d in vendor_dept_by_code.values() if str(d.get("status") or "") == "101" ), "would_create_count": len(would_create_dept), "would_create": would_create_dept[:sample_size], "would_update_count": len(would_update_dept), "would_update_sample": would_update_dept[:sample_size], "disabled_on_vendor_count": len(disabled_on_vendor), "disabled_on_vendor": disabled_on_vendor[:sample_size], "skipped_empty_count": len(skipped_nodes), "skipped_sample": [ {"name": n.org_node_name, "src_id": n.org_node_source_id, "level": n.org_level} for n in skipped_nodes[:sample_size] ], "would_disable_count": len(would_disable), "would_disable_sample": would_disable[:sample_size], "disable_brake_tripped": brake_tripped, "untouched_vendor_count": len(untouched_vendor), "untouched_vendor_sample": untouched_vendor[:sample_size], } finally: try: await client.aclose() except Exception: # noqa: BLE001 logger.warning( "dry_run_huilianyi: client.aclose() raised", exc_info=True, ) return { "target_id": str(target.id), "target_slug": target.slug, "kind": target.kind, "enabled": target.enabled, "users": users_plan, "departments": depts_plan, "warnings": warnings, "filter": filter_cfg.to_dict(), } # ── Preview (admin UI mapping debugger) ────────────────────────── async def preview_payload( self, *, person_id: UUID, mapping_config: dict[str, Any], runner_options: dict[str, Any] | None = None, ) -> dict[str, Any]: """Render the payload for one person against draft mapping config WITHOUT pushing. Returns context + payload + warnings. """ person = await self.db.get(IdentityPerson, person_id) if person is None or person.deleted_at is not None: raise NotFoundError("IdentityPerson", str(person_id)) membership = (await self.db.execute( select(IdentityOrgMembership).where( IdentityOrgMembership.person_id == person_id, IdentityOrgMembership.is_primary.is_(True), IdentityOrgMembership.deleted_at.is_(None), ) )).scalars().first() managers = await self._resolve_manager_emails([person_id]) mgr_email, mgr_name = managers.get(person_id, (None, None)) hrbps = await self._resolve_hrbp_emails([person_id]) hrbp_email, hrbp_name = hrbps.get(person_id, (None, None)) ctx = build_person_context( person, org_path=membership.org_path if membership else None, manager_email=mgr_email, manager_name=mgr_name, hrbp_email=hrbp_email, hrbp_name=hrbp_name, org_path_drop_root=(runner_options or {}).get("drop_root_segments") or [], org_node_source_id=( membership.org_node_source_id if membership else None ), ) # 汇联易 主岗直属领导 token. The mapping debugger renders a DRAFT # config with no target attached, so there is no kind to gate on — # inject it whenever it resolves (preview-only, never written) so an # admin testing a ``{manager_employee_no}`` rule sees a real value # instead of a silent blank. mgr_no_hly = ( await self._resolve_pushable_manager_nos(None, [person_id]) ).get(person_id) if mgr_no_hly: ctx["manager_employee_no"] = mgr_no_hly warnings: list[str] = [] try: mapping_cfg = MappingConfig.from_dict(mapping_config) except MappingError as e: return { "person_id": person_id, "person_email": person.work_email, "rendered_payload": {}, "rendered_payload_hash": "", "context": ctx, "rule_trace": [], "warnings": [f"mapping invalid: {e}"], } try: payload = render_payload(mapping_cfg, ctx) except MappingError as e: warnings.append(f"render error: {e}") payload = {} # Per-rule trace powers the admin "Mapping Preview" drawer — # each entry shows source → transform → value_map → final, so # admins can see exactly which rule produced (or dropped) a # given downstream field. Computed against the same mapping # config as the payload above; under normal circumstances # they agree, but a trace entry can be ``dropped=True`` while # the payload omits the field — that pairing is the explanation. try: from app.domains.identity_sync.mapping import trace_render rule_trace = trace_render(mapping_cfg, ctx) except MappingError as e: warnings.append(f"trace error: {e}") rule_trace = [] return { "person_id": person_id, "person_email": person.work_email, "rendered_payload": payload, "rendered_payload_hash": hash_payload(payload), "context": ctx, "rule_trace": rule_trace, "warnings": warnings, } # ── State queries (admin UI) ───────────────────────────────────── async def list_state( self, *, target_id: UUID | None = None, person_id: UUID | None = None, status: str | None = None, page: int = 1, page_size: int = 50, ) -> tuple[list[dict[str, Any]], int]: from sqlalchemy import func as sql_func q = select(IdentitySyncState) count_q = select(sql_func.count(IdentitySyncState.id)) conds = [] if target_id: conds.append(IdentitySyncState.target_id == target_id) if person_id: conds.append(IdentitySyncState.person_id == person_id) if status: conds.append(IdentitySyncState.last_status == status) if conds: q = q.where(and_(*conds)) count_q = count_q.where(and_(*conds)) total = (await self.db.execute(count_q)).scalar() or 0 rows = (await self.db.execute( q.order_by(desc(IdentitySyncState.updated_at)) .offset((page - 1) * page_size).limit(page_size) )).scalars().all() # Enrich with person email/name (single batch fetch). person_ids = list({r.person_id for r in rows}) persons_by_id: dict[UUID, IdentityPerson] = {} if person_ids: person_rows = (await self.db.execute( select(IdentityPerson).where(IdentityPerson.id.in_(person_ids)) )).scalars().all() persons_by_id = {p.id: p for p in person_rows} out: list[dict[str, Any]] = [] for r in rows: p = persons_by_id.get(r.person_id) out.append({ "id": r.id, "target_id": r.target_id, "person_id": r.person_id, "person_email": p.work_email if p else None, "person_name": ( p.name_pinyin if p and p.name_pinyin else (p.work_email if p else None) ), "person_employee_no": p.employee_no if p else None, "person_employment_status": p.employment_status if p else None, "external_id": r.external_id, "last_payload_hash": r.last_payload_hash, "last_status": r.last_status, "last_synced_at": r.last_synced_at, "last_error": r.last_error, "retry_count": r.retry_count, "updated_at": r.updated_at, }) return out, int(total) async def list_runs( self, *, target_id: UUID | None = None, page: int = 1, page_size: int = 50, ) -> tuple[list[IdentitySyncRun], int]: from sqlalchemy import func as sql_func q = select(IdentitySyncRun) count_q = select(sql_func.count(IdentitySyncRun.id)) if target_id: q = q.where(IdentitySyncRun.target_id == target_id) count_q = count_q.where(IdentitySyncRun.target_id == target_id) total = (await self.db.execute(count_q)).scalar() or 0 rows = (await self.db.execute( q.order_by(desc(IdentitySyncRun.started_at)) .offset((page - 1) * page_size).limit(page_size) )).scalars().all() return list(rows), int(total) async def get_run(self, run_id: UUID) -> IdentitySyncRun: """Fetch one run row by id (for UI polling of a background run).""" row = await self.db.get(IdentitySyncRun, run_id) if row is None: raise NotFoundError("IdentitySyncRun", str(run_id)) return row async def list_run_items( self, *, run_id: UUID | None = None, target_id: UUID | None = None, person_id: UUID | None = None, item_type: str | None = None, outcome: str | None = None, q: str | None = None, page: int = 1, page_size: int = 50, ) -> tuple[list[IdentitySyncRunItem], int]: """Page through the per-run ledger. Two call shapes: per-run drill-down (``run_id``) and per-person cross-run history (``person_id`` — "what has Helix ever sent for this person", the person-by-person cutover audit trail). ``q`` is a substring match on subject_display / external_key (LIKE metacharacters escaped). """ from sqlalchemy import func as sql_func from sqlalchemy import or_ conds = [] if run_id: conds.append(IdentitySyncRunItem.run_id == run_id) if target_id: conds.append(IdentitySyncRunItem.target_id == target_id) if person_id: conds.append(IdentitySyncRunItem.person_id == person_id) if item_type: conds.append(IdentitySyncRunItem.item_type == item_type) if outcome: conds.append(IdentitySyncRunItem.outcome == outcome) if q: like = ( q.strip() .replace("\\", "\\\\") .replace("%", "\\%") .replace("_", "\\_") ) like = f"%{like}%" conds.append(or_( IdentitySyncRunItem.subject_display.ilike(like), IdentitySyncRunItem.external_key.ilike(like), )) base = select(IdentitySyncRunItem) count_q = select(sql_func.count(IdentitySyncRunItem.id)) if conds: base = base.where(and_(*conds)) count_q = count_q.where(and_(*conds)) total = (await self.db.execute(count_q)).scalar() or 0 rows = (await self.db.execute( base.order_by(desc(IdentitySyncRunItem.created_at)) .offset((page - 1) * page_size).limit(page_size) )).scalars().all() return list(rows), int(total) async def cleanup_run_items(self, *, retention_days: int = 90) -> int: """Delete ledger rows older than the retention window. Runs / state rows are untouched — the ledger is the only unbounded-growth table in the domain (every cron sweep appends non-skip rows). Returns the number of rows deleted. """ from sqlalchemy import delete as sa_delete cutoff = datetime.now(UTC) - timedelta(days=max(retention_days, 1)) result = await self.db.execute( sa_delete(IdentitySyncRunItem).where( IdentitySyncRunItem.created_at < cutoff, ), # Bulk maintenance delete — never evaluate against # in-session objects (the Python evaluator chokes on # naive-vs-aware datetime comparison for rows whose # created_at came from server_default). execution_options={"synchronize_session": False}, ) await self.db.commit() return int(result.rowcount or 0) # ── Vendor snapshot (对端数据) ─────────────────────────────────── async def refresh_vendor_snapshot( self, *, target_id: UUID, actor_id: UUID | None = None, ) -> dict[str, Any]: """Pull the downstream system's CURRENT users / departments / groups (read-only) and replace this target's vendor-record rows with the fresh snapshot. Wipe-and-replace per (target, record_type): the snapshot is a point-in-time mirror, not an event log — stale rows would read as "still exists on the vendor". Each record is matched against identity_persons (email, then employee_no) at refresh time; unmatched user rows are the vendor-only population the admin must review before trusting any cutover. Zero writes on the vendor side. Works on disabled targets by design — "look before you flip" is the whole point. """ from sqlalchemy import delete as sa_delete target = await self.get_target(target_id) records, warnings = await self._collect_vendor_records(target) fetched_at = datetime.now(UTC) # Person match index — email first (every kind carries one), # employee_no fallback (huilianyi keys on 工号). persons = (await self.db.execute( select( IdentityPerson.id, IdentityPerson.work_email, IdentityPerson.employee_no, ).where(IdentityPerson.deleted_at.is_(None)) )).all() by_email: dict[str, UUID] = {} by_empno: dict[str, UUID] = {} for pid, email, empno in persons: if email: by_email.setdefault(email.lower(), pid) if empno: by_empno.setdefault(str(empno), pid) types_refreshed = sorted({r["record_type"] for r in records}) # Delete ALL record types this kind can produce — a type that # came back empty this round must not keep last round's rows. kind_types = { "huilianyi": ["user", "department"], "scim": ["user", "group"], "navan": ["user", "group"], "feishu_contact": ["user", "department"], "google_schema": ["user"], "zoom": ["user"], }.get(target.kind, types_refreshed) await self.db.execute( sa_delete(IdentitySyncVendorRecord).where( IdentitySyncVendorRecord.target_id == target.id, IdentitySyncVendorRecord.record_type.in_(kind_types), ), execution_options={"synchronize_session": False}, ) matched = 0 # Defensive dedup on the unique key — a vendor that re-delivers # a page mid-walk (or two Google users sharing an email-shaped # key) must degrade to last-write-wins, not an IntegrityError # that 500s the whole refresh. deduped: dict[tuple[str, str], dict[str, Any]] = {} for rec in records: deduped[(rec["record_type"], str(rec["external_key"]))] = rec records = list(deduped.values()) for rec in records: person_id = None if rec["record_type"] == "user": email = (rec.get("email") or "").lower() if email and email in by_email: person_id = by_email[email] elif rec.get("external_key") in by_empno: person_id = by_empno[rec["external_key"]] if person_id: matched += 1 self.db.add(IdentitySyncVendorRecord( id=uuid.uuid4(), target_id=target.id, record_type=rec["record_type"], external_key=str(rec["external_key"])[:255], display_name=(rec.get("display_name") or None) and str(rec["display_name"])[:255], email=(rec.get("email") or None) and str(rec["email"]).lower()[:255], vendor_status=(rec.get("vendor_status") or None) and str(rec["vendor_status"])[:32], is_active=rec.get("is_active"), matched_person_id=person_id, raw=_cap_json_for_storage(rec.get("raw") or {}), fetched_at=fetched_at, )) await self.db.commit() counts: dict[str, int] = {} for rec in records: counts[rec["record_type"]] = counts.get(rec["record_type"], 0) + 1 user_total = counts.get("user", 0) result = { "target_id": str(target.id), "fetched_at": fetched_at.isoformat(), "counts": counts, "matched_users": matched, "vendor_only_users": user_total - matched, "warnings": warnings, } await self.audit.log( trace_id=uuid.uuid4(), actor_type="user" if actor_id else "system", actor_user_id=actor_id, action="identity_sync.vendor_snapshot.refresh", status="success", resource_type="identity_sync_target", resource_id=str(target.id), detail={k: v for k, v in result.items() if k != "target_id"}, ) await self.db.commit() return result async def active_leaver_vendor_records( self, *, target_id: uuid.UUID, limit: int, ) -> list[tuple[IdentityPerson, IdentitySyncVendorRecord]]: """Signal-B population — the SHARED source of truth for the leaver zero-trust loop. Returns ``(person, vendor_record)`` for every ACTIVE vendor *user* record whose ``matched_person_id`` is a departed employee (``employment_status='left'``) NOT deliberately retained in the external-account registry, where the snapshot was taken strictly AFTER the person's termination date (a pre-departure snapshot legitimately shows them active — it cannot testify to a missed revocation; a NULL ``termination_date`` stays in scope, the zero-trust default so missing data never silences the alarm). This is EXACTLY what ``leaver_assurance`` reports (read) and what ``leaver_revocation_reconcile`` revokes (write), so the alarm and the fix can never disagree about the population — the reconcile provably drives assurance's Signal B to zero. """ from sqlalchemy import func as sql_func from sqlalchemy import or_ from app.domains.asset.models import ( EXTERNAL_ACCOUNT_KNOWN_CLASSIFICATIONS, ExternalAccountRegistry, ) retained_emails = ( select(sql_func.lower(ExternalAccountRegistry.email)) .where( ExternalAccountRegistry.email.is_not(None), ExternalAccountRegistry.classification.in_( EXTERNAL_ACCOUNT_KNOWN_CLASSIFICATIONS ), ) .scalar_subquery() ) rows = (await self.db.execute( select(IdentityPerson, IdentitySyncVendorRecord) .join( IdentityPerson, IdentityPerson.id == IdentitySyncVendorRecord.matched_person_id, ) .where( IdentitySyncVendorRecord.target_id == target_id, # Only user records revoke; dept/group rows carry no # matched_person_id and are excluded by the join anyway — # the explicit filter just makes the intent unambiguous. IdentitySyncVendorRecord.record_type == "user", IdentitySyncVendorRecord.is_active.is_(True), IdentityPerson.deleted_at.is_(None), IdentityPerson.employment_status == "left", sql_func.lower(IdentityPerson.work_email).notin_( retained_emails ), or_( IdentityPerson.termination_date.is_(None), sql_func.date(IdentitySyncVendorRecord.fetched_at) > IdentityPerson.termination_date, ), ) .limit(limit) )).all() return [(p, v) for p, v in rows] async def _mark_leaver_state_terminal( self, *, target: IdentitySyncTarget, person_id: uuid.UUID, external_key: str | None, status: str, error: str | None = None, bump_retry: bool = False, ) -> None: """Load-or-create the ``(target, person)`` state row and stamp a terminal revocation status — mirrors the sweep's persistence (service.py ~4219) so a reconcile-revoked leaver is never re-flagged by assurance's Signal A. ``person_id`` is always set (the population joins on ``matched_person_id``), so the NOT-NULL ``IdentitySyncState.person_id`` is satisfied even for an orphan with no prior link. """ state = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, IdentitySyncState.person_id == person_id, ) )).scalars().first() if state is None: state = IdentitySyncState( id=uuid.uuid4(), target_id=target.id, person_id=person_id, last_status="pending", ) self.db.add(state) state.last_synced_at = datetime.now(UTC) state.last_status = status if status == "deleted": state.external_id = None elif external_key is not None: state.external_id = external_key if status == "failed": state.last_error = _safe_error(error) if bump_retry: state.retry_count = (state.retry_count or 0) + 1 else: state.last_error = None if status == "excluded": state.retry_count = 0 async def revoke_leavers_on_target( self, target: IdentitySyncTarget, *, dry_run: bool = False, limit: int = 1000, person_id: uuid.UUID | None = None, ) -> dict[str, Any]: """Actively revoke every leaver account the vendor snapshot still shows ACTIVE on ``target`` — the WRITE sibling of the read-only ``leaver_assurance`` reconcile. Sources its population from :meth:`active_leaver_vendor_records` (the same Signal-B set assurance alarms on), so it revokes exactly what the alarm reports. Crucially it does NOT go through ``_load_candidates`` / the sweep, so it reaches the two populations the write path structurally cannot: * SF ``customString24=2061`` (不推送) leavers — excluded from the candidate set, so the sweep never revokes them. 2061 suppresses *provisioning* of an ACTIVE employee; it must not grant a departed one a permanently-live vendor account (DECISIONS 2026-07-18). * Orphans — a departed employee whose account predates Helix managing this target (no ``IdentitySyncState`` link), so no offboard event ever fanned out here (e.g. the 94 alicloud-sso accounts of people who left before the 2026-07-09 onboarding). Revokes by the vendor's OWN key (``external_key`` from the snapshot) — no prior Helix link required — then writes a terminal state row and flips the vendor record ``is_active=False`` so the same-night assurance pass (03:47 UTC, after this at 03:25) sees the heal instead of re-alarming off the pre-revoke snapshot. Per-row commit (a mid-batch vendor failure never loses prior progress); honors the target's ``on_offboard`` policy and the runner capability flags. ``dry_run`` reports what it WOULD revoke with no vendor call — the gated prod-remediation preview. """ pairs = await self.active_leaver_vendor_records( target_id=target.id, limit=limit, ) if person_id is not None: # Single-person mode (the 账号终态 page's button, 2026-09-05). # Filtering the SHARED population rather than running a second # query keeps one definition of "still open on this target" — # the operator closes exactly what the page showed them, and # never a batch they did not ask for. pairs = [(p, v) for p, v in pairs if p.id == person_id] summary: dict[str, Any] = { "target": target.slug, "scoped_person_id": str(person_id) if person_id else None, "candidates": len(pairs), "revoked": 0, "failed": 0, "skipped": 0, "errors": [], "would_revoke": [], } if not pairs: return summary policy = (target.on_offboard or "deactivate").lower() if policy not in ("deactivate", "delete"): # Callers only pass revocation-enabled targets; stay defensive. summary["skipped"] = len(pairs) return summary if dry_run: summary["would_revoke"] = [ { "email": p.work_email, "external_key": v.external_key, "push_code": p.sf_downstream_push_code or "", "employee_no": p.employee_no, } for p, v in pairs ] return summary runner = await self._build_runner(target) try: supported = ( (policy == "deactivate" and runner.supports_deactivate) or (policy == "delete" and runner.supports_delete) ) if not supported: # e.g. google_schema has supports_deactivate=False. Parity # with the sweep's capability gate: collapse to excluded, # no vendor call, no retry churn. for person, _vrec in pairs: await self._mark_leaver_state_terminal( target=target, person_id=person.id, external_key=None, status="excluded", ) summary["skipped"] += 1 await self.db.commit() return summary for person, vrec in pairs: if not person.work_email: # A leaver with a live vendor account but no work_email # can't be re-resolved by email (the runner's fallback) # — surface it, don't silently skip a would-be revoke. summary["failed"] += 1 if len(summary["errors"]) < 50: summary["errors"].append( f"{person.id}: active vendor account " f"(key={vrec.external_key}) but no work_email — " f"cannot resolve to revoke, needs manual review" ) continue try: if policy == "delete": outcome = await runner.delete( person_email=person.work_email, prior_external_id=vrec.external_key, ) else: outcome = await runner.deactivate( person_email=person.work_email, prior_external_id=vrec.external_key, ) except Exception as e: # noqa: BLE001 — per-row isolation logger.exception( "leaver_revocation_reconcile: %s %s for %s failed", policy, target.slug, person.work_email, ) outcome = PersonOutcome( status="failed", error=f"{type(e).__name__}: {e!s:.300}", ) if outcome.status in ("deactivated", "deleted"): await self._mark_leaver_state_terminal( target=target, person_id=person.id, external_key=( None if outcome.status == "deleted" else vrec.external_key ), status=outcome.status, ) # Self-heal the snapshot so tonight's assurance pass # (reads is_active) sees the revocation. vrec.is_active = False summary["revoked"] += 1 elif outcome.status == "excluded": # Vendor refused by policy (e.g. NetSuite Administrator # lock) — excluded, not failed; no retry churn. await self._mark_leaver_state_terminal( target=target, person_id=person.id, external_key=vrec.external_key, status="excluded", ) summary["skipped"] += 1 else: await self._mark_leaver_state_terminal( target=target, person_id=person.id, external_key=vrec.external_key, status="failed", error=outcome.error, bump_retry=True, ) summary["failed"] += 1 if len(summary["errors"]) < 50: summary["errors"].append( f"{person.work_email} ({policy}): " f"{outcome.error or 'unknown'}" ) await self.db.commit() finally: await runner.aclose() return summary async def _collect_vendor_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: """Per-kind read-only roster pull, normalized to the vendor-record shape. Raises AppException for kinds without a snapshot implementation.""" kind = target.kind if kind == "huilianyi": return await self._collect_huilianyi_records(target) if kind in ("scim", "navan"): return await self._collect_scim_records(target) if kind == "feishu_contact": return await self._collect_feishu_records(target) if kind == "google_schema": return await self._collect_google_records(target) if kind == "zoom": return await self._collect_zoom_records(target) raise AppException( code="IDENTITY_SYNC_SNAPSHOT_UNSUPPORTED", message=f"vendor snapshot not implemented for kind {kind!r}", status_code=400, ) async def _collect_zoom_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: """Read-only Zoom roster → vendor-record shape. Feeds the leaver assurance / revocation reconcile so departed users still holding a (paid) seat are caught and deactivated. ``external_key`` = the Zoom user id (so ``revoke_leavers_on_target`` → ``ZoomRunner.deactivate`` addresses the account by its stable id).""" from app.integrations.drivers.zoom_software import _PLAN_BY_TYPE runner = await self._build_runner(target) out: list[dict[str, Any]] = [] try: users = await runner.list_users() # type: ignore[attr-defined] finally: await runner.aclose() for u in users: status = str(u.get("_helix_status") or "active") uid = str(u.get("id") or "") email = (str(u.get("email") or "").strip().lower()) or None if not uid and not email: continue try: type_int = int(u.get("type") or 0) except (TypeError, ValueError): type_int = 0 tier = _PLAN_BY_TYPE.get(type_int, str(type_int)) first = str(u.get("first_name") or "").strip() last = str(u.get("last_name") or "").strip() display = (f"{first} {last}").strip() or email out.append({ "record_type": "user", "external_key": uid or email, "display_name": display, "email": email, # ``{status}/{tier}`` surfaces both "is it live" and "does it # cost a paid seat" (licensed) in one column for the admin. "vendor_status": f"{status}/{tier}", # active + pending occupy the account (and a licensed seat); # inactive is already closed → is_active=False so a prior # deactivation reads as converged, not a re-alarm. "is_active": status in ("active", "pending"), "raw": { "id": uid, "email": email, "type": type_int, "tier": tier, "status": status, }, }) return out, [] async def _collect_huilianyi_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: from app.domains.identity_sync.runners.huilianyi import ( _STATUS_RESIGNED, fetch_all_departments, fetch_all_employees, ) client = await self._build_huilianyi_client(target) try: vendor_users = await fetch_all_employees(client) vendor_depts = await fetch_all_departments(client) finally: await client.aclose() out: list[dict[str, Any]] = [] # raw allow-list (mirrors the feishu collector's trim): keep the # fields the drift comparator / admin view actually use; the # vendor roster row can carry extras we never want at rest # (certificate numbers, bank-ish payroll fields). user_raw_keys = ( "employeeID", "fullName", "email", "custDeptNumber", "companyCode", "status", "entryDate", "leavedDate", "mobile", "departmentName", "activated", "userOID", ) for u in vendor_users: emp_id = str(u.get("employeeID") or "") if not emp_id: continue status = str(u.get("status") or "") out.append({ "record_type": "user", "external_key": emp_id, "display_name": u.get("fullName"), "email": u.get("email"), "vendor_status": status, "is_active": status != _STATUS_RESIGNED, "raw": {k: u[k] for k in user_raw_keys if k in u}, }) for d in vendor_depts: code = str(d.get("custDeptNumber") or "") if not code: continue status = str(d.get("status") or "") out.append({ "record_type": "department", "external_key": code, "display_name": d.get("name"), "vendor_status": status, # 101 启用 / 102 禁用 / 103 删除 "is_active": status == "101", "raw": d, }) return out, [] async def _collect_scim_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: from app.domains.identity_sync.runners.scim import ScimError opts = dict(target.runner_options or {}) if opts.get("roster_source") == "aws_identity_store": return await self._collect_aws_identity_store_records(target) runner = await self._build_runner(target) warnings: list[str] = [] out: list[dict[str, Any]] = [] try: users = await runner.list_users() out.extend(_scim_user_snapshot_record(u) for u in users) try: groups = await runner.list_groups() except ScimError as e: # Navan-style tenants don't expose /Groups — surface, # don't fail the user half. groups = [] warnings.append(_safe_error( f"Groups listing unavailable: {e!s:.200}", )) out.extend(_scim_group_snapshot_record(g) for g in groups) finally: await runner.aclose() return [r for r in out if r["external_key"]], warnings async def _collect_aws_identity_store_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: """Vendor snapshot for an AWS IdC target, read via Identity Store. AWS IdC's SCIM endpoint cannot enumerate past 100 users (it ignores ``startIndex`` outright), so ``ScimRunner`` refuses to return a truncated roster and the daily snapshot fails by design. The Identity Store API is the same directory with real ``NextToken`` paging; records come back normalized into SCIM shape so ``external_key`` stays the SCIM ``/Users/{id}`` that revocation feeds to ``ScimRunner.deactivate``, and ``raw`` stays comparable on the drift board. Writes are untouched — SCIM still owns those. """ from app.domains.identity_sync.runners.aws_identity_store import ( AwsIdentityStoreError, fetch_identity_store_roster, ) opts = dict(target.runner_options or {}) slug = str(opts.get("aws_integration_slug") or "").strip() integ = (await self.db.execute( select(Integration).where(Integration.slug == slug), )).scalars().first() if integ is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"target {target.slug!r} points at AWS integration " f"{slug!r} for its roster, but no such integration " "exists." ), status_code=400, ) if not integ.enabled: # A disabled integration means "stop using these credentials". # Reading the roster with them anyway would quietly override # an admin's off switch. raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message=( f"AWS integration {slug!r} is disabled; enable it or " f"point target {target.slug!r} at another one." ), status_code=400, ) resolved = await _resolve_integration(self.db, integ) store_id = ( str(opts.get("identity_store_id") or "").strip() or str(resolved.config.get("sso_identity_store_id") or "").strip() ) region = ( str(opts.get("identity_store_region") or "").strip() or str(resolved.config.get("sso_region") or "").strip() ) if not store_id or not region: raise AppException( code="IDENTITY_SYNC_INVALID_RUNNER_OPTIONS", message=( f"AWS integration {slug!r} has no " "sso_identity_store_id / sso_region configured; set " "them there, or override with runner_options." "identity_store_id / identity_store_region." ), status_code=400, ) try: users, groups, raw_warnings = await fetch_identity_store_roster( access_key_id=resolved.config.get("access_key_id") or "", secret_access_key=( resolved.config.get("secret_access_key") or "" ), identity_store_id=store_id, region=region, # Only pull groups for targets that actually push them — # the others never had group rows in their snapshot and a # needless ListGroups walk just spends cron budget. include_groups=bool(opts.get("push_groups")), ) except AwsIdentityStoreError as e: # Surfaces as a failed target in the daily digest, same as a # SCIM failure would. Deliberately NOT degraded to a warning: # a partial roster reads as "these leavers are already gone". raise AppException( code="IDENTITY_SYNC_ROSTER_FETCH_FAILED", message=_safe_error(str(e)) or "roster fetch failed", status_code=502, ) from e out = [_scim_user_snapshot_record(u) for u in users] out.extend(_scim_group_snapshot_record(g) for g in groups) warnings = [w for w in (_safe_error(w) for w in raw_warnings) if w] return [r for r in out if r["external_key"]], warnings async def _collect_feishu_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: from app.approval.feishu_client import FeishuApprovalClient from app.domains.identity_sync.runners.feishu_contact import ( fetch_feishu_directory, ) if target.integration_id is None: raise AppException( code="IDENTITY_SYNC_NO_INTEGRATION", message="feishu_contact snapshot requires the feishu integration", status_code=400, ) integ = await self.db.get(Integration, target.integration_id) if integ is None: raise NotFoundError("Integration", str(target.integration_id)) resolved = await _resolve_integration(self.db, integ) app_id = resolved.config.get("app_id") app_secret = resolved.config.get("app_secret") if not app_id or not app_secret: raise AppException( code="IDENTITY_SYNC_NO_TOKEN", message=( "feishu integration is missing app_id/app_secret — " "configure credentials before pulling a snapshot." ), status_code=400, ) client = FeishuApprovalClient(app_id=app_id, app_secret=app_secret) users, depts = await fetch_feishu_directory(client) out: list[dict[str, Any]] = [] for u in users: uid = str(u.get("user_id") or u.get("open_id") or "") if not uid: continue frozen = bool((u.get("status") or {}).get("is_frozen")) email = u.get("enterprise_email") or u.get("email") out.append({ "record_type": "user", "external_key": uid, "display_name": u.get("name"), "email": email, "vendor_status": "frozen" if frozen else "active", "is_active": not frozen, # Trim: drop mobile + avatar PII/noise from the stored # raw; keep identity-shaped fields the admin verifies. "raw": { k: u.get(k) for k in ( "user_id", "name", "en_name", "email", "enterprise_email", "employee_no", "employee_id", "department_ids", "status", "job_title", ) if k in u }, }) for d in depts: od = str(d.get("open_department_id") or "") if not od: continue out.append({ "record_type": "department", "external_key": od, "display_name": d.get("name"), "vendor_status": "active", "is_active": True, "raw": { k: d.get(k) for k in ( "open_department_id", "name", "parent_department_id", "department_id", "member_count", "status", "code", ) if k in d }, }) return out, [] async def _collect_google_records( self, target: IdentitySyncTarget, ) -> tuple[list[dict[str, Any]], list[str]]: runner = await self._build_runner(target) try: driver = runner._driver # GoogleSchemaRunner wraps the driver except AttributeError: raise AppException( code="IDENTITY_SYNC_SNAPSHOT_UNSUPPORTED", message="google_schema runner exposes no directory driver", status_code=500, ) from None users = await driver.list_all_users( include_suspended=True, projection="full", ) out: list[dict[str, Any]] = [] for u in users: email = u.get("primaryEmail") if not email: continue suspended = bool(u.get("suspended")) out.append({ "record_type": "user", "external_key": str(email).lower(), "display_name": ((u.get("name") or {}).get("fullName")), "email": email, "vendor_status": "suspended" if suspended else "active", "is_active": not suspended, "raw": { "primaryEmail": email, "name": (u.get("name") or {}).get("fullName"), "suspended": suspended, "orgUnitPath": u.get("orgUnitPath"), "customSchemas": u.get("customSchemas"), }, }) return out, [] async def list_vendor_records( self, *, target_id: UUID, record_type: str | None = None, q: str | None = None, matched: bool | None = None, is_active: bool | None = None, page: int = 1, page_size: int = 50, ) -> tuple[list[IdentitySyncVendorRecord], int, datetime | None]: """Page the stored snapshot. Returns (rows, total, fetched_at of the newest row for the target — the snapshot age banner).""" from sqlalchemy import func as sql_func from sqlalchemy import or_ conds = [IdentitySyncVendorRecord.target_id == target_id] if record_type: conds.append(IdentitySyncVendorRecord.record_type == record_type) if matched is True: conds.append(IdentitySyncVendorRecord.matched_person_id.is_not(None)) elif matched is False: conds.append(IdentitySyncVendorRecord.matched_person_id.is_(None)) if is_active is True: conds.append(IdentitySyncVendorRecord.is_active.is_(True)) elif is_active is False: conds.append(IdentitySyncVendorRecord.is_active.is_(False)) if q: like = ( q.strip() .replace("\\", "\\\\") .replace("%", "\\%") .replace("_", "\\_") ) like = f"%{like}%" conds.append(or_( IdentitySyncVendorRecord.display_name.ilike(like), IdentitySyncVendorRecord.email.ilike(like), IdentitySyncVendorRecord.external_key.ilike(like), )) total = (await self.db.execute( select(sql_func.count(IdentitySyncVendorRecord.id)) .where(and_(*conds)) )).scalar() or 0 rows = (await self.db.execute( select(IdentitySyncVendorRecord) .where(and_(*conds)) .order_by( IdentitySyncVendorRecord.record_type, IdentitySyncVendorRecord.display_name, ) .offset((page - 1) * page_size).limit(page_size) )).scalars().all() fetched_at = (await self.db.execute( select(sql_func.max(IdentitySyncVendorRecord.fetched_at)) .where(IdentitySyncVendorRecord.target_id == target_id) )).scalar() return list(rows), int(total), fetched_at # ── Drift diff — Helix render vs vendor snapshot ───────────────── # feishu payloads carry fields the runner TRANSLATES before the # wire (dept codes → od-XXX, custom_attrs → wire shape) — only the # identity-shaped keys that exist verbatim in the trimmed vendor # raw are honestly comparable. _FEISHU_COMPARABLE_KEYS = frozenset({ "name", "email", "enterprise_email", "employee_no", "job_title", }) # 汇联易 write-field → read-field aliases. The vendor's write contract # (创建/更新员工V2) and its 增量查询员工 read contract use DIFFERENT names # for the same attribute: 入职时间 is written as ``entryTime`` and echoed # as ``entryDate``. Without the alias the drift comparator would look up # the write name in the snapshot, miss, and report the field # ``not_comparable`` forever — i.e. 入职时间 would silently stop being # audited. (``directManagerEmpoyeeId`` has NO read counterpart at all — # the roster endpoint doesn't echo 主岗直属领导 — so it stays # ``not_comparable``; verifying it needs POST /api/open/user/all/ # directmanager per person, too expensive for the snapshot sweep.) _HUILIANYI_READ_KEY_ALIASES = {"entryTime": "entryDate"} @staticmethod def _flatten_for_diff( value: Any, prefix: tuple[str, ...] = (), ) -> list[tuple[tuple[str, ...], Any]]: """Dict → (segment-tuple, leaf) pairs. Lists stay whole at their path (multiset-normalized at compare time, not exploded — index alignment between Helix and vendor is meaningless). Segment TUPLES, not dot-joined strings: SCIM enterprise extension payloads nest under a URN key that itself contains a dot ("…:2.0:User"), and a string join+split round-trip would shear that key apart and make every extension field unfindable on the vendor side. """ out: list[tuple[tuple[str, ...], Any]] = [] if isinstance(value, dict): for k, v in value.items(): seg = prefix + (str(k),) if isinstance(v, dict): out.extend( IdentitySyncService._flatten_for_diff(v, seg), ) else: out.append((seg, v)) return out # Midnight-datetime suffix — Helix renders dates as "YYYY-MM-DD" # while vendors echo them padded ("2023-04-01 00:00:00" on 汇联易, # "2023-04-01T00:00:00Z" on its 增量查询 reads). Both sides mean # the same date; without this normalization every person with a # date field mapped would report drift forever right after a # clean sweep. _MIDNIGHT_SUFFIX_RE = re.compile( r"^(\d{4}-\d{2}-\d{2})[T ]00:00:00(?:\.0+)?(?:Z|\+00:00)?$" ) @staticmethod def _norm_for_diff(v: Any) -> Any: """Comparison-normalize: trim strings (emails lowered, midnight datetimes collapsed to their date), coerce scalars to str, lists → sorted multiset (dicts inside lists collapse to their ``value`` field when present — the SCIM emails shape).""" if v is None: return None if isinstance(v, str): s = v.strip() m = IdentitySyncService._MIDNIGHT_SUFFIX_RE.match(s) if m: return m.group(1) return s.lower() if "@" in s else s if isinstance(v, bool): return str(v).lower() if isinstance(v, (int, float)): return str(v) if isinstance(v, list): items = [] for x in v: if isinstance(x, dict) and "value" in x: x = x.get("value") items.append( str(IdentitySyncService._norm_for_diff(x)), ) return sorted(items) return v @staticmethod def _vendor_value_at( raw: Any, segments: tuple[str, ...], ) -> tuple[bool, Any]: """Walk ``raw`` by path segments. Returns (present, value).""" cur = raw for seg in segments: if not isinstance(cur, dict) or seg not in cur: return False, None cur = cur[seg] return True, cur def _compare_fields( self, *, kind: str, payload: dict[str, Any], raw: dict[str, Any], schema_id: str | None, ) -> list[dict[str, Any]]: """Field-by-field verdicts. ``drift`` ONLY on a value-vs-value mismatch; a field the vendor read API simply doesn't echo (or a kind-untranslatable field) is ``not_comparable`` — counting those as drift would page the admin over read-API shape, not real divergence.""" fields: list[dict[str, Any]] = [] flat = sorted(self._flatten_for_diff(payload), key=lambda x: x[0]) for segments, hv in flat: display = ".".join(segments) if kind == "google_schema": lookup = ( ("customSchemas", str(schema_id), *segments) if schema_id else segments ) present, vv = self._vendor_value_at(raw, lookup) elif kind == "huilianyi": alias = self._HUILIANYI_READ_KEY_ALIASES.get(segments[0]) lookup = ( (alias, *segments[1:]) if alias else segments ) present, vv = self._vendor_value_at(raw, lookup) elif kind == "feishu_contact": if segments[0] not in self._FEISHU_COMPARABLE_KEYS: fields.append({ "field": display, "helix_value": hv, "vendor_value": None, "status": "not_comparable", }) continue present, vv = self._vendor_value_at(raw, segments) else: present, vv = self._vendor_value_at(raw, segments) if not present: fields.append({ "field": display, "helix_value": hv, "vendor_value": None, "status": "not_comparable", }) continue status = ( "match" if self._norm_for_diff(hv) == self._norm_for_diff(vv) else "drift" ) # Compare on real values above; mask phone-shaped fields # for DISPLAY only (suffix survives, so the admin can # still eyeball-verify a match). if _SENSITIVE_PHONE_KEY_RE.search(display): hv = mask_sensitive_display({display: hv})[display] vv = mask_sensitive_display({display: vv})[display] fields.append({ "field": display, "helix_value": hv, "vendor_value": vv, "status": status, }) return fields async def compute_drift( self, *, target_id: UUID, q: str | None = None, only_drift: bool = False, page: int = 1, page_size: int = 50, ) -> dict[str, Any]: """Per-person diff of the rendered Helix payload (what a sweep WOULD send right now) against the stored vendor snapshot (what the downstream system holds). Zero vendor API calls — render is pure data; the snapshot's staleness is surfaced via ``snapshot_fetched_at`` so the admin knows when to refresh. Scope = the target's filter (auto-include rehydration is a sweep-only concern and intentionally not replicated here). Vendor record location order: state.external_id → matched_person_id → email. """ target = await self.get_target(target_id) try: filter_cfg = FilterConfig.from_dict(target.filter_config or {}) mapping_cfg = MappingConfig.from_dict(target.mapping_config or {}) except (FilterError, MappingError) as e: raise AppException( code="IDENTITY_SYNC_INVALID_CONFIG", message=str(e), status_code=400, ) from e vendor_rows = (await self.db.execute( select(IdentitySyncVendorRecord).where( IdentitySyncVendorRecord.target_id == target.id, IdentitySyncVendorRecord.record_type == "user", ) )).scalars().all() snapshot_fetched_at = max( (r.fetched_at for r in vendor_rows if r.fetched_at), default=None, ) if not vendor_rows: return { "target_id": str(target.id), "snapshot_fetched_at": None, "summary": None, "rows": [], "total": 0, "needs_snapshot": True, } vendor_by_key = {r.external_key: r for r in vendor_rows} vendor_by_person = { r.matched_person_id: r for r in vendor_rows if r.matched_person_id } vendor_by_email = { r.email: r for r in vendor_rows if r.email } candidates, person_by_id, membership_by_person = ( await self._load_candidates() ) children_idx = await self._build_dept_children_index() dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) in_scope = [ c for c in candidates if matches(c, filter_cfg, department_source_ids=dept_set) ] in_scope_ids = {c.person_id for c in in_scope} managers_by_person = await self._resolve_manager_emails(in_scope_ids) hrbp_by_person = await self._resolve_hrbp_emails(in_scope_ids) # 汇联易 主岗直属领导 — drift compares the RENDERED payload against # the vendor snapshot, so it must render exactly what the sweep # sends; without this every 汇联易 person would read as drifted on # ``directManagerEmpoyeeId`` forever. hly_manager_nos: dict[UUID, str] = ( await self._resolve_pushable_manager_nos( target.filter_config, in_scope_ids, ) if target.kind == "huilianyi" else {} ) state_rows = (await self.db.execute( select(IdentitySyncState).where( IdentitySyncState.target_id == target.id, ) )).scalars().all() external_by_person = { s.person_id: s.external_id for s in state_rows if s.external_id } # When each linked person was last vendor-verified — surfaced on the # drift row so the admin can see the drift-reverify cadence (a row not # verified within the 7-day window will be re-pushed next sweep even # on an unchanged hash). verified_by_person = { s.person_id: s.last_verified_at for s in state_rows } runner_opts = target.runner_options or {} drop_root = runner_opts.get("drop_root_segments") or [] schema_id = ( (runner_opts.get("schema_id") or "Department_Information") if target.kind == "google_schema" else None ) rows: list[dict[str, Any]] = [] summary = { "in_scope": len(in_scope), "compared": 0, "clean": 0, "drifted": 0, "vendor_missing": 0, "render_failed": 0, } # NOTE: ``q`` is applied to the ROWS after the loop, never to # the summary — the stat tiles must answer "is this target # clean?" for the whole scope regardless of any search box. q_norm = (q or "").strip().lower() for cand in in_scope: person = person_by_id.get(cand.person_id) if person is None: continue membership = membership_by_person.get(cand.person_id) mgr_email, mgr_name = managers_by_person.get( cand.person_id, (None, None), ) hrbp_email, hrbp_name = hrbp_by_person.get( cand.person_id, (None, None), ) ctx = build_person_context( person, org_path=membership.org_path if membership else None, manager_email=mgr_email, manager_name=mgr_name, hrbp_email=hrbp_email, hrbp_name=hrbp_name, org_path_drop_root=drop_root, org_node_source_id=( membership.org_node_source_id if membership else None ), ) if hly_manager_nos.get(cand.person_id): ctx["manager_employee_no"] = hly_manager_nos[cand.person_id] _lv = verified_by_person.get(person.id) row: dict[str, Any] = { "person_id": str(person.id), "email": person.work_email, "name": person.name_pinyin or person.display_name or person.work_email, "last_verified_at": _lv.isoformat() if _lv else None, } try: # Mirror the sweep / single-person push: a linked person # is pushed with the UPDATE shape, so create_only fields # (e.g. NetSuite firstName, deliberately vendor-curated) # must be excluded here too — otherwise the drift board # shows a permanent, un-reconcilable red row for exactly # the divergence create_only exists to allow. payload = render_payload( mapping_cfg, ctx, exclude_create_only=bool(external_by_person.get(person.id)), ) except MappingError as e: summary["render_failed"] += 1 row.update({ "status": "render_failed", "error": str(e)[:300], "drift_count": 0, "fields": [], }) rows.append(row) continue ext = external_by_person.get(person.id) vrec = ( (vendor_by_key.get(ext) if ext else None) or vendor_by_person.get(person.id) or (vendor_by_email.get(person.work_email.lower()) if person.work_email else None) ) if vrec is None: summary["vendor_missing"] += 1 row.update({ "status": "vendor_missing", "drift_count": 0, "fields": [], }) rows.append(row) continue fields = self._compare_fields( kind=target.kind, payload=payload, raw=vrec.raw or {}, schema_id=schema_id, ) drift_count = sum(1 for f in fields if f["status"] == "drift") summary["compared"] += 1 if drift_count: summary["drifted"] += 1 else: summary["clean"] += 1 row.update({ "status": "drift" if drift_count else "clean", "external_key": vrec.external_key, "vendor_active": vrec.is_active, "drift_count": drift_count, "fields": fields, }) rows.append(row) if q_norm: rows = [ r for r in rows if q_norm in ( (r.get("email") or "").lower() + "|" + (r.get("name") or "").lower() ) ] if only_drift: rows = [ r for r in rows if r["status"] in ("drift", "vendor_missing", "render_failed") ] order = { "drift": 0, "vendor_missing": 1, "render_failed": 2, "clean": 3, } rows.sort(key=lambda r: ( order.get(r["status"], 9), -(r.get("drift_count") or 0), r.get("email") or "", )) total = len(rows) page_rows = rows[(page - 1) * page_size: page * page_size] # Field detail only for the returned page — keeping every # in-scope person's full field list in one response would be # hundreds of KB of mostly-clean rows. page_ids = {id(r) for r in page_rows} for r in rows: if id(r) not in page_ids: r.pop("fields", None) return { "target_id": str(target.id), "snapshot_fetched_at": ( snapshot_fetched_at.isoformat() if snapshot_fetched_at else None ), "summary": summary, "rows": page_rows, "total": total, "needs_snapshot": False, } # ── Overview cockpit ───────────────────────────────────────────── async def overview(self) -> dict[str, Any]: """One-call aggregation behind the 数据同步 cockpit (总览 tab). Everything the landing view needs in a single round-trip: upstream posture (headcount, HRIS integrations, data-quality summary) plus one card per target — integration health, last / next run, per-status state counts, the filter-evaluated in-scope count, and a recent-run sparkline. Pure reads — zero downstream API calls, so it is always safe to render even while a sweep is mid-flight. Per-target failures are isolated: a broken filter_config on one target degrades that card (``in_scope_count=None`` + ``filter_error``) instead of failing the whole endpoint. """ from sqlalchemy import func as sql_func now_iso = datetime.now(UTC).isoformat() targets = await self.list_targets() target_ids = [t.id for t in targets] # Per-target per-status counts — one grouped query per table. person_counts: dict[UUID, dict[str, int]] = {} dept_counts: dict[UUID, dict[str, int]] = {} if target_ids: rows = (await self.db.execute( select( IdentitySyncState.target_id, IdentitySyncState.last_status, sql_func.count(), ) .where(IdentitySyncState.target_id.in_(target_ids)) .group_by( IdentitySyncState.target_id, IdentitySyncState.last_status, ) )).all() for tid, status, n in rows: person_counts.setdefault(tid, {})[status] = int(n) rows = (await self.db.execute( select( IdentitySyncDepartmentState.target_id, IdentitySyncDepartmentState.last_status, sql_func.count(), ) .where(IdentitySyncDepartmentState.target_id.in_(target_ids)) .group_by( IdentitySyncDepartmentState.target_id, IdentitySyncDepartmentState.last_status, ) )).all() for tid, status, n in rows: dept_counts.setdefault(tid, {})[status] = int(n) # Recent runs for the per-card sparkline. Targets are few # (single digits), so a tiny indexed query per target beats a # window function on a table that grows daily. recent_runs: dict[UUID, list[dict[str, Any]]] = {} for tid in target_ids: runs = (await self.db.execute( select(IdentitySyncRun) .where(IdentitySyncRun.target_id == tid) .order_by(desc(IdentitySyncRun.started_at)) .limit(10) )).scalars().all() recent_runs[tid] = [ { "id": str(r.id), "status": r.status, "trigger_source": r.trigger_source, "started_at": ( r.started_at.isoformat() if r.started_at else None ), "duration_ms": r.duration_ms, "failed_count": r.failed_count, } for r in runs ] # Integration health for every referenced integration row. integration_ids = { t.integration_id for t in targets if t.integration_id } integrations_by_id: dict[UUID, Integration] = {} if integration_ids: integration_rows = (await self.db.execute( select(Integration).where(Integration.id.in_(integration_ids)) )).scalars().all() integrations_by_id = {i.id: i for i in integration_rows} # Filter-evaluated in-scope count per target. Candidates and # the dept-children index are loaded ONCE; per-target work is # pure in-memory filter evaluation. candidates, _persons, _memberships = await self._load_candidates() children_idx = await self._build_dept_children_index() def _in_scope_count(t: IdentitySyncTarget) -> tuple[int | None, str | None]: try: filter_cfg = FilterConfig.from_dict(t.filter_config or {}) except FilterError as e: return None, str(e) dept_set = expand_department_subtree( filter_cfg.departments, children_index=children_idx, include_subtree=filter_cfg.include_subtree, ) n = sum( 1 for c in candidates if matches(c, filter_cfg, department_source_ids=dept_set) ) return n, None target_cards: list[dict[str, Any]] = [] for t in targets: integration = ( integrations_by_id.get(t.integration_id) if t.integration_id else None ) in_scope, filter_error = _in_scope_count(t) counts = person_counts.get(t.id, {}) target_cards.append({ "id": str(t.id), "name": t.name, "slug": t.slug, "kind": t.kind, "description": t.description, "enabled": t.enabled, "on_offboard": t.on_offboard, "schedule": t.schedule_config or {}, "integration": ( { "id": str(integration.id), "name": integration.name, "provider": integration.provider, "enabled": integration.enabled, "last_test_status": integration.last_test_status, "last_tested_at": ( integration.last_tested_at.isoformat() if integration.last_tested_at else None ), } if integration else None ), "last_run_at": ( t.last_run_at.isoformat() if t.last_run_at else None ), "last_run_status": t.last_run_status, "last_run_summary": t.last_run_summary, "next_run_at": ( t.next_run_at.isoformat() if t.next_run_at else None ), "state_counts": counts, "dept_state_counts": dept_counts.get(t.id, {}), "in_scope_count": in_scope, "filter_error": filter_error, "covered_count": ( counts.get("success", 0) + counts.get("skipped", 0) ), "failed_count": counts.get("failed", 0), "pending_count": counts.get("pending", 0), "recent_runs": recent_runs.get(t.id, []), }) # Upstream posture — people by lifecycle status, the HRIS # source integrations, and the data-quality headline. Function- # local import keeps the app.identity ↔ identity_sync seam # one-directional at module load time (hub_service already # imports our models). people_rows = (await self.db.execute( select( IdentityPerson.employment_status, sql_func.count(), ) .where(IdentityPerson.deleted_at.is_(None)) .group_by(IdentityPerson.employment_status) )).all() people_by_status = {(status or "unknown"): int(n) for status, n in people_rows} hris_rows = (await self.db.execute( select(Integration) .where(Integration.kind == "hris") .order_by(Integration.name) )).scalars().all() from app.identity.data_quality_service import IdentityDataQualityService try: dq_summary = await IdentityDataQualityService(self.db).summary() data_quality = { "open_total": dq_summary.get("open_total", 0), "open_high": dq_summary.get("open_high", 0), "last_run_at": dq_summary.get("last_run_at"), } except Exception: # pragma: no cover — DQ must never sink the cockpit logger.exception("identity_sync.overview: data-quality summary failed") data_quality = None return { "generated_at": now_iso, "source": { "people": { "total": sum(people_by_status.values()), "by_status": people_by_status, }, "hris_integrations": [ { "id": str(i.id), "name": i.name, "enabled": i.enabled, "last_test_status": i.last_test_status, "last_tested_at": ( i.last_tested_at.isoformat() if i.last_tested_at else None ), } for i in hris_rows ], "data_quality": data_quality, }, "targets": target_cards, } # ── 监控看板 — time-windowed sync analytics ────────────────────── @staticmethod def _run_status_bucket(status: str | None) -> str: """Collapse the run status vocabulary into the buckets the 监控看板 charts on. ``ok`` and ``success`` are the same outcome recorded by different code paths (sweep vs single-person). ``cancelled`` (worker restart / deploy killed the sweep) is its own bucket — an infra event, deliberately NOT folded into ``failed`` so it never dilutes the data success rate.""" if status in ("ok", "success"): return "ok" if status == "partial": return "partial" if status == "failed": return "failed" if status == "cancelled": return "cancelled" if status == "running": return "running" return "other" @staticmethod def _trigger_bucket(trigger_source: str | None) -> str: """Normalize ``trigger_source`` (``manual:``, ``event:``, ``cron``/``scheduler``/``due``, …) into a stable display bucket.""" ts = (trigger_source or "").lower() head = ts.split(":", 1)[0] if head == "manual": return "manual" if head in ("event", "event_only") or head.startswith("event"): return "event" if head in ("cron", "scheduler", "due", "schedule"): return "scheduled" return "other" async def metrics( self, *, from_dt: datetime, to_dt: datetime, tz_name: str = "Asia/Shanghai", target_id: UUID | None = None, window_label: str = "", ) -> dict[str, Any]: """Time-windowed analytics behind the 监控看板 tab. Aggregates the run ledger (``identity_sync_runs``) and the per-operation ledger (``identity_sync_run_items``) over ``[from_dt, to_dt)`` into: the KPI strip, a daily status/volume trend, a per-target breakdown, the trigger-source mix, the upstream→downstream data-volume overview, and the most-recent failures (runs + items). Pure reads — never touches a downstream API, so it is always safe to render mid-sweep. Daily trend buckets are computed in ``tz_name`` (the operating timezone) so "今日 / 本周" align with the admin's wall clock rather than UTC midnight. Run volume is bounded (single-digit targets × a few runs/day), so runs are fetched and bucketed in Python — this sidesteps MySQL/TiDB ``DATE()``-with-timezone dialect traps and keeps the window arithmetic in one place. """ from datetime import date as _date from zoneinfo import ZoneInfo from sqlalchemy import func from app.identity.person_models import IdentityOrgNode try: tz = ZoneInfo(tz_name) except Exception: # pragma: no cover — bad tz string falls back to UTC tz = UTC # ``started_at`` / ``created_at`` are stored UTC wall-clock in naive # DATETIME columns (the ``timezone=True`` flag is a no-op on MySQL/ # TiDB). asyncmy binds an aware datetime by its *naive* local-clock # fields without converting, so a Shanghai-aware boundary would be # compared 8h off. Normalize the window bounds to UTC-naive for the # SQL comparisons — same convention as auditor/cmdb range queries. # The aware ``from_dt``/``to_dt`` are still used below for the # tz-anchored trend skeleton and the displayed window labels. from_naive = from_dt.astimezone(UTC).replace(tzinfo=None) to_naive = to_dt.astimezone(UTC).replace(tzinfo=None) # ---- Runs in window (lightweight column projection) ---- run_q = select( IdentitySyncRun.id, IdentitySyncRun.target_id, IdentitySyncRun.started_at, IdentitySyncRun.duration_ms, IdentitySyncRun.status, IdentitySyncRun.trigger_source, IdentitySyncRun.created_count, IdentitySyncRun.updated_count, IdentitySyncRun.skipped_count, IdentitySyncRun.deactivated_count, IdentitySyncRun.failed_count, IdentitySyncRun.blocked_count, ).where( IdentitySyncRun.started_at >= from_naive, IdentitySyncRun.started_at < to_naive, ) if target_id is not None: run_q = run_q.where(IdentitySyncRun.target_id == target_id) run_rows = (await self.db.execute(run_q)).all() # Target name/kind/enabled lookup for the per-target breakdown. targets = await self.list_targets() target_meta: dict[UUID, IdentitySyncTarget] = {t.id: t for t in targets} status_counts = { "ok": 0, "partial": 0, "failed": 0, "cancelled": 0, "running": 0, "other": 0, } persons = { "created": 0, "updated": 0, "skipped": 0, "deactivated": 0, "failed": 0, "blocked": 0, } trigger_counts: dict[str, int] = {} durations: list[int] = [] # Daily trend skeleton — pre-seed every day in the window (in tz) # so empty days render as zeros instead of gaps in the line. start_day = from_dt.astimezone(tz).date() end_day = (to_dt.astimezone(tz) - timedelta(microseconds=1)).date() trend: dict[_date, dict[str, int]] = {} cursor = start_day while cursor <= end_day: trend[cursor] = { "ok": 0, "partial": 0, "failed": 0, "cancelled": 0, "running": 0, "created": 0, "updated": 0, "failed_items": 0, } cursor += timedelta(days=1) per_target: dict[UUID, dict[str, Any]] = {} def _local_date(dt: datetime): # MySQL/TiDB DATETIME round-trips strip tzinfo (the column is # ``timezone=True`` but the backend has no real tz type), so a # freshly-loaded value comes back naive — and ``astimezone`` on # a naive datetime would assume the *server* tz. Stored values # are UTC wall-clock, so pin tzinfo=UTC before converting. if dt.tzinfo is None: dt = dt.replace(tzinfo=UTC) return dt.astimezone(tz).date() for r in run_rows: bucket = self._run_status_bucket(r.status) status_counts[bucket] += 1 persons["created"] += r.created_count or 0 persons["updated"] += r.updated_count or 0 persons["skipped"] += r.skipped_count or 0 persons["deactivated"] += r.deactivated_count or 0 persons["failed"] += r.failed_count or 0 persons["blocked"] += r.blocked_count or 0 if r.duration_ms is not None: durations.append(int(r.duration_ms)) tb = self._trigger_bucket(r.trigger_source) trigger_counts[tb] = trigger_counts.get(tb, 0) + 1 day = _local_date(r.started_at) slot = trend.get(day) if slot is not None: if bucket in ("ok", "partial", "failed", "cancelled", "running"): slot[bucket] += 1 slot["created"] += r.created_count or 0 slot["updated"] += r.updated_count or 0 pt = per_target.get(r.target_id) if pt is None: tgt = target_meta.get(r.target_id) pt = per_target[r.target_id] = { "target_id": str(r.target_id), "name": tgt.name if tgt else "—", "kind": tgt.kind if tgt else None, "enabled": tgt.enabled if tgt else False, "runs": 0, "ok": 0, "partial": 0, "failed": 0, "cancelled": 0, "running": 0, "created": 0, "updated": 0, "skipped": 0, "deactivated": 0, "failed_persons": 0, "blocked_persons": 0, "_durations": [], "last_run_at": tgt.last_run_at.isoformat() if tgt and tgt.last_run_at else None, "last_run_status": tgt.last_run_status if tgt else None, } pt["runs"] += 1 if bucket in ("ok", "partial", "failed", "cancelled", "running"): pt[bucket] += 1 pt["created"] += r.created_count or 0 pt["updated"] += r.updated_count or 0 pt["skipped"] += r.skipped_count or 0 pt["deactivated"] += r.deactivated_count or 0 pt["failed_persons"] += r.failed_count or 0 pt["blocked_persons"] += r.blocked_count or 0 if r.duration_ms is not None: pt["_durations"].append(int(r.duration_ms)) total_runs = len(run_rows) completed = status_counts["ok"] + status_counts["partial"] + status_counts["failed"] # Success rate = fully-clean runs over completed (terminal) runs; # in-flight ``running`` rows are excluded so a mid-sweep snapshot # doesn't dilute the rate. ``partial`` is NOT a success. success_rate = ( round(status_counts["ok"] / completed * 100, 1) if completed else None ) def _percentile(values: list[int], pct: float) -> int | None: if not values: return None ordered = sorted(values) # Nearest-rank percentile — exact, no interpolation games. k = max(0, min(len(ordered) - 1, int(round(pct / 100 * len(ordered) + 0.5)) - 1)) return ordered[k] avg_duration_ms = round(sum(durations) / len(durations)) if durations else None # ---- Failed run-items in window (count + recent sample) ---- item_count_q = select(func.count()).where( IdentitySyncRunItem.created_at >= from_naive, IdentitySyncRunItem.created_at < to_naive, IdentitySyncRunItem.outcome == "failed", ) if target_id is not None: item_count_q = item_count_q.where(IdentitySyncRunItem.target_id == target_id) failed_item_total = int((await self.db.execute(item_count_q)).scalar() or 0) # Per-day failed-item counts folded into the trend. fi_day_q = select( IdentitySyncRunItem.created_at, ).where( IdentitySyncRunItem.created_at >= from_naive, IdentitySyncRunItem.created_at < to_naive, IdentitySyncRunItem.outcome == "failed", ) if target_id is not None: fi_day_q = fi_day_q.where(IdentitySyncRunItem.target_id == target_id) for (created_at,) in (await self.db.execute(fi_day_q)).all(): day = _local_date(created_at) slot = trend.get(day) if slot is not None: slot["failed_items"] += 1 recent_items_q = ( select(IdentitySyncRunItem) .where( IdentitySyncRunItem.created_at >= from_naive, IdentitySyncRunItem.created_at < to_naive, IdentitySyncRunItem.outcome == "failed", ) .order_by(desc(IdentitySyncRunItem.created_at)) .limit(20) ) if target_id is not None: recent_items_q = recent_items_q.where( IdentitySyncRunItem.target_id == target_id ) recent_failed_items = [ { "run_id": str(it.run_id), "target_id": str(it.target_id), "target_name": ( target_meta[it.target_id].name if it.target_id in target_meta else "—" ), "item_type": it.item_type, "subject_display": it.subject_display, "external_key": it.external_key, "error": it.error, "created_at": it.created_at.isoformat() if it.created_at else None, } for it in (await self.db.execute(recent_items_q)).scalars().all() ] # ---- Recent failed / partial RUNS (top failures list) ---- fail_runs_q = ( select(IdentitySyncRun) .where( IdentitySyncRun.started_at >= from_naive, IdentitySyncRun.started_at < to_naive, IdentitySyncRun.status.in_(["failed", "partial"]), ) .order_by(desc(IdentitySyncRun.started_at)) .limit(10) ) if target_id is not None: fail_runs_q = fail_runs_q.where(IdentitySyncRun.target_id == target_id) top_failures = [ { "run_id": str(r.id), "target_id": str(r.target_id), "target_name": ( target_meta[r.target_id].name if r.target_id in target_meta else "—" ), "status": r.status, "trigger": self._trigger_bucket(r.trigger_source), "started_at": r.started_at.isoformat() if r.started_at else None, "duration_ms": r.duration_ms, "failed_count": r.failed_count, "error_message": r.error_message, } for r in (await self.db.execute(fail_runs_q)).scalars().all() ] # ---- Upstream posture (golden-record volume) ---- people_rows = (await self.db.execute( select(IdentityPerson.employment_status, func.count()) .where(IdentityPerson.deleted_at.is_(None)) .group_by(IdentityPerson.employment_status) )).all() people_by_status = {(status or "unknown"): int(n) for status, n in people_rows} org_node_total = int((await self.db.execute( select(func.count()).where( IdentityOrgNode.node_status == "active" ) )).scalar() or 0) hris_rows = (await self.db.execute( select(Integration).where(Integration.kind == "hris") .order_by(Integration.name) )).scalars().all() # ---- Downstream volume (what the targets currently hold) ---- enabled_targets = [t for t in targets if t.enabled] covered_total = 0 vendor_total = 0 if targets: tid_list = [t.id for t in targets] cov_rows = (await self.db.execute( select(IdentitySyncState.target_id, func.count()) .where( IdentitySyncState.target_id.in_(tid_list), IdentitySyncState.last_status.in_(["success", "skipped"]), ) .group_by(IdentitySyncState.target_id) )).all() covered_total = sum(int(n) for _tid, n in cov_rows) vendor_rows = (await self.db.execute( select(func.count()).where( IdentitySyncVendorRecord.target_id.in_(tid_list) ) )).scalar() vendor_total = int(vendor_rows or 0) # Finalize per-target durations. by_target = [] for pt in per_target.values(): durs = pt.pop("_durations") pt["avg_duration_ms"] = round(sum(durs) / len(durs)) if durs else None pt["success_rate"] = ( round(pt["ok"] / (pt["ok"] + pt["partial"] + pt["failed"]) * 100, 1) if (pt["ok"] + pt["partial"] + pt["failed"]) else None ) by_target.append(pt) by_target.sort(key=lambda x: (-(x["failed"] + x["partial"]), -x["runs"])) trend_list = [ {"date": d.isoformat(), **vals} for d, vals in sorted(trend.items()) ] return { "generated_at": datetime.now(UTC).isoformat(), "window": { "from": from_dt.isoformat(), "to": to_dt.isoformat(), "label": window_label, "tz": tz_name, "target_id": str(target_id) if target_id else None, }, "kpi": { "total_runs": total_runs, "ok_runs": status_counts["ok"], "partial_runs": status_counts["partial"], "failed_runs": status_counts["failed"], "cancelled_runs": status_counts["cancelled"], "running_runs": status_counts["running"], "success_rate": success_rate, "avg_duration_ms": avg_duration_ms, "p95_duration_ms": _percentile(durations, 95), "persons": persons, "failed_items": failed_item_total, }, "trend": trend_list, "by_target": by_target, "by_trigger": trigger_counts, "upstream": { "people_total": sum(people_by_status.values()), "people_by_status": people_by_status, "org_nodes_active": org_node_total, "hris_integrations": [ { "id": str(i.id), "name": i.name, "provider": i.provider, "enabled": i.enabled, "last_test_status": i.last_test_status, "last_tested_at": ( i.last_tested_at.isoformat() if i.last_tested_at else None ), } for i in hris_rows ], }, "downstream": { "targets_total": len(targets), "targets_enabled": len(enabled_targets), "persons_covered": covered_total, "vendor_records": vendor_total, }, "top_failures": top_failures, "recent_failed_items": recent_failed_items, }