import asyncio import logging import time from contextlib import asynccontextmanager from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from sqlalchemy import text from starlette.exceptions import HTTPException as StarletteHTTPException import app.integrations.drivers as _integration_drivers # noqa: F401 from app.api_clients.open_meta_router import router as open_meta_router from app.api_clients.router import router as api_clients_router from app.approval.router import router as approval_router from app.assistant import models as _assistant_models # noqa: F401 from app.assistant.evaluation import ( # noqa: F401 models as _assistant_eval_models, ) from app.assistant.evaluation.router import ( router as assistant_eval_router, ) from app.assistant.router import admin_router as assistant_admin_router from app.assistant.router import router as assistant_router from app.audit.router import router as audit_router from app.auth.oauth.router import router as oauth_router from app.auth.oauth.router import wellknown_router as oauth_wellknown_router from app.auth.router import router as auth_router from app.auth.tokens import models as _pat_models # noqa: F401 — register PAT ORM from app.auth.tokens.router import router as auth_tokens_router # Import models so Base.metadata + alembic autogen know about them from app.compliance import models as _compliance_models # noqa: F401 from app.compliance.router import router as compliance_router from app.core.access_cycle_listener import ( register_listener as _register_access_cycle_listener, ) from app.core.config import settings from app.core.exceptions import AppException, app_exception_handler from app.core.knowledge_loop import models as _knowledge_loop_models # noqa: F401 from app.core.knowledge_loop.router import router as knowledge_loop_router from app.core.middleware import ( SecurityHeadersMiddleware, TraceIdMiddleware, ) from app.core.redis_client import get_redis from app.core.response import success_response from app.core.safe_logging import configure_external_http_log_safety from app.core.workflow_journal import models as _wf_journal_models # noqa: F401 from app.dashboard import insights_models as _insights_models # noqa: F401 from app.dashboard.insights_router import router as dashboard_insights_router from app.dashboard.router import router as dashboard_router from app.domains.ai_labs import models as _ai_labs_models # noqa: F401 from app.domains.ai_labs.router import admin_router as ai_labs_admin_router from app.domains.ai_labs.router import router as ai_labs_router from app.domains.ai_spending import models as _ai_spending_models # noqa: F401 from app.domains.ai_spending.router import router as ai_spending_router from app.domains.ai_tool_application import models as _ai_tool_models # noqa: F401 from app.domains.ai_tool_application.router import ( router as ai_tool_application_router, ) from app.domains.announcement_request import models as _announcement_models # noqa: F401 from app.domains.announcement_request.router import ( router as announcement_request_router, ) from app.domains.apikey.router import router as apikey_router from app.domains.approval_registry import models as _approval_registry_models # noqa: F401 from app.domains.approval_registry.router import ( router as approval_registry_router, ) from app.domains.approval_registry.self_router import ( router as approval_registry_self_router, ) from app.domains.arcade import content_models as _arcade_content_models # noqa: F401 from app.domains.arcade import models as _arcade_models # noqa: F401 from app.domains.arcade import trivia_models as _arcade_trivia_models # noqa: F401 from app.domains.arcade.router import admin_router as arcade_admin_router from app.domains.arcade.router import router as arcade_router from app.domains.asset import ( scheduler_handlers as _asset_sched_handlers, # noqa: F401 ) from app.domains.asset.asset_health_router import ( router as asset_ledger_health_router, ) from app.domains.asset.hardware_prep_router import ( router as asset_hardware_prep_router, ) from app.domains.asset.inventory_router import router as asset_inventory_router from app.domains.asset.open_feishu_options_router import ( router as open_feishu_options_router, ) from app.domains.asset.oversight_notifier import ( register as _register_asset_oversight_notifier, ) from app.domains.asset.router import router as asset_router from app.domains.attendance import models as _attendance_models # noqa: F401 from app.domains.attendance import ( scheduler_handlers as _attendance_sched_handlers, # noqa: F401 ) from app.domains.attendance.me_router import router as attendance_me_router from app.domains.attendance.router import router as attendance_router from app.domains.auditor import models as _auditor_models # noqa: F401 from app.domains.auditor import ( # noqa: F401 security_faq_wiki as _security_faq_wiki, ) from app.domains.auditor.access_review_self_router import ( router as access_review_self_router, ) from app.domains.auditor.assessment_router import ( router as auditor_assessment_router, ) from app.domains.auditor.compliance_router import ( router as auditor_compliance_router, ) from app.domains.auditor.governance_router import ( router as auditor_governance_router, ) from app.domains.auditor.inbound_router import ( answer_library_router as auditor_answer_library_router, ) from app.domains.auditor.inbound_router import ( evidence_doc_router as auditor_evidence_doc_router, ) from app.domains.auditor.inbound_router import ( inbound_router as auditor_inbound_router, ) from app.domains.auditor.router import router as auditor_router from app.domains.auditor.security_faq_router import ( admin_router as auditor_security_faq_admin_router, ) from app.domains.auditor.security_faq_router import ( self_router as security_faq_self_router, ) from app.domains.auditor.self_check_router import ( router as auditor_self_check_router, ) from app.domains.auditor.vendor_assessment_router import ( router as auditor_vendor_assessment_router, ) from app.domains.automation_exception import ( # noqa: F401 scheduler_handlers as _autoexc_sched_handlers, ) from app.domains.automation_exception.router import ( router as automation_exception_router, ) from app.domains.bailian import models as _bailian_models # noqa: F401 from app.domains.bailian.router import router as bailian_router from app.domains.bounty import models as _bounty_models # noqa: F401 from app.domains.bounty import ( # Registers ``bounty.mail_intake_poll``. The WORKER imports this module # too, so the cron itself fired fine without this line — what did not # was the console's 「立即运行」 button, which executes in the BACKEND # process: ``execute_job`` found no handler, wrote a ``failed`` run row # and bumped ``failure_count``. Same class as the identity_sync miss in # PR #504 (handler imported by only one of the two processes), caught # 2026-08-28 the first time anyone triggered this job by hand. scheduler_handlers as _bounty_sched_handlers, # noqa: F401 ) from app.domains.bounty.router import router as bounty_router from app.domains.bounty.self_router import router as bounty_self_router from app.domains.cli_dist.router import plugin_router as plugin_dist_router from app.domains.cli_dist.router import router as cli_dist_router from app.domains.cmdb.cost_router import router as cmdb_cost_router from app.domains.cmdb.efficiency_router import router as cmdb_efficiency_router from app.domains.cmdb.lifecycle_router import router as cloud_lifecycle_router from app.domains.cmdb.open_billing_router import ( router as open_cloud_billing_router, ) from app.domains.cmdb.open_cloud_router import router as open_cloud_router from app.domains.cmdb.resource_directory_router import ( router as cmdb_resource_directory_router, ) from app.domains.cmdb.resource_reporting_router import ( router as cmdb_resource_reporting_router, ) from app.domains.cmdb.router import router as cmdb_router from app.domains.cmdb.savings_tracker import ( register as _register_cloud_savings_tracker, ) from app.domains.data_access import models as _data_access_models # noqa: F401 from app.domains.data_access.lifecycle_router import ( router as data_access_lifecycle_router, ) from app.domains.data_access.router import router as data_access_router from app.domains.domain_registration import models as _domain_reg_models # noqa: F401 from app.domains.domain_registration.router import ( router as domain_registration_router, ) from app.domains.email_group import models as _email_group_models # noqa: F401 from app.domains.email_group.admin_router import ( router as email_group_admin_router, ) from app.domains.email_group.dynamic_router import ( router as email_group_dynamic_router, ) from app.domains.email_group.manage_router import ( router as email_group_manage_router, ) from app.domains.email_group.personal_alias_router import ( me_router as email_alias_me_router, ) from app.domains.email_group.personal_alias_router import ( users_router as email_alias_users_router, ) from app.domains.email_group.router import router as email_group_router from app.domains.feed.router import me_router as feed_me_router from app.domains.feed.router import router as feed_router from app.domains.feishu_helpdesk import models as _fh_models # noqa: F401 from app.domains.feishu_helpdesk.router import router as feishu_helpdesk_router from app.domains.finance import close_models as _finance_close_models # noqa: F401 from app.domains.finance import models as _finance_models # noqa: F401 from app.domains.finance.close_router import router as finance_close_router from app.domains.finance.close_self_router import router as finance_close_self_router from app.domains.finance.close_task_router import router as finance_close_task_router from app.domains.finance.router import router as finance_router from app.domains.finance_flux import models as _finance_flux_models # noqa: F401 from app.domains.finance_flux.router import router as finance_flux_router from app.domains.gamification.router import admin_router as gamification_admin_router from app.domains.gamification.router import router as gamification_router from app.domains.github_org import models as _github_org_models # noqa: F401 from app.domains.github_org.handle_change_router import ( router as github_id_change_router, ) from app.domains.github_org.router import router as github_org_router from app.domains.github_org.self_router import ( router as github_org_self_router, ) from app.domains.google_workspace import models as _google_orphan_models # noqa: F401 from app.domains.google_workspace.router import ( router as google_workspace_router, ) from app.domains.handbook.router import router as handbook_router from app.domains.health.router import router as health_router from app.domains.hr.employee_number_router import router as hr_employee_number_router from app.domains.hr.holiday_router import router as hr_holiday_router from app.domains.hr.leave_reminder_models import ( # noqa: F401 — register on Base.metadata HrLeaveReminderBatch, ) from app.domains.hr.leave_reminder_router import ( router as hr_leave_reminder_router, ) from app.domains.hr.open_holiday_router import router as open_holiday_router from app.domains.hr.open_leave_router import router as open_leave_router from app.domains.hr.router import router as hr_router from app.domains.hr_tools import models as _hr_tools_models # noqa: F401 from app.domains.hr_tools.router import router as hr_tools_router from app.domains.identity_sync import models as _identity_sync_models # noqa: F401 from app.domains.identity_sync import scheduler_handler as _identity_sync_handler # noqa: F401 from app.domains.identity_sync.router import router as identity_sync_router from app.domains.leaver_blocked import models as _leaver_blocked_models # noqa: F401 from app.domains.leaver_blocked.listener import register_listener as _register_leaver_listener from app.domains.leaver_blocked.router import router as leaver_blocked_router from app.domains.me_universe.router import router as me_universe_router from app.domains.me_universe.router import team_router as me_team_router from app.domains.offboarding.router import ( router as offboarding_router, ) from app.domains.offboarding.router import ( transfer_router, ) from app.domains.onboarding.manager_router import ( router as onboarding_manager_router, ) from app.domains.onboarding.me_router import router as onboarding_me_router from app.domains.onboarding.router import router as onboarding_router from app.domains.partner_directory import ( # noqa: F401 models as _partner_directory_models, ) from app.domains.partner_directory.router import ( router as partner_directory_router, ) from app.domains.permission.router import router as permission_router from app.domains.platform_tips import models as _platform_tips_models # noqa: F401 from app.domains.platform_tips.router import ( admin_router as security_tips_admin_router, ) from app.domains.platform_tips.router import ( public_router as security_tips_public_router, ) from app.domains.policy_notices import models as _policy_notices_models # noqa: F401 from app.domains.policy_notices.router import ( admin_router as policy_notices_admin_router, ) from app.domains.policy_notices.router import ( public_router as policy_notices_public_router, ) from app.domains.procurement import models as _procurement_models # noqa: F401 from app.domains.procurement.router import router as procurement_router from app.domains.provisioning.router import router as provisioning_router from app.domains.reconciliation.router import router as reconciliation_router from app.domains.relationships import models as _relationships_models # noqa: F401 from app.domains.relationships.router import router as relationships_router from app.domains.security_awareness import ( # noqa: F401 models as _security_awareness_models, ) from app.domains.security_awareness import ( # Registers ``security_awareness.campaign_dispatch``. The WORKER # imports this too — the cron fires there — but the console's # 「立即运行」 button executes in the BACKEND process, and a handler # imported by only one of the two lands a ``failed`` run row saying # "no handler" the first time anybody triggers it by hand. scheduler_handlers as _security_awareness_sched_handlers, # noqa: F401 ) from app.domains.security_awareness.campaign_router import ( admin_router as security_awareness_campaign_admin_router, ) from app.domains.security_awareness.campaign_router import ( self_router as security_awareness_campaign_self_router, ) from app.domains.security_awareness.router import ( admin_router as security_awareness_admin_router, ) from app.domains.security_awareness.router import ( self_router as security_awareness_self_router, ) from app.domains.security_quiz import models as _secquiz_models # noqa: F401 from app.domains.security_quiz.router import ( admin_router as security_quiz_admin_router, ) from app.domains.security_quiz.router import ( router as security_quiz_router, ) from app.domains.sso import models as _sso_models # noqa: F401 from app.domains.sso import scheduler_handlers as _sso_sched_handlers # noqa: F401 from app.domains.sso.app_router import router as sso_app_router from app.domains.sso.router import router as sso_router # Imported for its side effect: ``kb_wiki`` registers the support-KB # adapter with the knowledge loop at import time, and every other reference # to it is a deferred in-function import. Without this line the queue would # accept proposals and then refuse to apply any of them, with a message # saying no handler is registered — a failure that only shows up on the # reviewer's first click. from app.domains.support import kb_wiki as _support_kb_wiki # noqa: F401 from app.domains.support import models as _support_models # noqa: F401 from app.domains.support.router import ( admin_router as support_admin_router, ) from app.domains.support.router import ( router as support_router, ) from app.domains.system_registry import models as _registry_models # noqa: F401 from app.domains.system_registry.router import router as system_registry_router from app.domains.system_registry.self_router import ( router as system_registry_self_router, ) from app.domains.system_registry.team_router import ( router as system_registry_team_router, ) from app.domains.vpn_application import models as _vpn_app_models # noqa: F401 from app.domains.vpn_application.router import ( router as vpn_application_router, ) from app.hr_lifecycle.mover_review_router import ( router as mover_review_router, ) from app.hr_lifecycle.role_change_router import router as role_change_router from app.hr_lifecycle.router import router as lifecycle_router from app.identity import data_quality_models as _identity_dq_models # noqa: F401 from app.identity.audit_listener import register_listener as _register_identity_audit_listener from app.identity.consumers.directory_router import ( router as open_directory_data_router, ) from app.identity.consumers.request_router import ( issue_router as identity_consumer_issue_router, ) from app.identity.consumers.request_router import ( request_router as identity_consumer_request_router, ) from app.identity.consumers.router import ( open_router as identity_consumer_open_router, ) from app.identity.consumers.router import ( router as identity_consumers_router, ) from app.identity.consumers.scim_router import router as scim_router from app.identity.data_quality_router import router as identity_dq_router from app.identity.open_people_router import router as open_people_router from app.identity.profile_router import router as identity_profile_router from app.integrations import health_check as _integration_health_check # noqa: F401 from app.integrations.router import ( router as integrations_router, ) from app.integrations.router import ( settings_router as platform_settings_router, ) from app.intelligence import models as _intelligence_models # noqa: F401 from app.intelligence.router import cloud_scope_router as intelligence_cloud_scope_router from app.intelligence.router import router as intelligence_router from app.mobile.router import router as mobile_router from app.notifications import models as _notification_models # noqa: F401 from app.notifications import ( # noqa: F401 preference_models as _notification_preference_models, ) from app.notifications.router import router as notifications_router from app.observability.client_errors import router as client_errors_router from app.observability.router import router as observability_router from app.scheduled_jobs import ( access_drift_handler as _sched_access_drift, # noqa: F401 ) from app.scheduled_jobs import ai_labs_handlers as _sched_ai_labs_handlers # noqa: F401 # Import handlers so @register_scheduled_job side effects fire in-process # (the router then lists them, and the worker can resolve handler_key). from app.scheduled_jobs import ( ai_spending_handlers as _sched_ai_spending_handlers, # noqa: F401 ) from app.scheduled_jobs import ( approval_registry_handlers as _sched_approval_registry_handlers, # noqa: F401 ) from app.scheduled_jobs import ( asset_inventory_handlers as _sched_asset_inventory_handlers, # noqa: F401 ) from app.scheduled_jobs import ( assistant_eval_handlers as _sched_assistant_eval_handlers, # noqa: F401 ) from app.scheduled_jobs import ( assistant_gap_handlers as _sched_assistant_gap_handlers, # noqa: F401 ) from app.scheduled_jobs import ( assistant_room_handlers as _sched_assistant_room_handlers, # noqa: F401 ) from app.scheduled_jobs import ( assistant_submission_handlers as _sched_assistant_submission_handlers, # noqa: F401 ) from app.scheduled_jobs import auditor_handlers as _sched_auditor_handlers # noqa: F401 from app.scheduled_jobs import bailian_handlers as _sched_bailian_handlers # noqa: F401 from app.scheduled_jobs import chain_handlers as _sched_chain_handlers # noqa: F401 from app.scheduled_jobs import ( cloud_owner_handlers as _sched_cloud_owner_handlers, # noqa: F401 ) from app.scheduled_jobs import ( cmdb_dns_request_handlers as _sched_cmdb_dns_request_handlers, # noqa: F401 ) from app.scheduled_jobs import ( cmdb_server_owner_handlers as _sched_cmdb_server_owner_handlers, # noqa: F401 ) from app.scheduled_jobs import ( # noqa: F401 data_access_handlers as _sched_data_access_handlers, ) from app.scheduled_jobs import ( # noqa: F401 data_access_lifecycle_handlers as _sched_data_access_lifecycle_handlers, ) from app.scheduled_jobs import ( evidence_precompile_handler as _sched_evidence_precompile, # noqa: F401 ) from app.scheduled_jobs import ( # noqa: F401 feed_celebrations_handlers as _sched_feed_celebrations_handlers, ) from app.scheduled_jobs import feishu_handlers as _sched_feishu_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 feishu_helpdesk_handlers as _sched_feishu_helpdesk_handlers, ) from app.scheduled_jobs import finance_handlers as _sched_finance_handlers # noqa: F401 from app.scheduled_jobs import gamification_handlers as _sched_gamification_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 github_org_handlers as _sched_github_org_handlers, ) from app.scheduled_jobs import google_handlers as _sched_google_handlers # noqa: F401 from app.scheduled_jobs import handlers as _sched_handlers # noqa: F401 from app.scheduled_jobs import hr_leave_handlers as _sched_hr_leave_handlers # noqa: F401 from app.scheduled_jobs import ( hr_leave_insights_handlers as _sched_hr_leave_insights_handlers, # noqa: F401 ) from app.scheduled_jobs import ( # noqa: F401 hygiene_handlers as _sched_hygiene_handlers, ) from app.scheduled_jobs import identity_handlers as _sched_identity_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 idle_reclaim_handlers as _sched_idle_reclaim_handlers, ) from app.scheduled_jobs import ( insights_handlers as _sched_insights_handlers, # noqa: F401 ) from app.scheduled_jobs import ( intelligence_handlers as _sched_intelligence_handlers, # noqa: F401 ) from app.scheduled_jobs import jira_handlers as _sched_jira_handlers # noqa: F401 from app.scheduled_jobs import ( knowledge_loop_handlers as _sched_knowledge_loop_handlers, # noqa: F401 ) from app.scheduled_jobs import mdm_handlers as _sched_mdm_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 nexuscrm_handlers as _sched_nexuscrm_handlers, ) from app.scheduled_jobs import ( # noqa: F401 offboarding_assurance_handlers as _sched_offboarding_assurance_handlers, ) from app.scheduled_jobs import ( # noqa: F401 onboarding_handlers as _sched_onboarding_handlers, ) from app.scheduled_jobs import ( # noqa: F401 partner_directory_handlers as _sched_partner_directory_handlers, ) from app.scheduled_jobs import ( # noqa: F401 permission_lifecycle_handlers as _sched_permission_lifecycle_handlers, ) from app.scheduled_jobs import ( posture_summary_handler as _sched_posture_summary, # noqa: F401 ) from app.scheduled_jobs import ( # noqa: F401 procurement_handlers as _sched_procurement_handlers, ) from app.scheduled_jobs import ( # noqa: F401 role_change_handlers as _sched_role_change_handlers, ) from app.scheduled_jobs import secquiz_handlers as _sched_secquiz_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 self_attestation_advice_handlers as _sched_self_attestation_advice_handlers, ) from app.scheduled_jobs import slack_handlers as _sched_slack_handlers # noqa: F401 from app.scheduled_jobs import storage_handlers as _sched_storage_handlers # noqa: F401 from app.scheduled_jobs import ( # noqa: F401 support_feishu_kb_handlers as _sched_support_feishu_kb_handlers, ) from app.scheduled_jobs import ( # noqa: F401 support_kb_gap_handlers as _sched_support_kb_gap_handlers, ) from app.scheduled_jobs import ( # noqa: F401 support_ticket_handlers as _sched_support_ticket_handlers, ) from app.scheduled_jobs import ( # noqa: F401 usage_signal_handlers as _sched_usage_signal_handlers, ) from app.scheduled_jobs import wrapped_handlers as _sched_wrapped_handlers # noqa: F401 from app.scheduled_jobs import zoom_handlers as _sched_zoom_handlers # noqa: F401 from app.scheduled_jobs.router import router as scheduled_jobs_router from app.storage_router import router as storage_router configure_external_http_log_safety() logger = logging.getLogger(__name__) @asynccontextmanager async def lifespan(app: FastAPI): # Startup from app.core.security import _JWT_ALGORITHMS_ALLOWED if settings.JWT_ALGORITHM not in _JWT_ALGORITHMS_ALLOWED: raise RuntimeError( f"JWT_ALGORITHM={settings.JWT_ALGORITHM!r} is not in the " f"allowed list {_JWT_ALGORITHMS_ALLOWED}. " "Set JWT_ALGORITHM=HS256 in your environment." ) if settings.APP_ENV != "development" and settings.ALLOW_PASSWORD_LOGIN: import logging _startup_log = logging.getLogger("app.startup") _startup_log.critical( "SECURITY WARNING: ALLOW_PASSWORD_LOGIN=True in non-development environment. " "Password login bypasses Google SSO and MFA. Disable immediately in production." ) if "*" in settings.CORS_ORIGINS: import logging _startup_log = logging.getLogger("app.startup") _startup_log.critical( "SECURITY WARNING: CORS_ORIGINS contains '*'. This is insecure in production. " "Set CORS_ORIGINS to specific allowed origins." ) if not settings.CREDENTIAL_ENCRYPTION_KEY: # In non-development environments this is a fatal config error. # Booting without the key means every cloud-integration access # raises RuntimeError on first use — admin sees opaque 500s, # K8s liveness flaps, no cloud features work. Fail fast at # startup so the operator sees the actual problem in logs and # the orchestrator stops trying. if settings.APP_ENV != "development": raise RuntimeError( "CREDENTIAL_ENCRYPTION_KEY is required in non-development " "environments (APP_ENV != 'development'). Set it to a " "base64url-encoded 32-byte key before booting." ) logger.warning( "CREDENTIAL_ENCRYPTION_KEY is not set — cloud provider credentials will not be encrypted. " "Set this to a base64url-encoded 32-byte key before adding provider accounts." ) if not settings.PLATFORM_METRICS_TOKEN: # PR-C (#613) ships GET /metrics with Prometheus exposition. # An empty token leaves the endpoint OPEN, which is fine only # when the FastAPI port is VPC-isolated. Helix prod serves user # APIs and /metrics on the SAME ALB-fronted port, so an unset # token in prod would expose cron topology, offboarding SLA # buckets, compliance-incident counts, etc. to the internet. # # We DO NOT raise on missing token (would brick rollouts when # the prod .env hasn't been pre-staged with a fresh secret). # Instead, the endpoint refuses to serve outside ``development`` # whenever the token is unset (see observability/router.py), # combined with a CRITICAL log here so the operator notices in # the startup log instead of discovering it via a scan. if settings.APP_ENV != "development": logger.critical( "PLATFORM_METRICS_TOKEN is NOT SET in non-development " "APP_ENV=%r. /metrics is REFUSING all requests until " "the token is configured (503 Service Unavailable). " "Generate a long random secret and set it on both the " "app .env and the Prometheus scrape config.", settings.APP_ENV, ) else: logger.warning( "PLATFORM_METRICS_TOKEN is not set — /metrics is OPEN. " "Acceptable in local dev / VPC-only deployments; required in prod.", ) # Attach the Leaver Blocked SQLAlchemy listener BEFORE the first # request is served so any unguarded `employment_status='left'` # write — including ones from background tasks — is intercepted. _register_leaver_listener() # Attach the identity_persons audit listener (PR 2026-05-30) so # every INSERT/UPDATE/DELETE on IdentityPerson lands an audit_log # row. Forensic gap discovered during the wangdi post-mortem: # ``identity_persons`` mutations were the only critical write path # in Helix with NO audit trail, so a 2026-04-30 cleanup of 4 ghost # rows + the wangdi disappearance had zero evidence of who / when. _register_identity_audit_listener() # Append-only holding ledgers for auth roles and software permissions. # Both parent tables destroy history on revoke (role rows are DELETEd; # permission rows are reset in place under UNIQUE(user_id, # permission_id)), so "was X an admin on 2026-03-01" / "who held # permission P in May" had no SQL answer. Must also be registered in # ``app/worker.py`` — the provisioning worker owns the canonical # PENDING→ACTIVE / →REVOKED permission flips. _register_access_cycle_listener() # Notify the global IT asset admin (孟凡生) whenever a regional # asset admin operates on an asset — an AuditService.log post-log # hook that catches every asset-item custody/lifecycle mutation. _register_asset_oversight_notifier() # Realized cloud-savings tracker — records a savings event when a # server that was flagged idle/oversized is destroyed / stopped, so the # leadership outcome view can show money actually saved (not estimated). _register_cloud_savings_tracker() # Identity Sync Center — orphan-run watchdog. A worker hard-exit # (SIGKILL / OOM) mid-sweep leaves an `IdentitySyncRun.status='running'` # row behind (the per-target Redis lock TTLs out, the run row # doesn't). On startup we sweep older-than-30-min `running` rows to # `cancelled` so the runs UI doesn't show phantom forever-running # entries and the next dispatch tick has a clean slate. `cancelled` # (not `failed`): a killed sweep is an infra event, not a data # failure — excluded from the success rate and failure alerting. The # recurring `identity_sync.cleanup_stale_runs` job (every 15 min) is # the primary sweeper; this boot-time pass just clears anything left # from the restart that is happening right now. Mirrors the # `cron.cleanup_stale_running` pattern (see migration 114). try: from datetime import UTC as _UTC from datetime import datetime as _dt from datetime import timedelta as _timedelta from sqlalchemy import update as _sa_update from app.core.database import async_session_factory as _asf from app.domains.identity_sync.models import IdentitySyncRun as _IsRun threshold = _dt.now(_UTC) - _timedelta(minutes=30) async with _asf() as _s: r = await _s.execute( _sa_update(_IsRun) .where( _IsRun.status == "running", _IsRun.started_at < threshold, ) .values( status="cancelled", finished_at=_dt.now(_UTC), error_message=( "orphaned by worker restart — finalised cancelled " "by startup watchdog" ), ) ) await _s.commit() if r.rowcount: logger.warning( "identity_sync startup watchdog: finalised %d " "orphan 'running' run rows older than 30 min as cancelled", r.rowcount, ) except Exception: # noqa: BLE001 # Watchdog must NEVER block startup. DB blip / table-doesn't- # exist (pre-migration boot) → log + continue. logger.warning( "identity_sync startup watchdog skipped (non-fatal)", exc_info=True, ) # External-origin Feishu approval templates (see PR #254 / migration # 149). For each ``integration_approval_flows`` row whose operator # set ``accept_external_origin = TRUE``, ensure the Feishu app is # subscribed to that template's events so submissions made # directly in Feishu (NOT via Helix ``instance/create``) push # webhooks we can dispatch. Idempotent — Feishu's subscribe # endpoint + the client's in-process cache short-circuit # repeat calls. Best-effort: a transient Feishu outage at boot # logs a warning; the next webhook in / next admin reload of the # flow definition picks the subscription back up. try: from sqlalchemy import select as _sa_select from app.approval.feishu_client import FeishuApprovalClient as _FsClient from app.core.database import async_session_factory as _asf from app.integrations.compat import get_feishu_config as _get_fs_cfg from app.integrations.models import ( IntegrationApprovalFlow as _IntApprovalFlow, ) async with _asf() as _s: rows = ( await _s.execute( _sa_select(_IntApprovalFlow.external_code) .where(_IntApprovalFlow.active.is_(True)) .where(_IntApprovalFlow.accept_external_origin.is_(True)) ) ).scalars().all() if rows: _cfg = await _get_fs_cfg(session=_s) _fs_client = _FsClient( app_id=str(_cfg.get("app_id") or ""), app_secret=str(_cfg.get("app_secret") or ""), ) for _code in rows: if _code: await _fs_client.ensure_subscribed(_code) logger.info( "feishu: ensured subscription on %d external-origin " "approval templates", len(rows), ) except Exception: # noqa: BLE001 # Subscription is best-effort at boot. Never block uvicorn. logger.warning( "feishu: external-origin subscription sweep skipped (non-fatal)", exc_info=True, ) # FastMCP's Streamable HTTP app maintains its own session manager # that needs lifespan startup/shutdown. Run the inner app's lifespan # alongside ours so the MCP endpoint is fully wired by the time # uvicorn accepts traffic. from app.mcp.server import mcp_server async with mcp_server.session_manager.run(): yield # Shutdown from app.core.database import engine from app.core.redis_client import close_redis await engine.dispose() await close_redis() _is_dev = getattr(settings, "APP_ENV", "development") == "development" app = FastAPI( title="Helix IT Platform", version="0.1.0", lifespan=lifespan, docs_url="/docs" if _is_dev else None, redoc_url="/redoc" if _is_dev else None, openapi_url="/openapi.json" if _is_dev else None, ) # Middleware app.add_middleware(SecurityHeadersMiddleware) app.add_middleware(TraceIdMiddleware) app.add_middleware( CORSMiddleware, allow_origins=settings.CORS_ORIGINS, allow_credentials=True, allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], # ``MCP-Protocol-Version`` / ``Mcp-Session-Id``: MCP spec clients # send these on every post-initialize request to the mounted # ``/api/v1/mcp`` app; without them in the preflight allowlist a # browser-origin MCP client from an allowed CORS_ORIGINS origin # can never connect. Native clients (CLI / desktop) are # unaffected. (Launch audit 2026-06-10.) allow_headers=[ "Authorization", "Content-Type", "X-Trace-ID", "MCP-Protocol-Version", "Mcp-Session-Id", ], expose_headers=["X-Trace-ID", "Mcp-Session-Id"], ) # Exception handlers async def _app_exception_handler( request: Request, exc: Exception, ) -> JSONResponse: if not isinstance(exc, AppException): raise exc return await app_exception_handler(request, exc) app.add_exception_handler(AppException, _app_exception_handler) @app.exception_handler(RequestValidationError) async def _validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse: """Convert FastAPI's default 422 response into our standard envelope. Each individual Pydantic error becomes one entry in ``errors[]`` so callers (and our existing API tests) can still match on the original validator message — we only restructure the wrapping shape, never swallow the per-field reason. """ trace_id = getattr(request.state, "trace_id", None) return JSONResponse( status_code=422, content={ "success": False, "data": None, "meta": {"trace_id": trace_id}, "errors": [ { "code": "VALIDATION_ERROR", "message": str(e.get("msg") or "Request validation failed."), "field": ".".join(str(p) for p in (e.get("loc") or [])[1:]) or None, } for e in exc.errors() ], }, ) @app.exception_handler(StarletteHTTPException) async def _http_exception_handler( request: Request, exc: StarletteHTTPException, ) -> JSONResponse: """Wrap bare ``HTTPException`` (anything that didn't go through :class:`AppException`) so the response envelope and any ``Retry-After`` / ``WWW-Authenticate`` headers stay consistent. Without this handler, ``raise HTTPException(429, ...)`` produced ``{"detail": "..."}`` while the frontend reads ``errors[0].message`` — rate-limit / auth-challenge messages were invisible to users. Headers from the raised exception are filtered through the same allowlist :func:`app_exception_handler` uses, so an unexpected ``Set-Cookie`` / CORS hop can't bleed through into a 4xx response. """ from app.core.exceptions import _SAFE_RESPONSE_HEADER_KEYS # noqa: PLC0415 trace_id = getattr(request.state, "trace_id", None) detail = getattr(exc, "detail", None) or "Request failed." if not isinstance(detail, str): detail = str(detail) raw_headers = getattr(exc, "headers", None) or None safe_headers: dict[str, str] | None = None if raw_headers: safe_headers = { k: v for k, v in raw_headers.items() if k.lower() in _SAFE_RESPONSE_HEADER_KEYS } if not safe_headers: safe_headers = None return JSONResponse( status_code=exc.status_code, headers=safe_headers, content={ "success": False, "data": None, "meta": {"trace_id": trace_id}, "errors": [{"code": str(exc.status_code), "message": detail}], }, ) @app.exception_handler(Exception) async def _unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: """Catch-all so unexpected errors never leak a stack trace to clients. The full exception is logged server-side (with the trace_id so it can be correlated to the failed request); the client only sees a generic INTERNAL_ERROR envelope. """ trace_id = getattr(request.state, "trace_id", None) logger.exception( "Unhandled exception while serving %s %s (trace_id=%s)", request.method, request.url.path, trace_id, ) return JSONResponse( status_code=500, content={ "success": False, "data": None, "meta": {"trace_id": trace_id}, "errors": [{"code": "INTERNAL_ERROR", "message": "An unexpected error occurred."}], }, ) # Routers app.include_router(auth_router) app.include_router(auth_tokens_router) # MCP OAuth 2.1 — discovery (root /.well-known) + AS flow endpoints. app.include_router(oauth_wellknown_router) app.include_router(oauth_router) app.include_router(audit_router) # hardware-prep and the asset-health router (idle-computers / # ledger-health / cost-reconcile) MUST register before asset_router: both live under # /api/v1/assets and asset_router's ``GET /assets/{asset_id}`` would # otherwise swallow their literal path segments as asset_id (422). app.include_router(asset_hardware_prep_router) app.include_router(asset_ledger_health_router) app.include_router(asset_router) app.include_router(asset_inventory_router) app.include_router(procurement_router) app.include_router(cmdb_router) app.include_router(cmdb_resource_directory_router) app.include_router(cmdb_resource_reporting_router) app.include_router(cloud_lifecycle_router) app.include_router(cmdb_cost_router) app.include_router(cmdb_efficiency_router) app.include_router(data_access_router) app.include_router(data_access_lifecycle_router) app.include_router(dashboard_router) app.include_router(dashboard_insights_router) app.include_router(hr_router) app.include_router(hr_leave_reminder_router) app.include_router(hr_holiday_router) app.include_router(hr_employee_number_router) app.include_router(hr_tools_router) app.include_router(apikey_router) app.include_router(bailian_router) app.include_router(finance_router) app.include_router(finance_close_router) app.include_router(finance_close_task_router) app.include_router(finance_close_self_router) app.include_router(finance_flux_router) app.include_router(ai_spending_router) app.include_router(ai_labs_router) app.include_router(ai_labs_admin_router) app.include_router(email_group_router) app.include_router(email_group_dynamic_router) # 邮件组管理 admin surface (HR / global admin): estate list + operation # log + cleanup workbench. Mounted after the two write routers whose # prefixes it nests under — its own prefix is # ``/email-groups/manage``, which cannot collide with # ``/email-groups/{group_id}`` because that path param is typed UUID. app.include_router(email_group_manage_router) # The WRITE half of the same desk, on the same prefix. Mounted AFTER the # read router so its literal paths (/manage/overview, /manage/groups, # /manage/audit-log …) are matched before this one's ``/{group_id}/…`` # patterns are considered — those take a typed UUID and could not match # a literal segment anyway, but registration order makes it moot. app.include_router(email_group_admin_router) app.include_router(attendance_router) app.include_router(attendance_me_router) app.include_router(google_workspace_router) app.include_router(partner_directory_router) app.include_router(email_alias_me_router) app.include_router(email_alias_users_router) app.include_router(provisioning_router) app.include_router(sso_router) app.include_router(sso_app_router) app.include_router(domain_registration_router) app.include_router(github_org_router) app.include_router(github_org_self_router) app.include_router(github_id_change_router) app.include_router(vpn_application_router) app.include_router(support_router) app.include_router(support_admin_router) app.include_router(feishu_helpdesk_router) app.include_router(security_quiz_router) app.include_router(security_quiz_admin_router) app.include_router(gamification_router) app.include_router(gamification_admin_router) app.include_router(arcade_router) app.include_router(arcade_admin_router) app.include_router(announcement_request_router) app.include_router(ai_tool_application_router) app.include_router(feed_router) app.include_router(feed_me_router) app.include_router(integrations_router) app.include_router(platform_settings_router) app.include_router(identity_sync_router) app.include_router(identity_profile_router) app.include_router(identity_dq_router) app.include_router(identity_consumers_router) app.include_router(identity_consumer_request_router) app.include_router(identity_consumer_issue_router) app.include_router(identity_consumer_open_router) app.include_router(open_directory_data_router) app.include_router(open_cloud_router) app.include_router(open_cloud_billing_router) app.include_router(open_feishu_options_router) app.include_router(open_leave_router) app.include_router(open_holiday_router) app.include_router(open_people_router) app.include_router(open_meta_router) app.include_router(api_clients_router) app.include_router(scim_router) app.include_router(onboarding_router) app.include_router(onboarding_manager_router) app.include_router(onboarding_me_router) app.include_router(offboarding_router) app.include_router(transfer_router) app.include_router(permission_router) app.include_router(approval_router) app.include_router(scheduled_jobs_router) app.include_router(compliance_router) app.include_router(notifications_router) app.include_router(lifecycle_router) app.include_router(role_change_router) # The mover review packet (migration 568). Its prefix nests under the # same tree as role_change_router; the two cannot shadow each other in # either order — the parent's only templated route is # "/{person_id}/grants", whose second segment is the literal "grants", # so it can never match "/reviews" (one segment) or "/reviews/{id}". app.include_router(mover_review_router) app.include_router(leaver_blocked_router) app.include_router(relationships_router) app.include_router(me_universe_router) app.include_router(me_team_router) app.include_router(auditor_router) app.include_router(auditor_self_check_router) app.include_router(auditor_vendor_assessment_router) app.include_router(auditor_answer_library_router) app.include_router(security_faq_self_router) app.include_router(auditor_security_faq_admin_router) app.include_router(auditor_evidence_doc_router) app.include_router(auditor_inbound_router) app.include_router(auditor_assessment_router) app.include_router(auditor_governance_router) app.include_router(auditor_compliance_router) app.include_router(bounty_router) app.include_router(bounty_self_router) app.include_router(system_registry_router) app.include_router(system_registry_self_router) app.include_router(system_registry_team_router) app.include_router(approval_registry_router) app.include_router(approval_registry_self_router) app.include_router(access_review_self_router) app.include_router(security_tips_public_router) app.include_router(security_tips_admin_router) app.include_router(policy_notices_public_router) app.include_router(policy_notices_admin_router) app.include_router(security_awareness_self_router) app.include_router(security_awareness_admin_router) app.include_router(security_awareness_campaign_self_router) app.include_router(security_awareness_campaign_admin_router) app.include_router(storage_router) app.include_router(cli_dist_router) app.include_router(plugin_dist_router) app.include_router(mobile_router) app.include_router(automation_exception_router) app.include_router(health_router) app.include_router(handbook_router) app.include_router(reconciliation_router) app.include_router(observability_router) # Frontend crash reports. Separate module from /metrics because the auth # models differ: /metrics is token-gated for scrapers, this is JWT-gated # for real users. app.include_router(client_errors_router) app.include_router(assistant_router) app.include_router(assistant_admin_router) app.include_router(assistant_eval_router) app.include_router(knowledge_loop_router) app.include_router(intelligence_router) app.include_router(intelligence_cloud_scope_router) # MCP server — Streamable HTTP under /api/v1/mcp/. Mount the # PAT-auth-wrapped app so unauthenticated callers are rejected before # they reach the JSON-RPC dispatcher. The inner app provides its own # Starlette routing (POST/GET/DELETE on the mount root) — we do not # add additional routes here. from app.mcp.server import build_mcp_app # noqa: E402 app.mount("/api/v1/mcp", build_mcp_app()) @app.get("/api/v1/health") async def health(request: Request): return success_response(data={"status": "healthy"}, request=request) _READINESS_CACHE_TTL_SECONDS = 2.0 # Hard cap on a single dependency probe. Well under the ALB's 30s interval # so a wedged dependency reports ``error`` (503 = degraded) instead of # leaving the probe hanging. _READINESS_PROBE_TIMEOUT_SECONDS = 5.0 _readiness_cache_until = 0.0 _readiness_cache: tuple[bool, dict[str, str]] | None = None _readiness_lock = asyncio.Lock() async def _readiness_checks() -> tuple[bool, dict[str, str]]: """Run dependency probes behind a short single-flight cache. ``/ready`` is intentionally public for the ALB, so letting every caller check out a DB connection and construct a Redis client turns the probe itself into a cheap resource-amplification endpoint. Two seconds is far shorter than the 30-second ALB interval while coalescing bursts from the ALB, Docker and deploy script. """ global _readiness_cache, _readiness_cache_until now = time.monotonic() if _readiness_cache is not None and now < _readiness_cache_until: ok, cached_checks = _readiness_cache return ok, dict(cached_checks) async with _readiness_lock: now = time.monotonic() if _readiness_cache is not None and now < _readiness_cache_until: ok, cached_checks = _readiness_cache return ok, dict(cached_checks) checks: dict[str, str] = {} try: from app.core.database import engine async with engine.connect() as conn: await conn.execute(text("SELECT 1")) checks["database"] = "ok" except Exception: logger.exception("readiness: database check failed") checks["database"] = "error" # Probe through the SHARED client rather than constructing a fresh # one per cache miss. A per-probe client re-runs the TCP handshake # every time and — before the 2026-08-13 hardening — carried no # connect deadline at all, so a black-holed Redis made ``/ready`` # hang on the OS SYN-retry ceiling (~2 min) while holding # ``_readiness_lock``, stalling every concurrent probe behind it. # The shared pool is bounded, keepalive'd and connect-deadlined; the # outer ``wait_for`` is the belt-and-braces cap so this endpoint can # never outlive the ALB's own timeout regardless of client config. try: await asyncio.wait_for( get_redis().ping(), timeout=_READINESS_PROBE_TIMEOUT_SECONDS, ) checks["redis"] = "ok" except Exception: logger.exception("readiness: redis check failed") checks["redis"] = "error" all_ok = all(value == "ok" for value in checks.values()) _readiness_cache = (all_ok, dict(checks)) _readiness_cache_until = time.monotonic() + _READINESS_CACHE_TTL_SECONDS return all_ok, checks @app.get("/api/v1/ready") async def readiness(request: Request): """Liveness/readiness check used by load balancers and probes. Never returns the underlying exception text — DB/Redis failure messages routinely leak hostnames and credentials in their string form. We log the real cause server-side and return a binary ``ok | error`` per check. """ all_ok, checks = await _readiness_checks() payload = success_response( data={"status": "ready" if all_ok else "degraded", "checks": checks}, request=request, ) return JSONResponse(status_code=200 if all_ok else 503, content=payload)