"""下线卡在「订阅里还有资源」时,Helix 自己清理,不再把错误甩回管理员 ## 这是在解决什么 管理员点「下线 it-team-us」,Azure 回 `400 ResourcesExistingOnSubscription` (订阅里还有一个存储账号 `billingexport123`),Helix 把这句话翻成一段带 `az group delete` 命令的红色弹窗甩回去 —— 也就是让刚刚要求 Helix 下线的人 自己去 Azure 把订阅下线。GCP 的 `liens_exist` 是同一个形状(让人去跑 `gcloud alpha resource-manager liens list`)。 `decommission_reconcile.py` 的模块开头早就写下了相反的原则:「One click is the whole interaction … Reporting work back to the person who asked you to do it is not diligence」——**下线之后**的异步收尾已经这样做了(云端关闭失败 Helix 自己重试)。缺的恰好是**前置条件**这一半。 ## 这次加的列(cmdb_cloud_scopes) 一次「已确认但云端还不能接受」的下线,状态存在这 8 列上: * `decommission_pending_since` / `_by` / `_reason` —— 确认过的那次下线意图。 * `decommission_cleanup_keys` (JSON) —— **管理员在弹窗里看到并确认过的 blocker key 列表**。清理永远只动这个列表里的东西:确认之后新建出来的 资源组不在列表里,所以不可能被静默删掉 —— resume 作业发现它就停下来, 改成 `needs_reconfirm` 等人重新确认。这一列就是「那次同意」本身。 * `decommission_cleanup_state` —— `cleaning` / `needs_reconfirm` / `blocked`(有 Helix 权限不够或不该自己动的 blocker,已通知过人一次)/ `stalled`(重试预算用尽,已通知过人一次)。 * `decommission_cleanup_attempts` / `_error` / `_trace_id`。 **`status` 故意不动**:全程还是 `active`。给 status 枚举加一个 `decommissioning` 会连坐到每一处读它的谓词(申请权限、闲置巡检、成本归属、 7 处前端面),而其中大多数问的是另一个问题。真正关心的查询读 `decommission_pending_since IS NOT NULL`。 ## 新增 cron `cmdb.decommission_cleanup_resume`,`*/5`。删除在云上是异步的(Azure 资源组 删除返回 202,之后在 `Deleting` 里待几十秒到几十分钟),所以点击那一刻不可能 等到结果。这个作业反复推进:重新读 blocker → 删掉还在的(只在确认过的 key 里)→ 重试 close → 成功就走既有的「落库 + 停 grant + 审计 + 通知点击的人 一条」路径。预算用尽或碰到删不掉的东西才通知人一次。 Seeded ENABLED —— 一个 disabled 的种子就是「功能上线了但从来没被打开」, 2026-09-05 的账本审计数出过 35 行这种。 Revision ID: 624 Revises: 623 """ from __future__ import annotations import uuid import sqlalchemy as sa from alembic import op revision = "624" # Re-read from ``origin/main`` immediately before push, never carried over # from the last local verification: 623 landed while this branch was being # written. A duplicate revision id is not two heads — it is one id meaning # two different things, and alembic refuses outright (see 621's note for # the seven-renumber history that made this a house rule). down_revision = "623" branch_labels = None depends_on = None SCOPES = "cmdb_cloud_scopes" JOBS = "scheduled_jobs" JOB_KEY = "cmdb.decommission_cleanup_resume" _JOB_DESC = ( "每 5 分钟推进一次「已确认但云端还不能接受」的下线。云厂商拒绝关闭的常见" "原因是 scope 里还有东西(Azure ResourcesExistingOnSubscription、GCP " "liens_exist),这个作业按管理员确认时看到的那份清单删掉它们,再重试关闭;" "删除在云上是异步的,所以它会重复推进直到云端接受。**只删管理员确认过的 " "key** —— 确认之后新出现的资源不会被静默删掉,而是把这次下线转成" "「需要重新确认」。重试预算用尽、或碰到 Helix 删不掉的东西(权限不够的" "资源锁等),给点击下线的人发一条通知并停下,不在页面上反复报错。" ) _COLUMNS: tuple[tuple[str, sa.Column], ...] = ( ( "decommission_pending_since", sa.Column( "decommission_pending_since", sa.DateTime(timezone=True), nullable=True, ), ), ( "decommission_pending_by", sa.Column("decommission_pending_by", sa.String(36), nullable=True), ), ( "decommission_pending_reason", sa.Column( "decommission_pending_reason", sa.String(500), nullable=True, ), ), ( "decommission_cleanup_keys", sa.Column("decommission_cleanup_keys", sa.JSON(), nullable=True), ), ( "decommission_cleanup_state", sa.Column("decommission_cleanup_state", sa.String(32), nullable=True), ), ( "decommission_cleanup_attempts", sa.Column( "decommission_cleanup_attempts", sa.Integer(), nullable=False, server_default="0", ), ), ( "decommission_cleanup_error", sa.Column("decommission_cleanup_error", sa.String(500), nullable=True), ), ( "decommission_cleanup_trace_id", sa.Column( "decommission_cleanup_trace_id", sa.String(64), nullable=True, ), ), ) def _has_table(bind, name: str) -> bool: return bool(bind.execute( sa.text( "SELECT 1 FROM information_schema.tables " "WHERE table_schema = DATABASE() AND table_name = :t LIMIT 1", ), {"t": name}, ).scalar()) def _has_column(bind, table: str, column: str) -> bool: return bool(bind.execute( sa.text( "SELECT 1 FROM information_schema.columns " "WHERE table_schema = DATABASE() AND table_name = :t " "AND column_name = :c LIMIT 1", ), {"t": table, "c": column}, ).scalar()) def _has_index(bind, table: str, index: str) -> bool: return bool(bind.execute( sa.text( "SELECT 1 FROM information_schema.statistics " "WHERE table_schema = DATABASE() AND table_name = :t " "AND index_name = :i LIMIT 1", ), {"t": table, "i": index}, ).scalar()) def upgrade() -> None: bind = op.get_bind() if _has_table(bind, SCOPES): for name, col in _COLUMNS: if not _has_column(bind, SCOPES, name): op.add_column(SCOPES, col) # The resume job's queue is "scopes with a pending decommission", # which is a handful of rows in a table of ~1000+. Without this # the */5 cron full-scans the table 288 times a day to find # nothing on almost every tick. if not _has_index(bind, SCOPES, "ix_cmdb_scopes_decommission_pending"): op.create_index( "ix_cmdb_scopes_decommission_pending", SCOPES, ["decommission_pending_since"], ) if _has_table(bind, JOBS): bind.execute( sa.text( f"INSERT INTO {JOBS} " "(id, name, display_name, description, handler_key, " " trigger_type, cron_expr, timezone, enabled, config, " " run_count, failure_count, created_at, updated_at) " "SELECT :id, :name, :display, :desc, :handler, 'cron', " ":cron, 'UTC', 1, :config, 0, 0, NOW(), NOW() " "FROM DUAL WHERE NOT EXISTS " f"(SELECT 1 FROM {JOBS} WHERE handler_key = :handler)" ), { "id": str(uuid.uuid4()), "name": JOB_KEY, "display": "下线前置清理 · 推进与重试", "desc": _JOB_DESC, "handler": JOB_KEY, "cron": "*/5 * * * *", "config": "{}", }, ) def downgrade() -> None: bind = op.get_bind() if _has_table(bind, JOBS): bind.execute( sa.text(f"DELETE FROM {JOBS} WHERE handler_key = :handler"), {"handler": JOB_KEY}, ) if _has_table(bind, SCOPES): if _has_index(bind, SCOPES, "ix_cmdb_scopes_decommission_pending"): op.drop_index("ix_cmdb_scopes_decommission_pending", SCOPES) for name, _col in reversed(_COLUMNS): if _has_column(bind, SCOPES, name): op.drop_column(SCOPES, name)