"""Scheduled jobs that own the DNS-request execution plane. Before 2026-08-14 the cloud write ran inline inside the Feishu approval webhook. That coupled a multi-second provider call to a callback Feishu gives ~3 seconds before it redelivers, and — worse — it made the request handler the only thing that could ever move a row out of ``executing``. When one of those handlers died mid-write (DNSPod rejected the TTL, the fail-path then blew up on an expired ORM attribute), the request sat at ``executing`` forever: the requester saw 处理中 indefinitely, no audit row, no notification, and no admin button that would touch it. Two crons replace that single point of failure: ``cmdb.dns_request_executor`` (every minute) Drains rows the approval callback left at ``approved`` and performs the provider write in the worker, where nothing is redelivering and no HTTP timeout applies. ``cmdb.dns_request_execution_reaper`` (every 10 minutes) Sweeps rows stuck at ``executing`` past ``stale_minutes``. ``create`` is reconciled against the live provider first — if the record is already there, the interrupted write DID land and the row is completed rather than re-driven into a duplicate. Everything else is idempotent and simply re-armed to ``approved``. Rows that exhaust ``max_attempts`` land in ``failed`` with a message the admin resume gate recognises. """ from __future__ import annotations import logging from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.domains.cmdb.dns_request_service import DnsRequestService from app.scheduled_jobs.registry import register_scheduled_job logger = logging.getLogger(__name__) @register_scheduled_job( key="cmdb.dns_request_executor", display_name="DNS 申请执行器", description=( "每分钟扫描 status='approved' 的 DNS 解析申请并执行云侧写入" "(腾讯云 DNSPod / Route53 等),成功置 executed,provider 报错置" " failed 并回写飞书评论。审批回调本身不再做云写:回调只落" " approved 后立即返回,避免飞书 3 秒超时重投与请求被中断导致" "行卡在 executing。config: batch_limit(默认 50)。" ), ) async def run_dns_request_executor( db: AsyncSession, config: dict[str, Any], ) -> dict[str, Any]: """Drain the ``approved`` backlog. Returns ``checked`` / ``executed`` / ``failed`` / ``errors``. ``failed`` is a normal outcome (the provider rejected the write and the row now says so); ``errors`` is not — it counts rows that threw out of the executor itself and is what should be watched. """ batch_limit = int(config.get("batch_limit") or 50) summary = await DnsRequestService(db).execute_approved_backlog( batch_limit=batch_limit, ) # A tick where every row threw is a systematic break (credentials # rotated, provider unreachable, a regression in the executor), not # bad luck. Raise so the failure notifier pages instead of letting a # green cron hide a fully broken execution plane. if summary["checked"] and summary["errors"] == summary["checked"]: raise RuntimeError( f"cmdb.dns_request_executor: all {summary['checked']} rows " "raised out of the executor" ) return summary @register_scheduled_job( key="cmdb.dns_request_execution_reaper", display_name="DNS 申请执行卡死回收", description=( "每 10 分钟扫描 status='executing' 且超过 stale_minutes(默认 10)" "未更新的 DNS 申请——这些行的执行进程已经不在了。create 先对账" "云端:记录已存在说明写入其实成功,补齐 Helix 侧台账并置 executed" "(绝不重复创建);其余动作幂等,直接回退 approved 交给执行器重跑。" "重试超过 max_attempts(默认 3)则置 failed 并写入可 resume 的错误" "信息,管理员在申请详情里点「重跑」即可,无需重新发起审批。" "config: stale_minutes、max_attempts、batch_limit(默认 50)。" ), ) async def run_dns_request_execution_reaper( db: AsyncSession, config: dict[str, Any], ) -> dict[str, Any]: """Un-strand rows parked at ``executing``. Returns ``checked`` / ``healed`` (cloud write had landed, bookkeeping completed) / ``rearmed`` / ``exhausted`` (gave up, now resumable) / ``errors``. """ summary = await DnsRequestService(db).reconcile_stranded_executions( stale_minutes=int(config.get("stale_minutes") or 10), max_attempts=int(config.get("max_attempts") or 3), batch_limit=int(config.get("batch_limit") or 50), ) if summary["checked"] and summary["errors"] == summary["checked"]: raise RuntimeError( f"cmdb.dns_request_execution_reaper: all {summary['checked']} " "rows failed to reconcile" ) if summary["exhausted"]: # Loud on purpose: an exhausted row means an approved DNS change # never reached the provider and now needs a human. logger.warning( "cmdb.dns_request_execution_reaper: %s request(s) gave up after " "repeated interrupted executions — awaiting an admin resume", summary["exhausted"], ) return summary