"""回填脚本的「静默缺口」护栏(2026-07-28 生产回填事故的回归)。 背景:`sync_*_for_integration` 遇到 per-scope 超时/驱动错误时**不抛 异常**,而是返回 ``status='partial_failure'``(细节在 ``errors`` / ``per_scope``)。修复前 `backfill_cost_facts.py` 只在 ``except`` 分支 往 ``result.errors`` 写,所以这种返回值只打一行 INFO、`errors` 为空、 脚本**退出码 0** —— 生产上两个最大的 AWS 账号在每个 30 天块都超时, 它们的 `amount_usd_list` 全程留 NULL,最后靠人工 facts-vs-billing_daily 对数才发现。 Pins: * ``_collect_chunk_failure``:成功形状返回 None;partial_failure 点名 出错的 scope(`per_scope` 里非 ok 的行)并带 reason/errors 明细。 * ``IntegrationResult.status`` 与退出码口径一致:返回值里的 partial_failure 必须让整轮判失败(而不是只看抛异常)。 * ``_chunks(chunk_days=N)``:可调块长(重租户用 7 天块避开 300s per-scope 闸),且切块完整覆盖窗口、不重不漏。 """ from __future__ import annotations import importlib.util import sys from datetime import date from pathlib import Path _SCRIPT = ( Path(__file__).resolve().parents[2] / "scripts" / "backfill_cost_facts.py" ) def _load_script_module(): spec = importlib.util.spec_from_file_location( "backfill_cost_facts_under_test", _SCRIPT, ) assert spec and spec.loader module = importlib.util.module_from_spec(spec) # 必须先注册进 sys.modules 再 exec:脚本用了 ``from __future__ import # annotations`` + @dataclass,dataclass 解析延迟注解时会去 # ``sys.modules[cls.__module__].__dict__`` 找名字,模块未注册时报 # ``'NoneType' object has no attribute '__dict__'``。 sys.modules[spec.name] = module spec.loader.exec_module(module) return module _mod = _load_script_module() # ── _collect_chunk_failure ──────────────────────────────────────── def test_success_shapes_are_not_failures() -> None: for status in ("ok", "skipped", "not_supported", "partial_success", "dry_run"): assert _mod._collect_chunk_failure( surface="cost_facts", start=date(2026, 6, 1), end=date(2026, 6, 30), payload={"status": status}, ) is None, status def test_partial_failure_names_offending_scopes() -> None: """生产形状:两个大账号 per-scope 超时,其余 scope ok。""" msg = _mod._collect_chunk_failure( surface="cost_facts", start=date(2026, 6, 1), end=date(2026, 6, 30), payload={ "status": "partial_failure", "errors": ["scope 632330501419 exceeded 300.0s timeout; advancing"], "per_scope": [ {"scope": "143458967504", "status": "ok", "facts": 12}, {"scope": "632330501419", "status": "timeout", "error": "exceeded 300.0s timeout"}, {"scope": "385595570414", "status": "timeout", "error": "exceeded 300.0s timeout"}, ], }, ) assert msg is not None assert "status=partial_failure" in msg assert "632330501419(timeout)" in msg assert "385595570414(timeout)" in msg # ok 的 scope 不该被点名 assert "143458967504" not in msg assert "300.0s" in msg # reason/errors 明细透出 def test_failed_status_without_per_scope_still_reports_reason() -> None: msg = _mod._collect_chunk_failure( surface="billing_daily", start=date(2026, 6, 1), end=date(2026, 6, 30), payload={"status": "failed", "reason": "driver load failed: boom"}, ) assert msg is not None assert "billing_daily" in msg and "driver load failed" in msg # ── 退出码口径 ──────────────────────────────────────────────────── def test_integration_result_status_flags_returned_partial_failure() -> None: result = _mod.IntegrationResult( integration_id="i-1", name="AWS", provider="aws", ) result.cost_facts.append({"status": "ok"}) assert result.status == "ok" # 未抛异常、仅返回 partial_failure —— 修复前这里仍算 ok 且 errors 空 result.cost_facts.append({"status": "partial_failure"}) assert result.status == "partial_failure" def test_failure_predicate_matches_status_not_only_exceptions() -> None: """退出码口径:status != ok 即判失败(镜像 main_async 的收尾判定)。""" ok = _mod.IntegrationResult( integration_id="i-ok", name="GCP", provider="gcp", ) ok.cost_facts.append({"status": "ok"}) partial = _mod.IntegrationResult( integration_id="i-partial", name="AWS", provider="aws", ) partial.cost_facts.append({"status": "partial_failure"}) failed = [r for r in (ok, partial) if r.errors or r.status != "ok"] assert [r.name for r in failed] == ["AWS"] # ── 可调块长 ────────────────────────────────────────────────────── def test_chunks_respect_chunk_days_and_cover_window() -> None: start, end = date(2026, 6, 1), date(2026, 6, 30) default_chunks = _mod._chunks(start, end) assert default_chunks == [(start, end)] # 30 天窗口 = 1 块 seven = _mod._chunks(start, end, chunk_days=7) assert len(seven) == 5 assert seven[0] == (date(2026, 6, 1), date(2026, 6, 7)) assert seven[-1][1] == end # 不重不漏:每块首日 = 上块末日 + 1 for (prev_s, prev_e), (nxt_s, _nxt_e) in zip(seven, seven[1:], strict=False): assert (nxt_s - prev_e).days == 1 assert prev_s <= prev_e