"""Per-driver ``get_scope_closure_state`` contract tests. This is the read-back that turns "Helix submitted a closure" into "the cloud confirms the closure happened". The three dispositions that matter are all failure-shaped, and each has a test per cloud: * ``failed`` — the provider says the closure died (Aliyun CheckFailed / DeleteFailed). Nothing retries it; the account is alive and billable. * ``alive`` — the provider says the scope is usable AGAIN. Not hypothetical: ``projects.undelete`` (GCP) and Azure subscription re-activation are self-service for the whole grace window and never reach Helix. * ``unknown`` — the read itself failed. This must NEVER be reported as ``closed``; keeping "we could not check" apart from "the provider confirms it is gone" is the entire point of the mechanism. Every driver mocks its own SDK / HTTP boundary, so no real cloud is hit. """ from __future__ import annotations from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.integrations.driver_base import CloudScopeRef from app.integrations.drivers import aliyun_cloud from app.integrations.drivers.alicloud_intl_cloud import AlicloudIntlCloudDriver from app.integrations.drivers.aliyun_cloud import AliyunCloudDriver from app.integrations.drivers.aws_cloud import AwsCloudDriver from app.integrations.drivers.azure_cloud import AzureCloudDriver from app.integrations.drivers.gcp_cloud import GcpCloudDriver from app.integrations.drivers.tencent_cloud import TencentCloudDriver from app.integrations.drivers.ucloud_cloud import UCloudDriver ALIYUN_SCOPE = CloudScopeRef( scope_type="aliyun_account", external_id="5995487234252122", ) AWS_SCOPE = CloudScopeRef(scope_type="aws_account", external_id="111111111111") GCP_SCOPE = CloudScopeRef(scope_type="gcp_project", external_id="proj-x") AZURE_SCOPE = CloudScopeRef( scope_type="azure_subscription", external_id="00000000-0000-0000-0000-000000000123", ) # ── Aliyun ──────────────────────────────────────────────────────── def _aliyun() -> AliyunCloudDriver: return AliyunCloudDriver( integration_id="int-aliyun", config={"access_key_id": "AK", "access_key_secret": "SK"}, ) def _aliyun_status_stub( *, status: str | None = None, deletion_type: int = 1, deletion_time: str = "", fail_reasons: list[dict] | None = None, status_error: str | None = None, account: dict | None = None, account_error: str | None = None, ): """Fake ``_acs3_call`` for the deletion-status read (+ its fallback).""" calls: list[str] = [] async def fake(**kwargs): action = kwargs["action"] calls.append(action) if action == "GetAccountDeletionStatus": if status_error: raise RuntimeError(status_error) return {"RdAccountDeletionStatus": { "AccountId": kwargs["params"]["AccountId"], "Status": status, "DeletionType": deletion_type, "DeletionTime": deletion_time, "FailReasonList": fail_reasons or [], }} if action == "GetAccount": if account_error: raise RuntimeError(account_error) return {"Account": account or {}} raise AssertionError(f"unexpected action {action}") return fake, calls @pytest.mark.asyncio @pytest.mark.parametrize( ("status", "disposition", "provider_state"), [ ("Success", "closed", "deleted"), ("Checking", "pending", "deletion_silence_period"), ("Deleting", "pending", "deleting"), ("CheckFailed", "failed", "deletion_failed"), ("DeleteFailed", "failed", "deletion_failed"), ], ) async def test_aliyun_maps_every_documented_deletion_status( status: str, disposition: str, provider_state: str, ) -> None: fake, _ = _aliyun_status_stub(status=status) with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == disposition assert state.provider_state == provider_state @pytest.mark.asyncio async def test_aliyun_failed_deletion_carries_the_blocking_items() -> None: """The admin's next action is "go clear these", so the reasons have to reach them — an alert saying only "it failed" is not actionable.""" fake, _ = _aliyun_status_stub( status="DeleteFailed", fail_reasons=[{"Name": "ECS", "Description": "3 instances remain"}], ) with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "failed" assert "3 instances remain" in (state.error_message or "") @pytest.mark.asyncio async def test_aliyun_unknown_status_is_unknown_not_closed() -> None: """A vocabulary we don't recognise must not be guessed into a terminal state — that would retire the scope from reconciliation on the strength of a string nobody has verified.""" fake, _ = _aliyun_status_stub(status="SomeFutureStatus") with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "unknown" assert state.provider_state is None @pytest.mark.asyncio async def test_aliyun_no_task_and_member_gone_is_closed() -> None: fake, calls = _aliyun_status_stub( status_error=( 'Aliyun GetAccountDeletionStatus 409: ' '{"Code":"EntityNotExists.CheckAccount"}' ), account_error=( 'Aliyun GetAccount 404: {"Code":"AccountId.NotExist"}' ), ) with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "closed" assert state.provider_state == "not_in_directory" assert "GetAccount" in calls @pytest.mark.asyncio async def test_aliyun_no_task_but_member_still_live_is_alive() -> None: """Someone switched the member back to a CloudAccount, which cancels the whole close and is a console operation Helix never hears about.""" fake, _ = _aliyun_status_stub( status_error=( 'Aliyun GetAccountDeletionStatus 409: ' '{"Code":"EntityNotExists.CheckAccount"}' ), account={"Type": "CloudAccount", "Status": "SwitchSuccess"}, ) with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "alive" assert state.provider_state == "active" @pytest.mark.asyncio async def test_aliyun_read_error_is_unknown_not_closed() -> None: fake, _ = _aliyun_status_stub( status_error="Aliyun GetAccountDeletionStatus 500: internal", ) with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await _aliyun().get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "unknown" @pytest.mark.asyncio async def test_alicloud_intl_inherits_the_state_read() -> None: """The 海外 tenant is where the CloudAccount members actually live.""" driver = AlicloudIntlCloudDriver( integration_id="int-intl", config={"access_key_id": "AK", "access_key_secret": "SK"}, ) fake, _ = _aliyun_status_stub(status="Success") with patch.object(AliyunCloudDriver, "_acs3_call", side_effect=fake): state = await driver.get_scope_closure_state(ALIYUN_SCOPE) assert state.disposition == "closed" assert aliyun_cloud._RD_HOST == "resourcemanager.aliyuncs.com" # ── AWS ─────────────────────────────────────────────────────────── def _aws() -> AwsCloudDriver: return AwsCloudDriver( integration_id="int-aws", config={"access_key_id": "AKIA", "secret_access_key": "s", "region": "us-east-1"}, ) @pytest.mark.asyncio @pytest.mark.parametrize( ("aws_status", "disposition", "provider_state"), [ ("SUSPENDED", "closed", "suspended"), ("PENDING_CLOSURE", "pending", "pending_closure"), ("ACTIVE", "alive", "active"), ], ) async def test_aws_maps_account_status( aws_status: str, disposition: str, provider_state: str, ) -> None: org = _MagicOrg({"Account": {"Status": aws_status}}) with patch.object(AwsCloudDriver, "_boto_client", return_value=org): state = await _aws().get_scope_closure_state(AWS_SCOPE) assert state.disposition == disposition assert state.provider_state == provider_state class _MagicOrg: def __init__(self, payload=None, exc=None): self._payload = payload self._exc = exc def describe_account(self, **_kwargs): if self._exc: raise self._exc return self._payload @pytest.mark.asyncio async def test_aws_account_gone_from_org_is_closed() -> None: from botocore.exceptions import ClientError org = _MagicOrg(exc=ClientError( {"Error": {"Code": "AccountNotFoundException", "Message": "gone"}}, "DescribeAccount", )) with patch.object(AwsCloudDriver, "_boto_client", return_value=org): state = await _aws().get_scope_closure_state(AWS_SCOPE) assert state.disposition == "closed" assert state.provider_state == "not_in_org" @pytest.mark.asyncio async def test_aws_access_denied_is_unknown_not_closed() -> None: """A credential that lost DescribeAccount must not silently retire every scope under that integration as "confirmed closed".""" from botocore.exceptions import ClientError org = _MagicOrg(exc=ClientError( {"Error": {"Code": "AccessDeniedException", "Message": "nope"}}, "DescribeAccount", )) with patch.object(AwsCloudDriver, "_boto_client", return_value=org): state = await _aws().get_scope_closure_state(AWS_SCOPE) assert state.disposition == "unknown" @pytest.mark.asyncio async def test_aws_standalone_iam_has_no_state_to_reconcile() -> None: with pytest.raises(NotImplementedError): await _aws().get_scope_closure_state( CloudScopeRef(scope_type="standalone_iam", external_id="x"), ) # ── GCP ─────────────────────────────────────────────────────────── def _gcp() -> GcpCloudDriver: return GcpCloudDriver( integration_id="int-gcp", config={"service_account_json": "{}", "organization_id": "123"}, ) @pytest.mark.asyncio @pytest.mark.parametrize( ("gcp_state", "disposition", "provider_state"), [ ("DELETE_REQUESTED", "pending", "delete_requested"), ("DELETE_IN_PROGRESS", "pending", "delete_in_progress"), ("ACTIVE", "alive", "active"), ], ) async def test_gcp_maps_project_state( gcp_state: str, disposition: str, provider_state: str, ) -> None: with patch.object(GcpCloudDriver, "_svc", return_value=MagicMock()), \ patch.object( GcpCloudDriver, "_execute_read", return_value={"state": gcp_state}, ): state = await _gcp().get_scope_closure_state(GCP_SCOPE) assert state.disposition == disposition assert state.provider_state == provider_state @pytest.mark.asyncio async def test_gcp_undeleted_project_is_flagged_alive() -> None: """``projects.undelete`` is self-service for the whole 30-day window and never tells Helix — this read is the only way it is ever noticed.""" with patch.object(GcpCloudDriver, "_svc", return_value=MagicMock()), \ patch.object( GcpCloudDriver, "_execute_read", return_value={"state": "ACTIVE"}, ): state = await _gcp().get_scope_closure_state(GCP_SCOPE) assert state.disposition == "alive" assert "undelete" in (state.error_message or "") # ── Azure ───────────────────────────────────────────────────────── def _azure() -> AzureCloudDriver: return AzureCloudDriver( integration_id="int-azure", config={ "tenant_id": "00000000-0000-0000-0000-000000000001", "client_id": "c", "client_secret": "s", }, ) class _AzureGetStub: def __init__(self, status: int, payload: dict | None): self.status = status self.payload = payload async def __aenter__(self): return self async def __aexit__(self, *exc): return False async def get(self, url, params=None, headers=None): return _Resp(self.status, self.payload) class _Resp: def __init__(self, status_code: int, payload: dict | None): self.status_code = status_code self._payload = payload self.text = "{}" if payload is None else "{}" def json(self): return self._payload or {} @pytest.mark.asyncio @pytest.mark.parametrize( ("azure_state", "disposition", "provider_state"), [ ("Deleted", "closed", "deleted"), ("Disabled", "pending", "disabled"), # ``Warned`` is where an ACCEPTED cancel parks the subscription # on its way to Disabled — measured in prod 2026-09-02, see the # driver's mapping comment. This row read ``alive``/``active`` # until then, which is the exact pair that produces a false # "someone re-activated this" alert and, because ``active`` is # terminal, retires the scope from the reconcile queue. ("Warned", "pending", "warned"), ("Enabled", "alive", "active"), ("PastDue", "alive", "active"), ], ) async def test_azure_maps_subscription_state( azure_state: str, disposition: str, provider_state: str, ) -> None: stub = _AzureGetStub(200, {"state": azure_state}) with patch.object(AzureCloudDriver, "_token", AsyncMock(return_value="b")), \ patch.object(AzureCloudDriver, "_http", return_value=stub): state = await _azure().get_scope_closure_state(AZURE_SCOPE) assert state.disposition == disposition assert state.provider_state == provider_state @pytest.mark.asyncio @pytest.mark.parametrize("azure_state", ["Warned", "Disabled"]) async def test_azure_in_flight_states_never_reach_the_terminal_set( azure_state: str, ) -> None: """A state a healthy closure passes through must not be terminal. The two failure modes this pins are different and both bad: an ``alive`` disposition alerts an admin about a closure that is working, and ANY ``provider_state`` inside ``TERMINAL_PROVIDER_STATES`` removes the scope from the nightly re-read, so the eventual Deleted is never confirmed and a closure that later dies is never noticed. Asserting the disposition alone would miss the second. """ from app.domains.cmdb.decommission_reconcile import ( TERMINAL_PROVIDER_STATES, ) stub = _AzureGetStub(200, {"state": azure_state}) with patch.object(AzureCloudDriver, "_token", AsyncMock(return_value="b")), \ patch.object(AzureCloudDriver, "_http", return_value=stub): state = await _azure().get_scope_closure_state(AZURE_SCOPE) assert state.disposition == "pending" assert state.error_message is None assert state.provider_state not in TERMINAL_PROVIDER_STATES @pytest.mark.asyncio async def test_azure_404_is_closed_and_403_is_unknown() -> None: with patch.object(AzureCloudDriver, "_token", AsyncMock(return_value="b")), \ patch.object( AzureCloudDriver, "_http", return_value=_AzureGetStub(404, None), ): gone = await _azure().get_scope_closure_state(AZURE_SCOPE) assert gone.disposition == "closed" with patch.object(AzureCloudDriver, "_token", AsyncMock(return_value="b")), \ patch.object( AzureCloudDriver, "_http", return_value=_AzureGetStub(403, None), ): denied = await _azure().get_scope_closure_state(AZURE_SCOPE) assert denied.disposition == "unknown" # ── IAM-evacuation clouds have nothing to reconcile ─────────────── @pytest.mark.asyncio @pytest.mark.parametrize("driver_cls", [TencentCloudDriver, UCloudDriver]) async def test_iam_evacuation_drivers_raise_not_implemented(driver_cls) -> None: """Their "closure" is a set of DeleteUser calls with no provider-side after-state. The reconciler reads NotImplementedError as "nothing to reconcile" and skips the scope instead of logging an error forever.""" driver = driver_cls(integration_id="int-x", config={}) with pytest.raises(NotImplementedError): await driver.get_scope_closure_state( CloudScopeRef(scope_type="standalone_iam", external_id="x"), )