"""Unit tests for the Azure CMDB provider. All HTTP mocked via respx. Covers: * capability declaration * constructor rejects missing credentials * subscription fan-out + per-sub 403 tolerance * list_servers parsing (VM → CloudServer + NIC IP resolution) * list_dns_records public zone expansion (A + CNAME + TXT + MX) * list_object_storage normalization (public/private access) * list_load_balancers collapses LB + Application Gateway """ from __future__ import annotations import httpx import pytest import respx from app.domains.cmdb.providers.azure import ( _API_DNS, _API_NETWORK, _API_STORAGE, _API_SUBSCRIPTIONS, _ARM_BASE, _LOGIN_BASE, AzureProvider, ) from app.domains.cmdb.providers.base import ( ProviderCapability, ProviderCredentials, ) from app.domains.cmdb.providers.exceptions import ProviderAuthError _TENANT = "11111111-1111-1111-1111-111111111111" _CID = "22222222-2222-2222-2222-222222222222" _SECRET = "topsecret" _SUB1 = "33333333-3333-3333-3333-333333333333" _SUB2 = "44444444-4444-4444-4444-444444444444" def _creds(**overrides) -> ProviderCredentials: fields = { "tenant_id": _TENANT, "client_id": _CID, "client_secret": _SECRET, } fields.update(overrides) return ProviderCredentials( provider_id="azure-test", provider_type="azure", account_alias="test", fields=fields, ) def _mock_token() -> None: respx.mock.post( f"{_LOGIN_BASE}/{_TENANT}/oauth2/v2.0/token", ).mock(return_value=httpx.Response(200, json={ "access_token": "tok", "token_type": "Bearer", "expires_in": 3599, })) def _mock_two_subs() -> None: respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled", "displayName": "Prod"}, {"subscriptionId": _SUB2, "state": "Enabled", "displayName": "Dev"}, {"subscriptionId": "x", "state": "Disabled", "displayName": "Old"}, ]})) # ── Capability declaration + construction ───────────────────── def test_supported_capabilities_match_impl() -> None: caps = AzureProvider.supported_capabilities() assert ProviderCapability.SERVERS in caps assert ProviderCapability.DNS_RECORDS in caps assert ProviderCapability.OBJECT_STORAGE in caps assert ProviderCapability.LOAD_BALANCERS in caps assert ProviderCapability.DOMAINS not in caps def test_constructor_rejects_missing_creds() -> None: with pytest.raises(ProviderAuthError): AzureProvider(_creds(tenant_id="")) # ── list_servers ────────────────────────────────────────────── @pytest.mark.asyncio @respx.mock async def test_list_servers_fans_out_and_enriches_ips() -> None: _mock_token() # Only one subscription for this test. respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) # SKU table. respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/skus", ).mock(return_value=httpx.Response(200, json={"value": [ {"name": "Standard_D2s_v3", "capabilities": [ {"name": "vCPUs", "value": "2"}, {"name": "MemoryGB", "value": "8"}, ]}, ]})) nic_id = ( f"/subscriptions/{_SUB1}/resourceGroups/rg/providers/" "Microsoft.Network/networkInterfaces/nic1" ) pip_id = ( f"/subscriptions/{_SUB1}/resourceGroups/rg/providers/" "Microsoft.Network/publicIPAddresses/pip1" ) vm = { "id": f"/subscriptions/{_SUB1}/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", "name": "vm1", "location": "eastus", "zones": ["1"], "tags": {"env": "prod"}, "properties": { "vmId": "9f8d4a21-7b1c-4e5f-9a32-cafedeadbeef", "provisioningState": "Succeeded", "hardwareProfile": {"vmSize": "Standard_D2s_v3"}, "storageProfile": { "osDisk": {"osType": "Linux"}, "imageReference": { "publisher": "Canonical", "offer": "0001-com-ubuntu-server-focal", "sku": "20_04-lts-gen2", "version": "latest", }, }, "networkProfile": { "networkInterfaces": [{"id": nic_id}], }, }, } # First call (plain list) + second call ($expand=instanceView) both # hit the same URL with different params. respx keys by URL+method, # so we return the same payload either way but attach instanceView # on the expanded one for the status lookup. expanded_vm = dict(vm) expanded_vm["properties"] = { **vm["properties"], "instanceView": {"statuses": [ {"code": "PowerState/running"}, {"code": "ProvisioningState/succeeded"}, ]}, } respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/virtualMachines", ).mock(side_effect=[ httpx.Response(200, json={"value": [vm]}), httpx.Response(200, json={"value": [expanded_vm]}), ]) respx.mock.get( f"{_ARM_BASE}{nic_id}", params={"api-version": _API_NETWORK}, ).mock(return_value=httpx.Response(200, json={ "properties": {"ipConfigurations": [ {"properties": { "privateIPAddress": "10.0.0.5", "publicIPAddress": {"id": pip_id}, }}, ]}, })) respx.mock.get( f"{_ARM_BASE}{pip_id}", params={"api-version": _API_NETWORK}, ).mock(return_value=httpx.Response(200, json={ "properties": {"ipAddress": "52.1.2.3"}, })) provider = AzureProvider(_creds()) try: servers = await provider.list_servers() finally: await provider.close() assert len(servers) == 1 s = servers[0] assert s.name == "vm1" # ``instance_id`` must be the short 36-char ``properties.vmId`` # GUID, NOT the full ARM resource path — the ARM path is ~180 # chars and overflows ``cmdb_servers.instance_id VARCHAR(128)``. # Regression guard for the 2026-04-24 sync failure. assert s.instance_id == "9f8d4a21-7b1c-4e5f-9a32-cafedeadbeef" assert len(s.instance_id) <= 128 # Full ARM path still available in raw for debugging / cross-ref. assert s.raw.get("id", "").endswith("/virtualMachines/vm1") assert s.status == "running" assert s.region == "eastus" assert s.zone == "1" assert s.vcpu == 2 assert s.memory_mb == 8 * 1024 assert s.private_ips == ["10.0.0.5"] assert s.public_ips == ["52.1.2.3"] assert s.os_type == "linux" assert s.tags == {"env": "prod"} @pytest.mark.asyncio @respx.mock async def test_list_servers_per_subscription_403_is_skipped() -> None: _mock_token() _mock_two_subs() respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/skus", ).mock(return_value=httpx.Response(200, json={"value": []})) # SUB1 → 403 (skip), SUB2 → empty. respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(403, json={"error": "denied"})) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB2}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(200, json={"value": []})) provider = AzureProvider(_creds()) try: servers = await provider.list_servers() finally: await provider.close() assert servers == [] # ── Round-26 (2026-05-18) admin-state 4xx classifier ──────────────── def test_classify_azure_4xx_missing_rp_with_target() -> None: """``MissingSubscriptionRegistration`` with ``details[0].target`` carrying the RP namespace classifies as ``rp_not_registered:Microsoft.Compute``.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = """{ "error": { "code": "MissingSubscriptionRegistration", "message": "The subscription is not registered to use namespace 'Microsoft.Compute'.", "details": [ {"target": "Microsoft.Compute", "code": "MissingSubscriptionRegistration"} ] } }""" assert _classify_azure_4xx_admin_skip(body) == ( "rp_not_registered", "Microsoft.Compute", ) def test_classify_azure_4xx_missing_rp_parses_namespace_from_message() -> None: """When ``details[*].target`` is absent the classifier falls back to extracting the namespace from the ``error.message`` text.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = """{ "error": { "code": "MissingSubscriptionRegistration", "message": "The subscription is not registered to use namespace 'Microsoft.Network'." } }""" assert _classify_azure_4xx_admin_skip(body) == ( "rp_not_registered", "Microsoft.Network", ) def test_classify_azure_4xx_subscription_not_found() -> None: from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = '{"error": {"code": "SubscriptionNotFound", "message": "Subscription X was not found."}}' assert _classify_azure_4xx_admin_skip(body) == ("sub_not_found", None) def test_classify_azure_4xx_disabled_subscription() -> None: from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = '{"error": {"code": "DisabledSubscription", "message": "Billing disabled."}}' assert _classify_azure_4xx_admin_skip(body) == ("sub_disabled", None) def test_classify_azure_4xx_subscription_does_not_exist_alias() -> None: """``SubscriptionDoesNotExist`` is a close alias of NotFound emitted by some ARM endpoints — must classify to the same ``sub_not_found`` bucket.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = '{"error": {"code": "SubscriptionDoesNotExist", "message": "gone"}}' assert _classify_azure_4xx_admin_skip(body) == ("sub_not_found", None) def test_classify_azure_4xx_real_authorization_failed_returns_none() -> None: """A genuine RBAC denial (``AuthorizationFailed``) MUST stay None so the failure-notifier pages and the admin fixes RBAC.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = """{ "error": { "code": "AuthorizationFailed", "message": "The client 'x' with object id 'y' does not have authorization." } }""" assert _classify_azure_4xx_admin_skip(body) is None def test_classify_azure_4xx_unknown_code_returns_none() -> None: from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip body = '{"error": {"code": "SomeBrandNewCode", "message": "weird"}}' assert _classify_azure_4xx_admin_skip(body) is None def test_classify_azure_4xx_malformed_body_returns_none() -> None: """Empty / non-JSON / oversized bodies must all return None.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip assert _classify_azure_4xx_admin_skip("") is None assert _classify_azure_4xx_admin_skip("503 from load balancer") is None assert _classify_azure_4xx_admin_skip("not json at all") is None # Body without ``error`` envelope. assert _classify_azure_4xx_admin_skip('{"foo": 1}') is None # Body where ``error.code`` is missing. assert _classify_azure_4xx_admin_skip('{"error": {"message": "x"}}') is None # Body where ``error.code`` is non-string. assert _classify_azure_4xx_admin_skip('{"error": {"code": 123}}') is None def test_classify_azure_4xx_oversized_body_short_circuits() -> None: """Bodies >64KB return None without attempting JSON parse — defense against pathological HTML interstitials and worker memory pressure.""" from app.domains.cmdb.providers.azure import _classify_azure_4xx_admin_skip huge = "x" * (65 * 1024) assert _classify_azure_4xx_admin_skip(huge) is None def test_azure_4xx_error_message_extracts_redacted_snippet() -> None: from app.domains.cmdb.providers.azure import _azure_4xx_error_message body = '{"error": {"code": "AuthorizationFailed", "message": "Client lacks RBAC."}}' msg = _azure_4xx_error_message(body) assert msg is not None assert "RBAC" in msg def test_azure_4xx_error_message_none_on_empty() -> None: from app.domains.cmdb.providers.azure import _azure_4xx_error_message assert _azure_4xx_error_message("") is None assert _azure_4xx_error_message('{"error": {"code": "X"}}') is None assert _azure_4xx_error_message('{"error": {"code": "X", "message": " "}}') is None @pytest.mark.asyncio @respx.mock async def test_list_servers_missing_rp_routes_to_skipped_legs() -> None: """End-to-end: a ``MissingSubscriptionRegistration`` 4xx on a fanned-out subscription must land in ``skipped_legs`` (capability stays green), NOT ``partial_failures`` (would page IT admin every cron tick).""" _mock_token() _mock_two_subs() respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/skus", ).mock(return_value=httpx.Response(200, json={"value": []})) rp_not_registered_body = { "error": { "code": "MissingSubscriptionRegistration", "message": "The subscription is not registered to use namespace 'Microsoft.Compute'.", "details": [ {"target": "Microsoft.Compute", "code": "MissingSubscriptionRegistration"}, ], }, } respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(409, json=rp_not_registered_body)) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB2}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(200, json={"value": []})) provider = AzureProvider(_creds()) try: servers = await provider.list_servers() finally: await provider.close() assert servers == [] # MUST NOT pollute partial_failures (would flip capability red). assert provider.partial_failures == [] # MUST land in skipped_legs with the structured reason. assert len(provider.skipped_legs) == 1 leg = provider.skipped_legs[0] assert leg["leg"] == f"subscription:{_SUB1}" assert leg["reason"] == "rp_not_registered:Microsoft.Compute" @pytest.mark.asyncio @respx.mock async def test_list_servers_real_auth_failure_attaches_error_message() -> None: """End-to-end: real ``AuthorizationFailed`` 4xx stays in ``partial_failures`` AND now carries ``error_message`` so the admin sees what ARM said without log-grepping.""" _mock_token() _mock_two_subs() respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/skus", ).mock(return_value=httpx.Response(200, json={"value": []})) auth_failed_body = { "error": { "code": "AuthorizationFailed", "message": ( "The client 'sp-helix' with object id 'abc' does not " "have authorization to perform action." ), }, } respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(403, json=auth_failed_body)) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB2}/providers/Microsoft.Compute/virtualMachines", ).mock(return_value=httpx.Response(200, json={"value": []})) provider = AzureProvider(_creds()) try: servers = await provider.list_servers() finally: await provider.close() assert servers == [] # Real auth failure MUST stay in partial_failures so failure-notifier pages. assert provider.skipped_legs == [] assert len(provider.partial_failures) == 1 leg = provider.partial_failures[0] assert leg["leg"] == f"subscription:{_SUB1}" assert leg["reason"] == "http_403" # Round-26 enrichment: error_message present and actionable. assert "error_message" in leg assert "authorization" in leg["error_message"].lower() # ── list_dns_records ───────────────────────────────────────── @pytest.mark.asyncio @respx.mock async def test_list_dns_records_public_zone_expansion() -> None: _mock_token() respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) zone_id = ( f"/subscriptions/{_SUB1}/resourceGroups/rg/providers/" "Microsoft.Network/dnszones/example.com" ) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Network/dnszones", params={"api-version": _API_DNS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"id": zone_id, "name": "example.com"}, ]})) respx.mock.get( f"{_ARM_BASE}{zone_id}/recordsets", params={"api-version": _API_DNS}, ).mock(return_value=httpx.Response(200, json={"value": [ { "id": f"{zone_id}/A/www", "name": "www", "properties": { "TTL": 300, "aRecords": [{"ipv4Address": "1.2.3.4"}, {"ipv4Address": "5.6.7.8"}], }, }, { "id": f"{zone_id}/CNAME/docs", "name": "docs", "properties": {"TTL": 600, "cnameRecord": {"cname": "target.example.com"}}, }, { "id": f"{zone_id}/MX/@", "name": "@", "properties": {"TTL": 3600, "mxRecords": [ {"preference": 10, "exchange": "mail.example.com"}, ]}, }, ]})) # Private DNS → 403 (skip). respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Network/privateDnsZones", ).mock(return_value=httpx.Response(403, json={"error": "forbidden"})) provider = AzureProvider(_creds()) try: records = await provider.list_dns_records() finally: await provider.close() # 2 A values + 1 CNAME + 1 MX = 4 rows. assert len(records) == 4 types = sorted({r.record_type for r in records}) assert types == ["A", "CNAME", "MX"] a_values = sorted(r.value for r in records if r.record_type == "A") assert a_values == ["1.2.3.4", "5.6.7.8"] cname = [r for r in records if r.record_type == "CNAME"][0] assert cname.value == "target.example.com" # ── list_object_storage ───────────────────────────────────── @pytest.mark.asyncio @respx.mock async def test_list_object_storage_normalizes_access_flag() -> None: _mock_token() respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Storage/storageAccounts", params={"api-version": _API_STORAGE}, ).mock(return_value=httpx.Response(200, json={"value": [ { "name": "prodstorage", "id": f"/subscriptions/{_SUB1}/.../storageAccounts/prodstorage", "location": "eastus", "kind": "StorageV2", "sku": {"name": "Standard_LRS"}, "properties": { "allowBlobPublicAccess": False, "provisioningState": "Succeeded", }, }, { "name": "publicstorage", "id": f"/subscriptions/{_SUB1}/.../storageAccounts/publicstorage", "location": "eastus", "kind": "StorageV2", "sku": {"name": "Standard_GRS"}, "properties": { "allowBlobPublicAccess": True, "provisioningState": "Succeeded", }, }, ]})) provider = AzureProvider(_creds()) try: buckets = await provider.list_object_storage() finally: await provider.close() by_name = {b["bucket_name"]: b for b in buckets} assert by_name["prodstorage"]["access_type"] == "private" assert by_name["prodstorage"]["region"] == "eastus" assert by_name["prodstorage"]["storage_class"] == "Standard_LRS" assert by_name["publicstorage"]["access_type"] == "public" # ── list_load_balancers ───────────────────────────────────── @pytest.mark.asyncio @respx.mock async def test_list_load_balancers_collapses_lb_and_appgw() -> None: _mock_token() respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Network/loadBalancers", ).mock(return_value=httpx.Response(200, json={"value": [ { "id": f"/subscriptions/{_SUB1}/.../loadBalancers/lb1", "name": "lb1", "location": "eastus", "sku": {"name": "Standard"}, "properties": { "provisioningState": "Succeeded", "frontendIPConfigurations": [{ "properties": {"publicIPAddress": {"id": "pip-ref"}}, }], }, }, ]})) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Network/applicationGateways", ).mock(return_value=httpx.Response(200, json={"value": [ { "id": f"/subscriptions/{_SUB1}/.../applicationGateways/agw1", "name": "agw1", "location": "eastus", "properties": { "provisioningState": "Succeeded", "frontendIPConfigurations": [{ "properties": {"privateIPAddress": "10.0.0.100"}, }], }, }, ]})) provider = AzureProvider(_creds()) try: lbs = await provider.list_load_balancers() finally: await provider.close() assert len(lbs) == 2 by_name = {lb.name: lb for lb in lbs} assert by_name["lb1"].network_type == "public" assert by_name["lb1"].lb_type in {"standard", "network"} assert by_name["agw1"].lb_type == "application" assert by_name["agw1"].ip_address == "10.0.0.100" # ── instance_id regression (2026-04-24) ───────────────────────── def test_vm_to_server_uses_vm_guid_not_arm_path() -> None: """Unit-level regression: the ARM path ``vm.id`` is ~180 chars and overflows ``cmdb_servers.instance_id VARCHAR(128)``. The helper must pick ``properties.vmId`` (a 36-char GUID) instead. """ from app.domains.cmdb.providers.azure import _vm_to_server vm = { "id": ( f"/subscriptions/{_SUB1}/resourceGroups/LONG-GROUP/providers/" "Microsoft.Compute/virtualMachines/some-very-long-vm-name-that" "-pushes-the-total-path-past-128-chars" ), "name": "some-very-long-vm-name-that-pushes-the-total-path-past-128-chars", "location": "eastus", "properties": { "vmId": "12345678-1234-1234-1234-123456789012", "provisioningState": "Succeeded", }, } server = _vm_to_server( vm, _SUB1, nic_lookup={}, power_state_lookup={}, sku_table={}, ) assert server.instance_id == "12345678-1234-1234-1234-123456789012" assert len(server.instance_id) == 36 def test_vm_to_server_falls_back_to_sha1_when_vm_id_missing() -> None: """If Azure unexpectedly returns no ``properties.vmId`` (shouldn't happen for normal VMs but possible on stub/mock data), the helper falls back to SHA1 of the ARM path — 40 chars, deterministic, still fits in VARCHAR(128). The original ARM path stays in ``raw``. """ from app.domains.cmdb.providers.azure import _vm_to_server arm_path = ( f"/subscriptions/{_SUB1}/resourceGroups/rg/providers/" "Microsoft.Compute/virtualMachines/vm-without-vmid" ) vm = { "id": arm_path, "name": "vm-without-vmid", "location": "eastus", "properties": {"provisioningState": "Succeeded"}, # no vmId } server = _vm_to_server( vm, _SUB1, nic_lookup={}, power_state_lookup={}, sku_table={}, ) # SHA1 is 40 hex chars — well within VARCHAR(128). assert len(server.instance_id) == 40 assert all(c in "0123456789abcdef" for c in server.instance_id) # Deterministic — same input → same hash. again = _vm_to_server( vm, _SUB1, nic_lookup={}, power_state_lookup={}, sku_table={}, ) assert again.instance_id == server.instance_id assert server.raw["id"] == arm_path # ── Transport-layer retry helper (2026-05-24) ─────────────────────── # # Regression coverage for the 2026-05-24 prod incident where # ``cmdb.resource_sync 6310dc15-.../object_storage`` failed with # ``httpx.ReadTimeout × 1 [subscription:d3a296ee-...]``. Pre-fix the # Azure driver's HTTP call sites had NO transport-layer retry — a # single ReadTimeout / ConnectError / PoolTimeout bypassed every # status-code branch and landed in ``partial_failures``. class _FakeAsyncClient: """Minimal stub for testing :func:`_request_with_transport_retry`. Each ``get``/``post`` call returns the next value from ``responses`` — if the value is an Exception subclass it's raised, otherwise it's returned as-is. The fake records every call in ``calls`` so assertions can verify the retry budget burned. """ def __init__(self, responses: list) -> None: self.responses = list(responses) self.calls: list[tuple[str, str, dict]] = [] async def get(self, url: str, **kwargs): return self._dispatch("GET", url, **kwargs) async def post(self, url: str, **kwargs): return self._dispatch("POST", url, **kwargs) def _dispatch(self, method: str, url: str, **kwargs): self.calls.append((method, url, kwargs)) item = self.responses.pop(0) if isinstance(item, BaseException): raise item return item @pytest.mark.asyncio async def test_request_with_transport_retry_read_timeout_then_success() -> None: """ReadTimeout on attempt 1 retries, succeeds on attempt 2.""" from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([httpx.ReadTimeout("timed out"), success]) resp = await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_read_timeout", ) assert resp.status_code == 200 assert len(client.calls) == 2 @pytest.mark.asyncio async def test_request_with_transport_retry_covers_connect_error() -> None: from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([ httpx.ConnectError("nope"), success, ]) resp = await _request_with_transport_retry( "POST", client, "https://example.com", endpoint_label="test_connect", ) assert resp.status_code == 200 assert len(client.calls) == 2 @pytest.mark.asyncio async def test_request_with_transport_retry_covers_pool_timeout() -> None: from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([httpx.PoolTimeout("pool"), success]) resp = await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_pool", ) assert resp.status_code == 200 assert len(client.calls) == 2 @pytest.mark.asyncio async def test_request_with_transport_retry_covers_remote_protocol_error() -> None: from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([ httpx.RemoteProtocolError("framing"), success, ]) resp = await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_remote", ) assert resp.status_code == 200 assert len(client.calls) == 2 @pytest.mark.asyncio async def test_request_with_transport_retry_covers_write_and_read_errors() -> None: """WriteError / ReadError on the wire are part of the retry class — h11 raises these for half-closed sockets.""" from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([ httpx.WriteError("half-close"), httpx.ReadError("socket reset"), success, ]) resp = await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_w_r", ) assert resp.status_code == 200 assert len(client.calls) == 3 @pytest.mark.asyncio async def test_request_with_transport_retry_budget_exhausted_reraises() -> None: """5 attempts then re-raise the LAST transport exception (preserves type for ``format_exception_for_log`` — the PR #465 observability guarantee that failure-notifier text shows the real httpx class). """ from app.domains.cmdb.providers.azure import _request_with_transport_retry client = _FakeAsyncClient([ httpx.ReadTimeout("1"), httpx.ReadTimeout("2"), httpx.ReadTimeout("3"), httpx.ReadTimeout("4"), httpx.ReadTimeout("5"), ]) with pytest.raises(httpx.ReadTimeout): await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_exhausted", ) # All 5 attempts burned. assert len(client.calls) == 5 @pytest.mark.asyncio async def test_request_with_transport_retry_passes_kwargs_through() -> None: """``params=`` / ``headers=`` / ``json=`` / ``data=`` forward to ``client.request`` verbatim.""" from app.domains.cmdb.providers.azure import _request_with_transport_retry success = httpx.Response(200, json={"ok": True}) client = _FakeAsyncClient([success]) await _request_with_transport_retry( "POST", client, "https://example.com", endpoint_label="test_kwargs", params={"q": "x"}, headers={"Authorization": "Bearer tok"}, json={"a": 1}, ) method, url, kwargs = client.calls[0] assert method == "POST" assert url == "https://example.com" assert kwargs["params"] == {"q": "x"} assert kwargs["headers"] == {"Authorization": "Bearer tok"} assert kwargs["json"] == {"a": 1} @pytest.mark.asyncio async def test_request_with_transport_retry_does_not_retry_status_codes() -> None: """A 500 response is NOT a transport error — the helper returns it verbatim so the caller can apply its own status policy. (Confirms the helper's invariant: only transport exceptions trigger retry.)""" from app.domains.cmdb.providers.azure import _request_with_transport_retry five_hundred = httpx.Response(500, text="boom") client = _FakeAsyncClient([five_hundred]) resp = await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_status", ) assert resp.status_code == 500 assert len(client.calls) == 1 @pytest.mark.asyncio async def test_request_with_transport_retry_uses_exponential_backoff( monkeypatch: pytest.MonkeyPatch, ) -> None: """Backoff doubles per attempt with a 60s ceiling. We intercept ``asyncio.sleep`` to assert the delay sequence without burning real wall-clock time in CI.""" from app.domains.cmdb.providers import azure as _azure_mod from app.domains.cmdb.providers.azure import _request_with_transport_retry sleeps: list[float] = [] async def _fake_sleep(d: float) -> None: sleeps.append(d) monkeypatch.setattr(_azure_mod.asyncio, "sleep", _fake_sleep) client = _FakeAsyncClient([ httpx.ReadTimeout("1"), httpx.ReadTimeout("2"), httpx.ReadTimeout("3"), httpx.Response(200, json={"ok": True}), ]) await _request_with_transport_retry( "GET", client, "https://example.com", endpoint_label="test_backoff", ) # 3 sleeps for 3 retries → 1, 2, 4 seconds (2 ** 0, 2 ** 1, 2 ** 2). assert sleeps == [1, 2, 4] @pytest.mark.asyncio @respx.mock async def test_arm_get_retries_on_read_timeout() -> None: """Integration: _arm_get successfully recovers from a transport flake. Mirrors the prod failure path on the object_storage leg.""" _mock_token() # First call: ReadTimeout. Second call: 200 with the storage list. respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Storage/storageAccounts", ).mock(side_effect=[ httpx.ReadTimeout("simulated"), httpx.Response(200, json={"value": [ { "name": "acct1", "location": "eastus", "kind": "StorageV2", "sku": {"name": "Standard_LRS"}, "properties": {"provisioningState": "Succeeded"}, }, ]}), ]) # Only one subscription. respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) provider = AzureProvider(_creds()) try: # Need a faster sleep for the retry path; patch the module sleep. from app.domains.cmdb.providers import azure as _azure_mod async def _noop_sleep(_d: float) -> None: return None original_sleep = _azure_mod.asyncio.sleep _azure_mod.asyncio.sleep = _noop_sleep try: storage = await provider.list_object_storage() finally: _azure_mod.asyncio.sleep = original_sleep finally: await provider.close() assert len(storage) == 1 assert storage[0]["bucket_name"] == "acct1" # ── Round-30 (2026-05-24) ARM 5xx retry ────────────────────────── @pytest.mark.asyncio @respx.mock async def test_arm_get_retries_on_transient_5xx( monkeypatch: pytest.MonkeyPatch, ) -> None: """Prod incident 2026-05-24: a single 5xx on a NIC GET flipped the resource_sync cron to red. New behaviour: ``_arm_request`` retries 5xx with exponential backoff up to ``_DEFAULT_ARM_5XX_RETRIES`` extra attempts. First two 5xx then a 200 should now succeed.""" from app.domains.cmdb.providers import azure as _azure_mod _mock_token() respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) # SUB1 storage list: 502, 503, 200 (3rd attempt = initial + 2 retries) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Storage/storageAccounts", ).mock(side_effect=[ httpx.Response(502, text="bad gateway"), httpx.Response(503, text="busy"), httpx.Response(200, json={"value": [ { "name": "acct1", "location": "eastus", "kind": "StorageV2", "sku": {"name": "Standard_LRS"}, "properties": {"provisioningState": "Succeeded"}, }, ]}), ]) sleeps: list[float] = [] async def _capture_sleep(d: float) -> None: sleeps.append(d) monkeypatch.setattr(_azure_mod.asyncio, "sleep", _capture_sleep) provider = AzureProvider(_creds()) try: storage = await provider.list_object_storage() finally: await provider.close() assert len(storage) == 1 assert storage[0]["bucket_name"] == "acct1" # First 5xx → sleep(1), second 5xx → sleep(2). No 3rd sleep # because the 200 returns immediately. assert 1 in sleeps and 2 in sleeps @pytest.mark.asyncio @respx.mock async def test_arm_get_exhausted_5xx_records_partial_failure( monkeypatch: pytest.MonkeyPatch, ) -> None: """When 5xx persists past the retry budget, _arm_get raises ProviderTransientError with an "(after N attempts)" message — ``_fan_out_per_subscription``'s generic exception catcher logs it and lands a partial_failure leg (the sync engine's mechanism for surfacing transient capability errors).""" from app.domains.cmdb.providers import azure as _azure_mod _mock_token() respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": [ {"subscriptionId": _SUB1, "state": "Enabled"}, ]})) # All 3 attempts return 503. respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Storage/storageAccounts", ).mock(return_value=httpx.Response(503, text="busy")) async def _noop_sleep(_d: float) -> None: return None monkeypatch.setattr(_azure_mod.asyncio, "sleep", _noop_sleep) provider = AzureProvider(_creds()) try: result = await provider.list_object_storage() partials = provider.partial_failures finally: await provider.close() # Capability returns no rows when the only sub 503'd through the # full retry budget — admin sees partial_failure with the leg. assert result == [] assert any( _SUB1 in leg.get("leg", "") and leg.get("capability") == "object_storage" and leg.get("reason") == "ProviderTransientError" for leg in partials ), partials @pytest.mark.asyncio async def test_arm_request_5xx_retry_budget_returns_final_resp( monkeypatch: pytest.MonkeyPatch, ) -> None: """Unit test of :meth:`_arm_request` budget exhaustion. After ``_DEFAULT_ARM_5XX_RETRIES + 1`` attempts all returning 5xx, the final 5xx response is returned (caller decides whether to raise ProviderTransientError). Verifies the helper that backs both ``_arm_get`` and the ``_arm_list_all`` nextLink hop — pre-fix the nextLink path had NO 5xx retry and dropped pagination on a single transient flake.""" from app.domains.cmdb.providers import azure as _azure_mod from app.domains.cmdb.providers.azure import ( _DEFAULT_ARM_5XX_RETRIES, ) sleeps: list[float] = [] async def _capture_sleep(d: float) -> None: sleeps.append(d) monkeypatch.setattr(_azure_mod.asyncio, "sleep", _capture_sleep) # Pre-cache the token so _arm_request doesn't try to fetch it. provider = AzureProvider(_creds()) provider._token_cache[_azure_mod._ARM_BASE] = "tok" fake = _FakeAsyncClient([ httpx.Response(503, text="busy"), httpx.Response(502, text="bad gateway"), httpx.Response(503, text="busy again"), ]) monkeypatch.setattr(provider, "_http", lambda: fake) try: resp = await provider._arm_request( "https://management.azure.com/subscriptions/x/y?api-version=z", endpoint_label="test_budget", ) finally: await provider.close() # All N+1 attempts burned. assert len(fake.calls) == _DEFAULT_ARM_5XX_RETRIES + 1 # Final response is the last 5xx (caller raises ProviderTransientError). assert resp.status_code == 503 assert resp.text == "busy again" # Sleep delays: 1, 2 (exponential, ``_DEFAULT_ARM_5XX_RETRIES`` total # gaps for ``_DEFAULT_ARM_5XX_RETRIES + 1`` attempts — last attempt # doesn't sleep because the loop exits via ``return resp`` once the # budget is exhausted). assert sleeps == [1, 2] @pytest.mark.asyncio async def test_arm_request_401_invalidates_token_cache( monkeypatch: pytest.MonkeyPatch, ) -> None: """Round-30 code-review fix: a 401 mid-sync (token rotation / expiry) must drop the cached token so the next ARM call re-fetches. Without this, dozens of in-flight NIC GETs all reuse the stale cached token and 401 in lockstep — the entire capability flips red even though a single refresh would recover.""" from app.domains.cmdb.providers import azure as _azure_mod async def _noop_sleep(_d: float) -> None: return None monkeypatch.setattr(_azure_mod.asyncio, "sleep", _noop_sleep) provider = AzureProvider(_creds()) # Pre-populate the cache with a stale token. provider._token_cache[_azure_mod._ARM_BASE] = "stale-tok" fake = _FakeAsyncClient([ httpx.Response(401, text="token expired"), ]) monkeypatch.setattr(provider, "_http", lambda: fake) try: resp = await provider._arm_request( "https://management.azure.com/x?api-version=z", endpoint_label="test_401_invalidate", ) finally: await provider.close() assert resp.status_code == 401 # Token cache was invalidated so the next call will re-fetch. assert _azure_mod._ARM_BASE not in provider._token_cache @pytest.mark.asyncio async def test_arm_request_5xx_then_recovers( monkeypatch: pytest.MonkeyPatch, ) -> None: """Unit test: ``_arm_request`` returns the first 2xx after retrying past intermediate 5xx — proves the nextLink path now recovers without manual intervention.""" from app.domains.cmdb.providers import azure as _azure_mod async def _noop_sleep(_d: float) -> None: return None monkeypatch.setattr(_azure_mod.asyncio, "sleep", _noop_sleep) provider = AzureProvider(_creds()) provider._token_cache[_azure_mod._ARM_BASE] = "tok" success_body = {"value": [{"name": "ok"}], "nextLink": None} fake = _FakeAsyncClient([ httpx.Response(503, text="busy"), httpx.Response(200, json=success_body), ]) monkeypatch.setattr(provider, "_http", lambda: fake) try: resp = await provider._arm_request( "https://management.azure.com/subscriptions/x/y?api-version=z", endpoint_label="test_recover", ) finally: await provider.close() assert resp.status_code == 200 assert resp.json() == success_body assert len(fake.calls) == 2 @pytest.mark.asyncio @respx.mock async def test_token_endpoint_5xx_retries( monkeypatch: pytest.MonkeyPatch, ) -> None: """AAD token endpoint 5xx is now retried — prior behaviour aborted the entire sync on a single AAD hiccup.""" from app.domains.cmdb.providers import azure as _azure_mod respx.mock.post( f"{_LOGIN_BASE}/{_TENANT}/oauth2/v2.0/token", ).mock(side_effect=[ httpx.Response(503, text="aad busy"), httpx.Response(200, json={ "access_token": "tok", "token_type": "Bearer", "expires_in": 3599, }), ]) respx.mock.get( f"{_ARM_BASE}/subscriptions", params={"api-version": _API_SUBSCRIPTIONS}, ).mock(return_value=httpx.Response(200, json={"value": []})) async def _noop_sleep(_d: float) -> None: return None monkeypatch.setattr(_azure_mod.asyncio, "sleep", _noop_sleep) provider = AzureProvider(_creds()) try: # Trigger ``_token`` indirectly via any ARM-listing call. await provider.list_servers() finally: await provider.close() @pytest.mark.asyncio @respx.mock async def test_fan_out_per_subscription_attaches_capability_label( monkeypatch: pytest.MonkeyPatch, ) -> None: """Subscription-level partial_failures and skipped_legs now carry the ``capability`` field, matching the NIC-level leg. Without this the failure-notifier could not differentiate which capability a leg belonged to in its dedup signature.""" from app.domains.cmdb.providers import azure as _azure_mod _mock_token() _mock_two_subs() # SUB1 → admin-config skip (MissingSubscriptionRegistration → skipped_legs) # SUB2 → non-403/404 4xx (lands in partial_failures via _fan_out's # generic 4xx branch; ``_list_storage_for_subscription`` short- # circuits 403/404 itself, so we pick 400 to ensure the leg reaches # the fan-out catcher). respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB1}/providers/Microsoft.Storage/storageAccounts", ).mock(return_value=httpx.Response(409, json={ "error": { "code": "MissingSubscriptionRegistration", "message": "namespace 'Microsoft.Storage' not registered", "details": [{ "code": "MissingSubscriptionRegistration", "target": "Microsoft.Storage", }], }, })) respx.mock.get( f"{_ARM_BASE}/subscriptions/{_SUB2}/providers/Microsoft.Storage/storageAccounts", ).mock(return_value=httpx.Response(400, json={ "error": {"code": "BadRequest", "message": "malformed"}, })) async def _noop_sleep(_d: float) -> None: return None monkeypatch.setattr(_azure_mod.asyncio, "sleep", _noop_sleep) provider = AzureProvider(_creds()) try: await provider.list_object_storage() partial = provider.partial_failures skipped = provider.skipped_legs finally: await provider.close() # SUB2's 403 lands in partial_failures with capability='object_storage'. assert any( leg.get("capability") == "object_storage" and _SUB2 in leg.get("leg", "") for leg in partial ), partial # SUB1's MissingSubscriptionRegistration lands in skipped_legs with # capability='object_storage'. assert any( leg.get("capability") == "object_storage" and _SUB1 in leg.get("leg", "") for leg in skipped ), skipped