"""``cmdb.decommission_reconcile`` — the read-back after a closure.

``decommission_scope`` writes ``status='closed'`` the moment the provider
*accepts* the closure, but every cloud then works on it asynchronously
for weeks. These tests lock in what the daily reconciler does with each
answer the provider can give, and — more importantly — what it refuses
to do:

* it never reopens a scope on the strength of one provider read;
* it never turns a failed read into a confirmed closure;
* it stops re-reading a scope once the answer is terminal, in BOTH
  directions (finished, and alerted-contradiction), so the alert does
  not repeat daily until people learn to ignore it.

Runs against the real test DB; the driver is a stub so no cloud is hit.
"""
from __future__ import annotations

import uuid
from datetime import UTC, datetime, timedelta
from typing import Any

import pytest
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.audit.models import AuditLog
from app.domains.cmdb import decommission_reconcile as reconcile_mod
from app.domains.cmdb.decommission_reconcile import (
    MAX_AUTO_CLOSE_RETRIES,
    MAX_AUTO_RETRY_ATTEMPTS,
    TERMINAL_PROVIDER_STATES,
    reconcile_decommissioned_scopes,
)
from app.domains.cmdb.models import (
    SCOPE_STATUS_ACTIVE,
    SCOPE_STATUS_CLOSED,
    CmdbCloudScope,
)
from app.identity.person_models import IdentityPerson
from app.integrations.driver_base import CloseScopeResult, ScopeClosureState
from app.integrations.models import Integration

pytestmark = pytest.mark.asyncio


# ── helpers ───────────────────────────────────────────────────────

async def _make_integration(db: AsyncSession, provider: str = "alicloud_intl"):
    row = Integration(
        id=uuid.uuid4(), kind="cloud", provider=provider,
        name=f"{provider}-recon-test",
        slug=f"{provider}-recon-{uuid.uuid4().hex[:6]}",
        enabled=True, is_primary=False, capabilities=["servers"],
    )
    db.add(row)
    await db.commit()
    return row


async def _make_closed_scope(
    db: AsyncSession,
    integration: Integration,
    *,
    provider_state: str | None = "deletion_silence_period",
    status: str = SCOPE_STATUS_CLOSED,
    scope_type: str = "aliyun_account",
    days_ago: int = 3,
) -> CmdbCloudScope:
    now = datetime.now(UTC)
    s = CmdbCloudScope(
        id=uuid.uuid4(),
        integration_id=str(integration.id),
        scope_type=scope_type,
        external_id=uuid.uuid4().hex[:16],
        display_name=f"scope-{uuid.uuid4().hex[:6]}",
        status=status,
        first_seen_at=now - timedelta(days=200),
        last_synced_at=now,
        environment="dev",
        decommissioned_at=(
            now - timedelta(days=days_ago)
            if status == SCOPE_STATUS_CLOSED else None
        ),
        decommission_reason="test",
        decommission_provider_state=provider_state,
    )
    db.add(s)
    await db.commit()
    return s


class _StubDriver:
    def __init__(self, state: ScopeClosureState | Exception):
        self._state = state
        self.calls: list[str] = []

    async def get_scope_closure_state(self, ref) -> ScopeClosureState:
        self.calls.append(ref.external_id)
        if isinstance(self._state, Exception):
            raise self._state
        return self._state


def _patch_driver(monkeypatch, driver: Any) -> None:
    async def _fake(_session, _integration):
        return driver
    monkeypatch.setattr(reconcile_mod, "_load_driver", _fake)


async def _audit_actions(db: AsyncSession, scope_id) -> list[tuple[str, str]]:
    rows = (await db.execute(
        select(AuditLog.action, AuditLog.status)
        .where(AuditLog.resource_id == str(scope_id))
    )).all()
    return [(a, s) for a, s in rows]


# ── the four dispositions ─────────────────────────────────────────

async def test_confirmed_closure_advances_state_and_audits(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="closed", provider_state="deleted",
        detail={"aliyun_status": "Success"},
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert summary["checked"] == 1
    assert summary["confirmed_closed"] == 1
    assert summary["state_advanced"] == 1
    assert scope.decommission_provider_state == "deleted"
    assert ("cloud.scope.decommission_confirmed", "success") in (
        await _audit_actions(db_session, scope.id)
    )


async def test_still_pending_keeps_the_scope_in_the_queue(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="pending", provider_state="deletion_silence_period",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert summary["still_pending"] == 1
    assert summary["state_advanced"] == 0  # unchanged string, no write
    assert scope.status == SCOPE_STATUS_CLOSED
    assert scope.decommission_provider_state not in TERMINAL_PROVIDER_STATES


async def test_pending_state_that_moved_on_is_written_back(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """Aliyun Checking → Deleting is still pending, but it is progress the
    archive tab should be able to show."""
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="pending", provider_state="deleting",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert summary["still_pending"] == 1
    assert summary["state_advanced"] == 1
    assert scope.decommission_provider_state == "deleting"


async def test_account_alive_again_alerts_and_never_reopens_the_scope(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """GCP projects.undelete / Azure re-activation are self-service for
    the whole grace window and never tell Helix."""
    integ = await _make_integration(db_session, provider="gcp")
    scope = await _make_closed_scope(
        db_session, integ, provider_state="delete_requested",
        scope_type="gcp_project",
    )
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="alive", provider_state="active",
        error_message="GCP reports this project as ACTIVE",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert summary["reopened_alive"] == 1
    assert scope.decommission_provider_state == "active"
    assert scope.status == SCOPE_STATUS_CLOSED
    assert ("cloud.scope.decommission_contradicted", "failed") in (
        await _audit_actions(db_session, scope.id)
    )


async def test_unreadable_provider_changes_nothing(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """"We could not check" must never become "confirmed closed" — that
    collapse is the exact failure this whole module exists to prevent."""
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="unknown", error_message="403 from provider",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert summary["unreadable"] == 1
    assert summary["confirmed_closed"] == 0
    assert scope.decommission_provider_state == "deletion_silence_period"
    assert await _audit_actions(db_session, scope.id) == []


async def test_a_driver_that_raises_does_not_take_the_sweep_down(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(RuntimeError("boom")))

    summary = await reconcile_decommissioned_scopes(db_session)

    assert summary["checked"] == 1
    assert summary["unreadable"] == 1
    assert any("boom" in e for e in summary["errors"])
    del scope


async def test_not_implemented_is_skipped_quietly(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """IAM-evacuation clouds have no provider-side after-state; that is
    "nothing to reconcile", not an error worth an alert every night."""
    integ = await _make_integration(db_session, provider="tencent")
    await _make_closed_scope(
        db_session, integ, provider_state="pending_closure",
        scope_type="standalone_iam",
    )
    _patch_driver(monkeypatch, _StubDriver(NotImplementedError()))

    summary = await reconcile_decommissioned_scopes(db_session)

    assert summary["unsupported"] == 1
    assert summary["unreadable"] == 0
    assert summary["errors"] == []


# ── queue membership ──────────────────────────────────────────────

@pytest.mark.parametrize("terminal_state", sorted(TERMINAL_PROVIDER_STATES))
async def test_terminal_states_leave_the_queue(
    db_session: AsyncSession, monkeypatch, terminal_state: str,
) -> None:
    """Both directions of terminal: a finished closure, and one already
    alerted on. Re-alerting nightly on the same contradiction is how an
    alert becomes noise."""
    integ = await _make_integration(db_session)
    await _make_closed_scope(db_session, integ, provider_state=terminal_state)
    driver = _StubDriver(ScopeClosureState(disposition="closed"))
    _patch_driver(monkeypatch, driver)

    summary = await reconcile_decommissioned_scopes(db_session)

    assert summary["checked"] == 0
    assert driver.calls == []


async def test_active_scopes_and_legacy_null_state_are_not_swept(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """A NULL provider_state is a legacy / manual-console close with no
    submission to track. Prod carries ~1000 such GCP rows; sweeping them
    would fan out a thousand provider reads to learn nothing."""
    integ = await _make_integration(db_session)
    await _make_closed_scope(db_session, integ, provider_state=None)
    await _make_closed_scope(
        db_session, integ, provider_state="pending_closure",
        status=SCOPE_STATUS_ACTIVE,
    )
    driver = _StubDriver(ScopeClosureState(disposition="closed"))
    _patch_driver(monkeypatch, driver)

    summary = await reconcile_decommissioned_scopes(db_session)

    assert summary["checked"] == 0
    assert driver.calls == []


async def test_over_limit_is_reported_never_silently_dropped(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """A bounded sweep that doesn't say what it skipped reads as "all
    reconciled" when it wasn't."""
    integ = await _make_integration(db_session)
    for _ in range(3):
        await _make_closed_scope(db_session, integ)
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="pending", provider_state="deletion_silence_period",
    )))

    summary = await reconcile_decommissioned_scopes(db_session, limit=2)

    assert summary["checked"] == 2
    assert summary["skipped_over_limit"] == 1


async def test_oldest_submission_is_checked_first(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """Under the cap, the closures that have been silently stuck longest
    are the ones worth the budget."""
    integ = await _make_integration(db_session)
    newest = await _make_closed_scope(db_session, integ, days_ago=1)
    oldest = await _make_closed_scope(db_session, integ, days_ago=60)
    driver = _StubDriver(ScopeClosureState(
        disposition="pending", provider_state="deletion_silence_period",
    ))
    _patch_driver(monkeypatch, driver)

    await reconcile_decommissioned_scopes(db_session, limit=1)

    assert driver.calls == [oldest.external_id]
    assert newest.external_id not in driver.calls


# ── Helix finishes the job; the admin clicked once ────────────────
#
# A closure the provider let die is unfinished work, not news to
# forward. The 2026-09-01 shape detected it and alerted — correct, and
# useless: the admin had already made the only decision there was, and
# handing them a 16-item list of Aliyun check ids asked them to re-do it.


class _RetryDriver:
    """Reports a dead closure, then records the re-submission."""

    def __init__(self, close_result: Any = None, close_exc: Exception | None = None):
        self._close_result = close_result
        self._close_exc = close_exc
        self.close_calls = 0

    async def get_scope_closure_state(self, ref) -> ScopeClosureState:
        return ScopeClosureState(
            disposition="failed", provider_state="deletion_failed",
            error_message="Aliyun reports the member deletion CheckFailed.",
            detail={"fail_reasons": ["instance number:5"]},
        )

    async def close_scope_on_provider(self, ref, **_kw):
        self.close_calls += 1
        if self._close_exc:
            raise self._close_exc
        return self._close_result


def _close_ok(provider_state="deletion_silence_period"):
    return CloseScopeResult(ok=True, provider_state=provider_state)


async def _retry_audits(db: AsyncSession, scope_id) -> int:
    return len([
        a for a, _ in await _audit_actions(db, scope_id)
        if a == "cloud.scope.decommission_auto_retry"
    ])


async def test_failed_closure_is_resubmitted_by_helix(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    driver = _RetryDriver(close_result=_close_ok())
    _patch_driver(monkeypatch, driver)

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert driver.close_calls == 1
    assert summary["auto_retried"] == 1
    assert summary["auto_retry_succeeded"] == 1
    assert summary["stalled"] == 0
    assert scope.decommission_provider_state == "deletion_silence_period"
    # Still Helix's job — it must stay in the queue for tomorrow.
    assert scope.decommission_provider_state not in TERMINAL_PROVIDER_STATES
    assert await _retry_audits(db_session, scope.id) == 1


async def test_a_resubmission_still_in_flight_does_not_burn_the_budget(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """Aliyun's 3-5 minute pre-check means the retry legitimately returns
    "running". That is progress, not a spent attempt — and on a daily
    cadence, charging the budget for it would spend the whole allowance
    on the mechanism working correctly."""
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    driver = _RetryDriver(close_result=CloseScopeResult(
        ok=False, error_code="deletion_precheck_running",
        error_message="pre-check running",
    ))
    _patch_driver(monkeypatch, driver)

    # More runs than the failure budget — every one of them in flight.
    for _ in range(MAX_AUTO_CLOSE_RETRIES + 2):
        await reconcile_decommissioned_scopes(db_session)
        await db_session.commit()

    assert driver.close_calls == MAX_AUTO_CLOSE_RETRIES + 2
    await db_session.refresh(scope)
    assert scope.decommission_provider_state != "closure_stalled"


async def test_perpetually_in_flight_still_hits_an_absolute_ceiling(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """The budget spends on failures only, so the ceiling is what stops a
    closure that reports "in flight" forever from being retried nightly
    for eternity."""
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    driver = _RetryDriver(close_result=CloseScopeResult(
        ok=False, error_code="deletion_precheck_running",
        error_message="pre-check running",
    ))
    _patch_driver(monkeypatch, driver)

    for _ in range(MAX_AUTO_RETRY_ATTEMPTS + 2):
        await reconcile_decommissioned_scopes(db_session)
        await db_session.commit()

    assert driver.close_calls == MAX_AUTO_RETRY_ATTEMPTS
    await db_session.refresh(scope)
    assert scope.decommission_provider_state == "closure_stalled"


async def test_retries_are_bounded_then_handed_to_a_human(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """Retrying forever is how a stuck account stays invisible; the
    budget is what turns "Helix owns it" into "and it will tell you when
    it can't"."""
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    driver = _RetryDriver(close_result=CloseScopeResult(
        ok=False, error_code="delete_failed", error_message="still stuck",
    ))
    _patch_driver(monkeypatch, driver)

    for _ in range(MAX_AUTO_CLOSE_RETRIES):
        await reconcile_decommissioned_scopes(db_session)
        await db_session.commit()
    assert driver.close_calls == MAX_AUTO_CLOSE_RETRIES

    # Budget spent — the next run escalates instead of retrying again.
    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()
    await db_session.refresh(scope)

    assert driver.close_calls == MAX_AUTO_CLOSE_RETRIES
    assert summary["stalled"] == 1
    assert scope.decommission_provider_state == "closure_stalled"
    assert scope.decommission_provider_state in TERMINAL_PROVIDER_STATES
    assert ("cloud.scope.decommission_contradicted", "failed") in (
        await _audit_actions(db_session, scope.id)
    )
    # And it leaves the queue, so the alert is sent once, not nightly.
    after = await reconcile_decommissioned_scopes(db_session)
    assert after["checked"] == 0


async def test_a_retry_that_raises_is_recorded_not_swallowed(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    scope = await _make_closed_scope(db_session, integ)
    driver = _RetryDriver(close_exc=RuntimeError("provider exploded"))
    _patch_driver(monkeypatch, driver)

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()

    assert summary["auto_retried"] == 1
    assert summary["auto_retry_succeeded"] == 0
    assert await _retry_audits(db_session, scope.id) == 1


async def test_alive_is_never_auto_reclosed(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """A person deliberately reversed this closure. Finishing your own
    unfinished work is not the same act as overruling someone."""
    integ = await _make_integration(db_session, provider="gcp")
    await _make_closed_scope(
        db_session, integ, provider_state="delete_requested",
        scope_type="gcp_project",
    )
    driver = _RetryDriver(close_result=_close_ok())

    async def _alive(_ref):
        return ScopeClosureState(
            disposition="alive", provider_state="active",
            error_message="GCP reports this project as ACTIVE",
        )
    driver.get_scope_closure_state = _alive  # type: ignore[method-assign]
    _patch_driver(monkeypatch, driver)

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()

    assert driver.close_calls == 0
    assert summary["reopened_alive"] == 1
    assert summary["auto_retried"] == 0


async def test_completion_notifies_the_person_who_clicked(
    db_session: AsyncSession, monkeypatch,
) -> None:
    """One click, one message: the answer to the action they took."""
    integ = await _make_integration(db_session)
    person = IdentityPerson(
        id=uuid.uuid4(), work_email=f"{uuid.uuid4().hex[:8]}@pingcap.com",
        display_name="Closer", employment_status="active",
        source_system="successfactors", hris_person_id=f"sf-{uuid.uuid4().hex[:8]}",
    )
    db_session.add(person)
    await db_session.commit()
    scope = await _make_closed_scope(db_session, integ)
    scope.decommissioned_by = str(person.id)
    await db_session.commit()

    sent: list[Any] = []

    async def _dispatch(_self, envelope, **_kw):
        sent.append(envelope)
    monkeypatch.setattr(
        reconcile_mod.NotificationService, "dispatch", _dispatch,
    )
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="closed", provider_state="deleted",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)
    await db_session.commit()

    assert summary["completion_notices"] == 1
    assert len(sent) == 1
    assert sent[0].recipient.id == person.id


async def test_completion_without_a_known_closer_notifies_nobody(
    db_session: AsyncSession, monkeypatch,
) -> None:
    integ = await _make_integration(db_session)
    await _make_closed_scope(db_session, integ)
    sent: list[Any] = []

    async def _dispatch(_self, envelope, **_kw):
        sent.append(envelope)
    monkeypatch.setattr(
        reconcile_mod.NotificationService, "dispatch", _dispatch,
    )
    _patch_driver(monkeypatch, _StubDriver(ScopeClosureState(
        disposition="closed", provider_state="deleted",
    )))

    summary = await reconcile_decommissioned_scopes(db_session)

    assert summary["confirmed_closed"] == 1
    assert summary["completion_notices"] == 0
    assert sent == []