from __future__ import annotations from pathlib import Path import pytest from pingcap_datapipelines.netsuite_revenue_admission import AdmissionError, admit_candidates from pingcap_datapipelines.netsuite_revenue_candidate import ( CandidateContractError, LocalCandidateStore, freeze_workbook_fixture, validate_restlet_envelope, ) from pingcap_datapipelines.netsuite_revenue_config import load_revenue_sync_config from pingcap_datapipelines.netsuite_revenue_sync import StaticAccountLookup CONFIG_PATH = Path("configs/jobs/netsuite_revenue_to_nexuscrm.yaml") def _config(): return load_revenue_sync_config(CONFIG_PATH) def _envelope(config, *, source_key: str = "7429932:100:200", raw_account_id="", display_account_id=""): values_by_header = {mapping.header: f"value:{mapping.header}" for mapping in config.mappings} values_by_header["Transaction Amount"] = "12.5" values_by_header["Amount"] = "10.0" values_by_header["Account ID"] = "44" columns = [ {"header": mapping.header, "fieldId": f"column_{index:02d}"} for index, mapping in enumerate(config.mappings, start=1) ] account_field_id = columns[16]["fieldId"] values = {column["fieldId"]: values_by_header[column["header"]] for column in columns} display_values = dict(values) values[account_field_id] = raw_account_id display_values[account_field_id] = display_account_id return { "schemaVersion": "netsuite-saved-search-export/v2", "reportSchemaVersion": "revenue-voucher-excel/v1", "source": {"accountRealm": "7429932", "savedSearchId": "customsearch1710"}, "scope": {"postingPeriodId": "245", "transactionType": "Journal", "accountNumberPrefix": "6001"}, "page": {"cursor": "0", "pageSize": 500, "nextCursor": None}, "columns": columns, "items": [ { "sourceKey": source_key, "sourceIdentity": {"transactionInternalId": "100", "lineUniqueKey": "200", "accountingBookId": "1"}, "values": values, "displayValues": display_values, "reportValues": list(values_by_header.values()), "valuesByHeader": values_by_header, } ], } def test_restlet_candidate_uses_raw_or_display_for_account_admission_not_excel_44(tmp_path: Path) -> None: config = _config() blank_rows, _ = validate_restlet_envelope( config, _envelope(config, raw_account_id="", display_account_id=""), posting_period_id="245", expected_cursor=None, seen_keys=set(), ) literal_44_rows, _ = validate_restlet_envelope( config, _envelope(config, source_key="7429932:101:201", raw_account_id="44", display_account_id=""), posting_period_id="245", expected_cursor=None, seen_keys=set(), ) store = LocalCandidateStore(tmp_path, run_id="admission") admitted, manifest = admit_candidates( [*blank_rows, *literal_44_rows], account_lookup=StaticAccountLookup({"44": ["001-account"]}), store=store, ) assert [row.candidate.source_key for row in admitted] == ["7429932:101:201"] assert admitted[0].candidate.payload[config.account_id_mapping.field] == "44" assert manifest.source_total == 2 assert manifest.admitted_exact_match == 1 assert manifest.excluded_blank_account_id == 1 assert manifest.excluded_missing_account == 0 assert manifest.excluded_ambiguous_account == 0 assert (store.root / "admission/rejection-ledger.jsonl").read_text().count("Blank") == 1 def test_admission_quarantines_missing_and_ambiguous_without_target_candidates(tmp_path: Path) -> None: config = _config() rows = [] for position, account_id in enumerate(("present", "missing", "ambiguous"), start=1): page = _envelope(config, source_key=f"7429932:{position}:{position}", raw_account_id=account_id) item_rows, _ = validate_restlet_envelope( config, page, posting_period_id="245", expected_cursor=None, seen_keys=set() ) rows.extend(item_rows) store = LocalCandidateStore(tmp_path, run_id="quarantine") admitted, manifest = admit_candidates( rows, account_lookup=StaticAccountLookup({"present": ["001"], "missing": [], "ambiguous": ["002", "003"]}), store=store, ) assert len(admitted) == 1 assert ( manifest.source_total == manifest.admitted_exact_match + manifest.excluded_missing_account + manifest.excluded_ambiguous_account ) ledger = (store.root / "admission/rejection-ledger.jsonl").read_text() assert "Missing" in ledger and "Ambiguous" in ledger assert "present" not in ledger def test_schema_cursor_and_duplicate_key_drift_fail_closed() -> None: config = _config() envelope = _envelope(config) envelope["columns"] = list(reversed(envelope["columns"])) with pytest.raises(CandidateContractError, match="35-column order"): validate_restlet_envelope(config, envelope, posting_period_id="245", expected_cursor=None, seen_keys=set()) envelope = _envelope(config) envelope["page"]["nextCursor"] = "0" rows, next_cursor = validate_restlet_envelope( config, envelope, posting_period_id="245", expected_cursor=None, seen_keys=set() ) assert len(rows) == 1 and next_cursor == "0" with pytest.raises(CandidateContractError, match="duplicated"): validate_restlet_envelope( config, envelope, posting_period_id="245", expected_cursor=None, seen_keys={rows[0].source_key} ) def test_local_workbook_fixture_generates_the_verified_1789_row_candidate(tmp_path: Path) -> None: workbook = Path("/Users/amber/Downloads/2607 revenue voucher list.xlsx") if not workbook.is_file(): pytest.skip("verified local workbook fixture is not mounted") config = _config() store = LocalCandidateStore(tmp_path, run_id="workbook-245") rows, manifest = freeze_workbook_fixture( config, workbook=workbook, store=store, run_id="workbook-245", posting_period_id="245", ) assert len(rows) == 1789 assert manifest.source_total == 1789 assert len(manifest.headers) == 35 assert manifest.fixture is True with pytest.raises(AdmissionError, match="raw/display"): admit_candidates(rows, account_lookup=StaticAccountLookup({}), store=store, artifact_prefix="fixture-admission")