"""Requestable-dataset schema browser HTTP contract tests.""" from __future__ import annotations import uuid from types import SimpleNamespace import pytest from starlette.requests import Request from app.core.exceptions import AppException from app.domains.data_access import router as router_module from app.domains.data_access.lineage import ( DataAccessLineageEdge, DataAccessLineageNode, DataAccessTableLineage, ) from app.domains.data_access.lineage_owners import DataAccessLineageDatasetOwner from app.integrations.driver_base import ( DataAccessDatasetFieldSchema, DataAccessDatasetTablePage, DataAccessDatasetTableSchema, DataAccessDatasetTableSummary, ) def _request(path: str) -> Request: return Request({ "type": "http", "method": "GET", "path": path, "root_path": "", "scheme": "http", "server": ("testserver", 80), "client": ("127.0.0.1", 1234), "headers": [], "query_string": b"", }) @pytest.mark.asyncio async def test_person_labels_expose_name_without_email() -> None: person_id = uuid.uuid4() class _Rows: @staticmethod def all(): return [(person_id, "朱晓鹏", "xiaopeng.zhu@pingcap.com")] class _Db: async def execute(self, _statement): return _Rows() labels = await router_module._person_labels(_Db(), [person_id]) assert labels == {str(person_id): "朱晓鹏"} @pytest.mark.asyncio async def test_table_list_revalidates_catalog_and_filters_provider_rows(monkeypatch) -> None: resource_id = uuid.uuid4() owner_id = uuid.uuid4() calls: list[tuple[str, ...]] = [] resource = SimpleNamespace( project_id="analytics-prod", dataset_id="warehouse", owner_person_id=owner_id, ) async def _allow(_db, _current_user) -> None: calls.append(("authorize",)) async def _resource(_db, selected_resource_id): calls.append(("resource", str(selected_resource_id))) return resource class _Driver: async def list_dataset_tables(self, project_id: str, dataset_id: str): calls.append(("list", project_id, dataset_id)) return ( DataAccessDatasetTableSummary("account_daily", "TABLE"), DataAccessDatasetTableSummary("monthly_summary", "VIEW"), ) async def _driver(_db): return _Driver() async def _labels(_db, person_ids): assert person_ids == [owner_id] return {str(owner_id): "Dataset Owner"} class _Scalars: @staticmethod def all(): return [] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search="daily", page=1, page_size=100, scope="all", cursor=None, current_user=SimpleNamespace(id=owner_id), db=_Db(), ) assert response["data"] == [{ "table_id": "account_daily", "table_type": "TABLE", "resource_id": None, "owner_person_id": str(owner_id), "owner_label": "Dataset Owner", "owner_source": "dataset_inherited", "owner_inherited": True, "caller_can_manage_owner": True, "metadata_source": "provider", "catalog_tier": "discovered", "lineage_tracked": False, "downstream_reference_count": 0, "lineage_direction": None, "selectable": True, "schema_available": True, }] assert response["meta"]["total"] == 1 assert calls == [ ("authorize",), ("resource", str(resource_id)), ("list", "analytics-prod", "warehouse"), ] @pytest.mark.asyncio async def test_table_list_falls_back_to_dataset_owner_when_table_owner_is_empty( monkeypatch, ) -> None: resource_id = uuid.uuid4() table_resource_id = uuid.uuid4() dataset_owner_id = uuid.uuid4() resource = SimpleNamespace( project_id="analytics-prod", dataset_id="warehouse", owner_person_id=dataset_owner_id, ) table_resource = SimpleNamespace( id=table_resource_id, table_id="account_daily", owner_person_id=None, owner_source="manual", ) async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _selected_resource_id): return resource class _Driver: async def list_dataset_tables(self, _project_id: str, _dataset_id: str): return (DataAccessDatasetTableSummary("account_daily", "TABLE"),) async def _driver(_db): return _Driver() async def _labels(_db, person_ids): assert person_ids == [dataset_owner_id] return {str(dataset_owner_id): "Dataset Owner"} class _Scalars: @staticmethod def all(): return [table_resource] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search=None, page=1, page_size=100, scope="all", cursor=None, current_user=SimpleNamespace(id=dataset_owner_id), db=_Db(), ) assert response["data"] == [{ "table_id": "account_daily", "table_type": "TABLE", "resource_id": str(table_resource_id), "owner_person_id": str(dataset_owner_id), "owner_label": "Dataset Owner", "owner_source": "dataset_inherited", "owner_inherited": True, "caller_can_manage_owner": True, "metadata_source": "provider", "catalog_tier": "managed", "lineage_tracked": False, "downstream_reference_count": 0, "lineage_direction": None, "selectable": True, "schema_available": True, }] @pytest.mark.asyncio async def test_table_list_allows_only_the_effective_table_owner_to_transfer( monkeypatch, ) -> None: resource_id = uuid.uuid4() table_resource_id = uuid.uuid4() dataset_owner_id = uuid.uuid4() table_owner_id = uuid.uuid4() resource = SimpleNamespace( project_id="analytics-prod", dataset_id="warehouse", owner_person_id=dataset_owner_id, ) table_resource = SimpleNamespace( id=table_resource_id, table_id="account_daily", owner_person_id=table_owner_id, owner_source="manual", ) async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _selected_resource_id): return resource class _Driver: async def list_dataset_tables(self, _project_id: str, _dataset_id: str): return (DataAccessDatasetTableSummary("account_daily", "TABLE"),) async def _driver(_db): return _Driver() async def _labels(_db, person_ids): assert person_ids == [table_owner_id] return {str(table_owner_id): "Table Owner"} class _Scalars: @staticmethod def all(): return [table_resource] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search=None, page=1, page_size=100, scope="all", cursor=None, current_user=SimpleNamespace(id=table_owner_id), db=_Db(), ) assert response["data"][0]["owner_person_id"] == str(table_owner_id) assert response["data"][0]["caller_can_manage_owner"] is True @pytest.mark.asyncio async def test_table_owner_transfer_response_removes_previous_owner_permission( monkeypatch, ) -> None: dataset_resource_id = uuid.uuid4() table_resource_id = uuid.uuid4() previous_owner_id = uuid.uuid4() next_owner_id = uuid.uuid4() dataset = SimpleNamespace( project_id="analytics-prod", dataset_id="warehouse", ) async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _resource_id): return dataset class _Driver: async def get_dataset_table_schema(self, project_id, dataset_id, table_id): return DataAccessDatasetTableSchema( project_id=project_id, dataset_id=dataset_id, table_id=table_id, table_type="TABLE", description=None, fields=(), ) async def _driver(_db): return _Driver() async def _resolve(self, _db, **_kwargs): return None async def _transfer(self, _db, **_kwargs): return SimpleNamespace( resource_id=table_resource_id, table_id="account_daily", table_type="TABLE", owner_person_id=str(next_owner_id), owner_source="manual", ) async def _labels(_db, person_ids): assert person_ids == [next_owner_id] return {str(next_owner_id): "Next Owner"} class _Db: async def commit(self): return None monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) monkeypatch.setattr( router_module.DataAccessService, "resolve_requestable_bigquery_table_resource", _resolve, ) monkeypatch.setattr( router_module.DataAccessService, "reassign_bigquery_table_owner", _transfer, ) response = await router_module.update_requestable_dataset_table_owner( request=_request( f"/api/v1/data-access/requestable-resources/{dataset_resource_id}" "/tables/account_daily/owner", ), body=router_module.DataAccessOwnerTransferRequest( owner_person_id=next_owner_id, ), resource_id=dataset_resource_id, table_id="account_daily", current_user=SimpleNamespace(id=previous_owner_id), db=_Db(), ) assert response["data"]["owner_person_id"] == str(next_owner_id) assert response["data"]["caller_can_manage_owner"] is False @pytest.mark.asyncio async def test_table_list_falls_back_to_read_only_lineage_catalog_without_integration( monkeypatch, ) -> None: resource_id = uuid.uuid4() owner_id = uuid.uuid4() resource = SimpleNamespace( project_id="gcp-digital-bi", dataset_id="dwh_finance", owner_person_id=owner_id, ) async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _resource_id): return resource async def _missing_driver(_db): raise AppException( code="INTEGRATION_NOT_CONFIGURED", message="No primary data_access integration is configured.", status_code=503, ) async def _labels(_db, person_ids): assert person_ids == [owner_id] return {str(owner_id): "Xiaopeng Zhu - xiaopeng.zhu@pingcap.com"} class _Scalars: @staticmethod def all(): return [] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _missing_driver) monkeypatch.setattr(router_module, "_person_labels", _labels) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search="gcp_monthly_billing_with_support_cost_alloc", page=1, page_size=100, scope="all", cursor=None, current_user=SimpleNamespace(id=owner_id), db=_Db(), ) assert response["data"] == [{ "table_id": "gcp_monthly_billing_with_support_cost_alloc", "table_type": "TABLE", "resource_id": None, "owner_person_id": str(owner_id), "owner_label": "Xiaopeng Zhu - xiaopeng.zhu@pingcap.com", "owner_source": "dataset_inherited", "owner_inherited": True, "caller_can_manage_owner": False, "metadata_source": "lineage_catalog", "catalog_tier": "lineage", "lineage_tracked": True, "downstream_reference_count": 0, "lineage_direction": "upstream", "selectable": False, "schema_available": False, }] assert response["meta"]["total"] == 1 @pytest.mark.asyncio async def test_tracked_table_scope_uses_lineage_catalog_without_provider_enumeration( monkeypatch, ) -> None: resource_id = uuid.uuid4() owner_id = uuid.uuid4() resource = SimpleNamespace( project_id="gcp-digital-bi", dataset_id="nexuscrm_final", owner_person_id=owner_id, ) calls: list[object] = [] async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _resource_id): return resource class _Driver: async def list_dataset_tables(self, *_args, **_kwargs): raise AssertionError("tracked scope must not enumerate provider tables") async def _driver(_db): calls.append("driver-built") return _Driver() async def _labels(_db, person_ids): assert person_ids == [owner_id] return {str(owner_id): "Dataset Owner"} class _Scalars: @staticmethod def all(): return [] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) monkeypatch.setattr( router_module, "list_lineage_catalog_tables", lambda **_kwargs: ( router_module.DataAccessLineageCatalogTable( "nexuscrm_account", "TABLE", downstream_reference_count=2, lineage_direction="downstream", ), ), ) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search=None, page=1, page_size=50, scope="tracked", cursor=None, current_user=SimpleNamespace(id=owner_id), db=_Db(), ) assert response["data"] == [{ "table_id": "nexuscrm_account", "table_type": "TABLE", "resource_id": None, "owner_person_id": str(owner_id), "owner_label": "Dataset Owner", "owner_source": "dataset_inherited", "owner_inherited": True, "caller_can_manage_owner": True, "metadata_source": "lineage_catalog", "catalog_tier": "lineage", "lineage_tracked": True, "downstream_reference_count": 2, "lineage_direction": "downstream", "selectable": True, "schema_available": True, }] assert response["meta"] == { "trace_id": None, "timestamp": response["meta"]["timestamp"], "page": 1, "page_size": 50, "total": 1, "scope": "tracked", "next_cursor": None, "has_more": False, } assert calls == ["driver-built"] @pytest.mark.asyncio async def test_other_table_scope_uses_one_provider_page_and_excludes_tracked( monkeypatch, ) -> None: resource_id = uuid.uuid4() owner_id = uuid.uuid4() resource = SimpleNamespace( project_id="gcp-digital-bi", dataset_id="nexuscrm_final", owner_person_id=owner_id, ) calls: list[object] = [] async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _resource_id): return resource class _Driver: async def list_dataset_tables_page( self, project_id, dataset_id, *, page_size, page_token, ): calls.append((project_id, dataset_id, page_size, page_token)) return DataAccessDatasetTablePage( items=( DataAccessDatasetTableSummary("nexuscrm_account", "TABLE"), DataAccessDatasetTableSummary("nexuscrm_contact", "TABLE"), ), next_page_token="provider-next-page", total_rows=174, ) async def _driver(_db): return _Driver() async def _labels(_db, person_ids): assert person_ids == [owner_id] return {str(owner_id): "Dataset Owner"} class _Scalars: @staticmethod def all(): return [] class _Db: async def scalars(self, _statement): return _Scalars() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) monkeypatch.setattr(router_module, "_person_labels", _labels) monkeypatch.setattr( router_module, "list_lineage_catalog_tables", lambda **_kwargs: ( router_module.DataAccessLineageCatalogTable("nexuscrm_account", "TABLE"), ), ) response = await router_module.list_requestable_dataset_tables( request=_request(f"/api/v1/data-access/requestable-resources/{resource_id}/tables"), resource_id=resource_id, search=None, page=1, page_size=20, scope="other", cursor="provider-current-page", current_user=SimpleNamespace(id=owner_id), db=_Db(), ) assert [row["table_id"] for row in response["data"]] == ["nexuscrm_contact"] assert response["data"][0]["catalog_tier"] == "discovered" assert response["data"][0]["lineage_tracked"] is False assert response["meta"]["total"] == 173 assert response["meta"]["next_cursor"] == "provider-next-page" assert response["meta"]["has_more"] is True assert calls == [ ("gcp-digital-bi", "nexuscrm_final", 20, "provider-current-page"), ] @pytest.mark.asyncio async def test_table_schema_serializes_nested_field_paths(monkeypatch) -> None: resource_id = uuid.uuid4() resource = SimpleNamespace(project_id="analytics-prod", dataset_id="warehouse") async def _allow(_db, _current_user) -> None: return None async def _resource(_db, _resource_id): return resource class _Driver: async def get_dataset_table_schema( self, project_id: str, dataset_id: str, table_id: str, ) -> DataAccessDatasetTableSchema: assert (project_id, dataset_id, table_id) == ( "analytics-prod", "warehouse", "account_daily", ) return DataAccessDatasetTableSchema( project_id=project_id, dataset_id=dataset_id, table_id=table_id, table_type="TABLE", description="Daily accounts", fields=( DataAccessDatasetFieldSchema( name="department", field_path="owner.department", field_type="STRING", mode="NULLABLE", description="Owning department", depth=1, ordinal_position=1, ), ), ) async def _driver(_db): return _Driver() monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "_dataset_schema_driver", _driver) response = await router_module.get_requestable_dataset_table_schema( request=_request( f"/api/v1/data-access/requestable-resources/{resource_id}" "/tables/account_daily/schema", ), resource_id=resource_id, table_id="account_daily", current_user=SimpleNamespace(), db=SimpleNamespace(), ) assert response["data"]["fields"] == [{ "name": "department", "field_path": "owner.department", "field_type": "STRING", "mode": "NULLABLE", "description": "Owning department", "depth": 1, "ordinal_position": 1, }] @pytest.mark.asyncio async def test_table_lineage_reads_the_deployed_catalog_without_a_provider_call(monkeypatch) -> None: resource_id = uuid.uuid4() owner_id = uuid.uuid4() source_resource_id = uuid.uuid4() resource = SimpleNamespace( project_id="gcp-digital-bi", dataset_id="dws_orders", integration_id="dax-fixture", ) calls: list[str] = [] async def _allow(_db, _current_user) -> None: calls.append("authorize") async def _resource(_db, _resource_id): calls.append("resource") return resource def _lineage(**kwargs): calls.append(f"lineage:{kwargs['table_id']}") return DataAccessTableLineage( status="available", direction="upstream", catalog_id="gcp-digital-bi-dbt", source_version="dbt-main-fixture", generated_at="2026-08-20T00:00:00Z", project_id=resource.project_id, dataset_id=resource.dataset_id, table_id="orders_dashboard", nodes=( DataAccessLineageNode( node_id="source.erp.orders", project_id="gcp-digital-bi", # A source declaration may share a physical dataset with # downstream dbt models. Owner metadata is still source-only. dataset_id=resource.dataset_id, table_id="orders", node_type="source", materialization=None, description=None, level=0, is_root=True, is_target=False, ), DataAccessLineageNode( node_id="model.orders_dashboard", project_id=resource.project_id, dataset_id=resource.dataset_id, table_id="orders_dashboard", node_type="model", materialization="view", description=None, level=1, is_root=False, is_target=True, ), ), edges=( DataAccessLineageEdge( upstream_node_id="source.erp.orders", downstream_node_id="model.orders_dashboard", relationship="source", ), ), truncated=False, ) async def _owners(_db, *, integration_id, nodes): calls.append("owners") assert integration_id == "dax-fixture" source_nodes = list(nodes) assert [node.node_type for node in source_nodes] == ["source"] return { ("gcp-digital-bi", resource.dataset_id): DataAccessLineageDatasetOwner( dataset_resource_id=source_resource_id, owner_person_id=str(owner_id), owner_label="朱晓鹏 - xiaopeng.zhu@pingcap.com", owner_source="catalog", requestable=False, ), } monkeypatch.setattr(router_module, "_require_data_access_request_author", _allow) monkeypatch.setattr(router_module, "_requestable_dataset_for_schema_browser", _resource) monkeypatch.setattr(router_module, "get_table_lineage", _lineage) monkeypatch.setattr(router_module, "resolve_lineage_dataset_owners", _owners) response = await router_module.get_requestable_dataset_table_lineage( request=_request( f"/api/v1/data-access/requestable-resources/{resource_id}" "/tables/orders_dashboard/lineage", ), resource_id=resource_id, table_id="orders_dashboard", current_user=SimpleNamespace(), db=SimpleNamespace(), ) assert calls == ["authorize", "resource", "lineage:orders_dashboard", "owners"] assert response["data"]["direction"] == "upstream" assert response["data"]["nodes"][0]["table_id"] == "orders" assert response["data"]["nodes"][0]["dataset_cataloged"] is True assert response["data"]["nodes"][0]["dataset_requestable"] is False assert response["data"]["nodes"][0]["dataset_owner_label"].startswith("朱晓鹏") assert response["data"]["nodes"][1]["layer"] == "serving" assert response["data"]["nodes"][1]["dataset_cataloged"] is False assert response["data"]["nodes"][1]["dataset_owner_label"] is None @pytest.mark.asyncio async def test_non_requestable_resource_is_hidden_as_not_found() -> None: class _Db: async def scalar(self, _statement): return None with pytest.raises(AppException) as exc_info: await router_module._requestable_dataset_for_schema_browser( _Db(), uuid.uuid4(), ) assert exc_info.value.code == "DATA_ACCESS_RESOURCE_NOT_REQUESTABLE" assert exc_info.value.status_code == 404