from __future__ import annotations from conftest import table_definition, write_incremental_schema, write_text from increment.paths import SchemaPath from increment.schema import compute_table_definition_checksum, load_table_definition from increment.schema_index import load_latest_storage_schema_index, load_storage_schema_index from metadata import TableName from storage.object_store import LocalObjectStore class NoListLocalObjectStore(LocalObjectStore): def list_keys(self, prefix: str) -> list[str]: raise AssertionError("schema index should stream keys with iter_keys()") def test_incremental_v856_schema_checksum_matches_storage_sink_filename() -> None: raw = { "Table": "fruit1", "Schema": "test", "Version": 1, "TableVersion": 466763842401075205, "Query": "", "Type": 0, "TableColumns": [ { "ColumnName": "id", "ColumnType": "BIGINT", "ColumnPrecision": "20", "ColumnNullable": "false", "ColumnIsPk": "true", }, { "ColumnName": "name", "ColumnType": "VARCHAR", "ColumnDefault": "", "ColumnPrecision": "20", "ColumnNullable": "false", }, ], "TableColumnsTotal": 2, } schema_path = SchemaPath( key="increment/test/fruit1/meta/schema_466763842401075205_3548947668.json", source_schema="test", source_table="fruit1", table_version=466763842401075205, checksum=3548947668, ) assert compute_table_definition_checksum(raw) == 3548947668 table_def = load_table_definition(raw, schema_path) assert table_def.name == TableName("test", "fruit1") assert table_def.quoted == "`test`.`fruit1`" def test_storage_schema_index_streams_keys_without_list_keys(workspace) -> None: write_incremental_schema(workspace, table_definition(version=120)) schema_index = load_storage_schema_index(NoListLocalObjectStore(str(workspace)), "increment") assert schema_index.get(TableName("test", "worker"), 120) is not None def test_latest_storage_schema_index_reads_only_latest_table_schema(workspace) -> None: write_text(workspace / "increment/test/worker/meta/schema_100_0.json", "{not-json") write_incremental_schema(workspace, table_definition(version=120)) schema_index = load_latest_storage_schema_index(LocalObjectStore(str(workspace)), "increment") assert set(schema_index.table_defs) == {(TableName("test", "worker"), 120)} def test_storage_schema_index_uses_structured_table_identity(workspace) -> None: write_incremental_schema(workspace, table_definition(schema="a.b", table="c", version=120)) write_incremental_schema(workspace, table_definition(schema="a", table="b.c", version=130)) schema_index = load_storage_schema_index(LocalObjectStore(str(workspace)), "increment") assert set(schema_index.table_defs) == { (TableName("a.b", "c"), 120), (TableName("a", "b.c"), 130), }