from __future__ import annotations from config import ConnectorConfig from errors import ManifestError from increment.schema import TableDefinition from increment.schema_index import StorageSchemaIndex from metadata import Column, TableName from observability import warning from snapshot.schema import SnapshotTable, fivetran_table_name from snapshot.tables import SnapshotTables from type_mapping import is_source_type_syncable, map_source_type_to_fivetran def build_fivetran_schema( snapshot_tables: SnapshotTables, schema_index: StorageSchemaIndex, config: ConnectorConfig, ) -> list[dict[str, object]]: result: list[dict[str, object]] = [] snapshot_table_by_name = snapshot_tables.table_by_name latest_table_defs = schema_index.latest_table_definitions() table_names = sorted(set(snapshot_table_by_name) | set(latest_table_defs)) for table_name in table_names: table = snapshot_table_by_name.get(table_name) table_def = latest_table_defs.get(table_name) table_schema = _build_table_schema( table_name=table_name, snapshot_table=table, table_def=table_def, include_diagnostic_columns=config.enable_diagnostic_columns, ) if table_schema is not None: result.append(table_schema) return result def _build_table_schema( *, table_name: TableName, snapshot_table: SnapshotTable | None, table_def: TableDefinition | None, include_diagnostic_columns: bool, ) -> dict[str, object] | None: source_columns = _source_columns(snapshot_table, table_def) if source_columns is None: return None primary_key = _primary_key(table_name, snapshot_table, table_def) if not primary_key: warning( "skipping table without primary key", phase="schema", source_schema=table_name.source_schema, source_table=table_name.source_table, ) return None unsupported_primary_key_columns = _unsupported_primary_key_columns(source_columns, primary_key) if unsupported_primary_key_columns: warning( "skipping table because primary key contains unsupported column type", phase="schema", source_schema=table_name.source_schema, source_table=table_name.source_table, columns=unsupported_primary_key_columns, ) return None columns = _map_columns_to_fivetran(source_columns, table_name) if not columns: warning( "skipping table because all columns are unsupported", phase="schema", source_schema=table_name.source_schema, source_table=table_name.source_table, ) return None if include_diagnostic_columns: _add_diagnostic_columns(columns) source_schema, source_table = _source_identity(table_name, snapshot_table, table_def) return { "table": fivetran_table_name(source_schema, source_table), "primary_key": list(primary_key), "columns": columns, } def _source_columns( snapshot_table: SnapshotTable | None, table_def: TableDefinition | None, ) -> tuple[Column, ...] | None: if snapshot_table is not None: if table_def is not None and table_def.columns: return _merge_columns(snapshot_table.columns, table_def.columns) return snapshot_table.columns if table_def is not None and table_def.columns: return table_def.columns return None def _merge_columns(snapshot_columns: tuple[Column, ...], incremental_columns: tuple[Column, ...]) -> tuple[Column, ...]: columns = {column.name: column for column in snapshot_columns} for column in incremental_columns: columns[column.name] = column return tuple(columns.values()) def _add_diagnostic_columns(columns: dict[str, object]) -> None: columns.update( { "_incremental_commit_ts": "LONG", "_incremental_source_schema": "STRING", "_incremental_source_table": "STRING", } ) def _map_columns_to_fivetran(columns: tuple[Column, ...], table_name: TableName) -> dict[str, object]: result: dict[str, object] = {} for column in columns: mapped = _map_column_to_fivetran(column, table_name) if mapped is None: warning( "skipping unsupported column type", phase="schema", source_schema=table_name.source_schema, source_table=table_name.source_table, column=column.name, source_type=column.source_type, precision=column.precision, scale=column.scale, ) continue result[column.name] = mapped return result def _unsupported_primary_key_columns(columns: tuple[Column, ...], primary_key: tuple[str, ...]) -> list[str]: primary_key_set = set(primary_key) return [ column.name for column in columns if column.name in primary_key_set and not is_source_type_syncable(column.source_type, precision=column.precision) ] def _map_column_to_fivetran(column: Column, table_name: TableName) -> object | None: fivetran_type = map_source_type_to_fivetran(column.source_type, precision=column.precision) if fivetran_type is None: return None if fivetran_type != "DECIMAL": return fivetran_type if _is_bigint_unsigned(column.source_type): return _decimal_type(20, 0) if column.precision is None or column.scale is None: raise ManifestError(f"DECIMAL column {table_name.quoted}.{column.name} is missing precision or scale") return _decimal_type(column.precision, column.scale) def _is_bigint_unsigned(source_type: str) -> bool: tokens = source_type.strip().upper().split() return tokens[:1] == ["BIGINT"] and "UNSIGNED" in tokens def _decimal_type(precision: int, scale: int) -> dict[str, object]: return { "type": "DECIMAL", "precision": precision, "scale": scale, } def _primary_key( table_name: TableName, snapshot_table: SnapshotTable | None, table_def: TableDefinition | None, ) -> tuple[str, ...]: if table_def is not None and table_def.primary_key: return table_def.primary_key if snapshot_table is not None: return snapshot_table.primary_key return () def _source_identity( table_name: TableName, snapshot_table: SnapshotTable | None, table_def: TableDefinition | None, ) -> tuple[str, str]: if table_def is not None: return table_def.source_schema, table_def.source_table if snapshot_table is not None: return snapshot_table.source_schema, snapshot_table.source_table return table_name.source_schema, table_name.source_table __all__ = ["build_fivetran_schema"]