from __future__ import annotations from collections.abc import Iterable from dataclasses import dataclass from typing import Any from config import ConnectorConfig from csv_parser import new as new_csv_parser from errors import IncrementalError from metadata import Column from row_decoder import CSVRowDecoder as EventDecoder from .schema import TableDefinition @dataclass(frozen=True) class Event: operation: str source_schema: str source_table: str commit_ts: int | None data: dict[str, Any] event_offset: int @dataclass(frozen=True) class RawEvent: operation: str source_schema: str source_table: str commit_ts: int | None raw_values: list[str] event_offset: int input_offset: int def _business_columns(table_def: TableDefinition) -> tuple[Column, ...]: if not table_def.columns: raise IncrementalError(f"incremental schema has no table columns for {table_def.quoted}") return table_def.columns def iter_events( handle: Iterable[str], *, table_def: TableDefinition, config: ConnectorConfig, start_event: int = 0, ) -> Iterable[Event]: columns = _business_columns(table_def) decoders = { 3: EventDecoder.new( columns, null_marker=config.incremental.csv_null_value, binary_encoding=config.incremental.csv_binary_encoding, input_offset=3, ), 4: EventDecoder.new( columns, null_marker=config.incremental.csv_null_value, binary_encoding=config.incremental.csv_binary_encoding, input_offset=4, ), } for event in iter_raw_events(handle, table_def=table_def, start_event=start_event): data: dict[str, Any] = {} decoders[event.input_offset].decode_into(event.raw_values, data) yield Event( operation=event.operation, source_schema=event.source_schema, source_table=event.source_table, commit_ts=event.commit_ts, data=data, event_offset=event.event_offset, ) def iter_raw_events( handle: Iterable[str], *, table_def: TableDefinition, start_event: int = 0, ) -> Iterable[RawEvent]: reader = new_csv_parser( error_type=IncrementalError, context="incremental", strict=False, ).iter_rows(handle) columns = _business_columns(table_def) expected_values = len(columns) input_offset: int | None = None for event_offset, raw_values in enumerate(reader): if event_offset < start_event: continue if input_offset is None: has_commit_ts = len(raw_values) == expected_values + 4 input_offset = 4 if has_commit_ts else 3 yield RawEvent( operation=raw_values[0], source_schema=raw_values[2], source_table=raw_values[1], commit_ts=int(raw_values[3]) if input_offset == 4 else None, raw_values=raw_values, event_offset=event_offset, input_offset=input_offset, ) def add_metadata_columns(event: Event, data: dict[str, Any]) -> dict[str, Any]: data["_incremental_commit_ts"] = event.commit_ts or 0 data["_incremental_source_schema"] = event.source_schema data["_incremental_source_table"] = event.source_table return data __all__ = ["Event", "RawEvent", "add_metadata_columns", "iter_events", "iter_raw_events"]