from __future__ import annotations import base64 import json from collections.abc import Callable from typing import Any from errors import ConnectorError TEXT_BASE_TYPES = {"CHAR", "VARCHAR", "TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT", "SET"} INTEGER_BASE_TYPES = {"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT", "YEAR"} FLOAT_BASE_TYPES = {"FLOAT", "DOUBLE", "REAL"} DOUBLE_BASE_TYPES = {"DOUBLE", "REAL"} DECIMAL_BASE_TYPES = {"DECIMAL", "NUMERIC", "NEWDECIMAL"} BINARY_BASE_TYPES = {"BINARY", "VARBINARY", "TINYBLOB", "BLOB", "MEDIUMBLOB", "LONGBLOB"} type ValueCoercer = Callable[[str | None], Any] def base_source_type(source_type: str) -> str: raw = source_type.strip().upper() before_params = raw.split("(", 1)[0] parts = before_params.split(maxsplit=1) if not parts: return raw return parts[0] def is_unsigned_source_type(source_type: str) -> bool: return "UNSIGNED" in source_type.strip().upper().split() def _tinyint_maps_to_boolean(source_type: str, precision: int | None) -> bool: return base_source_type(source_type) == "TINYINT" and precision == 1 def _bit_width(precision: int | None) -> int: return 1 if precision is None else precision def _bit_maps_to_boolean(source_type: str, precision: int | None) -> bool: return base_source_type(source_type) == "BIT" and _bit_width(precision) == 1 def is_source_type_syncable(source_type: str, *, precision: int | None = None) -> bool: return base_source_type(source_type) != "BIT" or _bit_maps_to_boolean(source_type, precision) def _is_zero_datetime(raw: str) -> bool: return raw.startswith("0000-00-00") and raw[11:19] == "00:00:00" def _timestamp_default(primary_key: bool) -> str | None: return "1970-01-01T00:00:00Z" if primary_key else None def coerce_timestamp(raw: str, *, primary_key: bool) -> str | None: if _is_zero_datetime(raw): return _timestamp_default(primary_key) return f"{raw[:10]}T{raw[11:]}Z" def _coerce_bit_boolean(raw: str) -> bool: return raw == "1" def _coerce_tinyint_boolean(raw: str) -> bool: return raw == "1" def coerce_binary(raw: str, *, binary_encoding: str) -> bytes: if binary_encoding == "hex": return bytes.fromhex(raw) if binary_encoding == "base64": return base64.b64decode(raw) if binary_encoding == "utf8": return raw.encode("utf-8") raise ConnectorError(f"unsupported binary encoding method: {binary_encoding!r}") def map_source_type_to_fivetran(source_type: str, *, precision: int | None = None) -> str | None: base = base_source_type(source_type) if base in {"BOOL", "BOOLEAN"}: return "BOOLEAN" if base == "TINYINT": return "BOOLEAN" if _tinyint_maps_to_boolean(source_type, precision) else "INT" if base == "SMALLINT": return "INT" if is_unsigned_source_type(source_type) else "SHORT" if base in {"MEDIUMINT", "INT", "INTEGER"}: return "LONG" if is_unsigned_source_type(source_type) else "INT" if base == "BIGINT": return "DECIMAL" if is_unsigned_source_type(source_type) else "LONG" if base in DECIMAL_BASE_TYPES: return "DECIMAL" if base == "FLOAT": return "FLOAT" if base in DOUBLE_BASE_TYPES: return "DOUBLE" if base == "DATE": return "NAIVE_DATE" if base == "DATETIME": return "NAIVE_DATETIME" if base == "YEAR": return "INT" if base == "TIME": return "STRING" if base == "TIMESTAMP": return "UTC_DATETIME" if base == "BIT": return "BOOLEAN" if _bit_maps_to_boolean(source_type, precision) else None if base in BINARY_BASE_TYPES: return "BINARY" if base == "JSON": return "JSON" if base == "VECTOR": return "STRING" if base == "SET": return "JSON" if base == "XML": return "XML" return "STRING" def coerce_value( raw: str | None, source_type: str, *, null_marker: str, binary_encoding: str, precision: int | None = None, primary_key: bool = False, ) -> Any: if raw is None or raw == null_marker: return None base = base_source_type(source_type) if raw == "" and base not in TEXT_BASE_TYPES: return None try: if base in {"BOOL", "BOOLEAN"}: return raw.lower() in {"1", "true", "t", "yes", "y"} if _bit_maps_to_boolean(source_type, precision): return _coerce_bit_boolean(raw) if _tinyint_maps_to_boolean(source_type, precision): return _coerce_tinyint_boolean(raw) if base == "BIGINT" and is_unsigned_source_type(source_type): return raw if base in INTEGER_BASE_TYPES: return int(raw) if base in FLOAT_BASE_TYPES: return float(raw) if base == "TIMESTAMP": return coerce_timestamp(raw, primary_key=primary_key) if base in DECIMAL_BASE_TYPES: return raw if base == "JSON": return json.loads(raw) if base == "SET": return [] if raw == "" else raw.split(",") if base == "BIT": return int(raw) if base in BINARY_BASE_TYPES: return coerce_binary(raw, binary_encoding=binary_encoding) except (ValueError, TypeError, json.JSONDecodeError) as exc: raise ConnectorError(f"failed to coerce {raw!r} as {source_type}: {type(exc).__name__}: {exc}") from exc return raw def build_value_coercer( source_type: str, *, null_marker: str, binary_encoding: str, precision: int | None = None, primary_key: bool = False, ) -> ValueCoercer: base = base_source_type(source_type) def normalize(raw: str | None) -> str | None: if raw is None or raw == null_marker: return None if raw == "" and base not in TEXT_BASE_TYPES: return None return raw def wrap(func: Callable[[str], Any]) -> ValueCoercer: def decode_value(raw: str | None) -> Any: normalized = normalize(raw) if normalized is None: return None try: return func(normalized) except (ValueError, TypeError, json.JSONDecodeError) as exc: raise ConnectorError( f"failed to coerce {normalized!r} as {source_type}: {type(exc).__name__}: {exc}" ) from exc return decode_value if base in {"BOOL", "BOOLEAN"}: return wrap(lambda raw: raw.lower() in {"1", "true", "t", "yes", "y"}) if _bit_maps_to_boolean(source_type, precision): return wrap(_coerce_bit_boolean) if _tinyint_maps_to_boolean(source_type, precision): return wrap(_coerce_tinyint_boolean) if base == "BIGINT" and is_unsigned_source_type(source_type): return wrap(lambda raw: raw) if base in INTEGER_BASE_TYPES: return wrap(int) if base in FLOAT_BASE_TYPES: return wrap(float) if base == "TIMESTAMP": return wrap(lambda raw: coerce_timestamp(raw, primary_key=primary_key)) if base in DECIMAL_BASE_TYPES: return wrap(lambda raw: raw) if base == "JSON": return wrap(json.loads) if base == "SET": return wrap(lambda raw: [] if raw == "" else raw.split(",")) if base == "BIT": return wrap(int) if base in BINARY_BASE_TYPES: return wrap(lambda raw: coerce_binary(raw, binary_encoding=binary_encoding)) return wrap(lambda raw: raw) __all__ = [ "base_source_type", "build_value_coercer", "coerce_binary", "coerce_timestamp", "coerce_value", "is_source_type_syncable", "is_unsigned_source_type", "map_source_type_to_fivetran", ]