from abc import ABC, abstractmethod from dataclasses import dataclass from datetime import datetime from typing import Any, Optional from utils.tidb_topn_decoder import decode_tidb_topn_value_safe, is_tidb_encoded_value from .inverse_transform_sampler import HistogramBucket, TopNItem # Core parsing utilities (shared logic) def _handle_tidb_encoded_bytes( value: bytes, expected_types: tuple = (int, float, str) ) -> Any: """Handle TiDB-encoded binary data with type validation Args: value: TiDB-encoded bytes value expected_types: Tuple of expected Python types after decoding Returns: Decoded value if it matches expected types Raises: ValueError: If decoded value doesn't match expected types """ decoded_value = decode_tidb_topn_value_safe(value) if decoded_value is not None and isinstance(decoded_value, expected_types): return decoded_value else: raise ValueError( f"TiDB-encoded value type mismatch. Expected {expected_types}, got {type(decoded_value)}: {decoded_value}" ) def _parse_numeric_value(value: Any) -> float: """Parse numeric value to float (shared by buckets and TopN) Handles numeric types (int, float, str, bytes as UTF-8). This function assumes the input is already decoded (not TiDB-encoded binary). Args: value: Decoded value (int, float, str, bytes as UTF-8) Returns: float: Parsed numeric value Raises: ValueError: If value cannot be parsed as numeric """ if isinstance(value, (int, float)): return float(value) elif isinstance(value, str): try: return float(value) except ValueError: raise ValueError(f"Invalid numeric value: {value}") elif isinstance(value, bytes): # For bucket bounds: bytes should be simple UTF-8 text try: return float(value.decode("utf-8")) except (ValueError, UnicodeDecodeError): raise ValueError(f"Invalid numeric value: {value}") else: raise ValueError( f"Invalid numeric value type: {type(value)}. Expected int, float, str, or bytes." ) def _parse_datetime_value(value: Any, data_type: str = "DATETIME") -> float: """Parse datetime value to scalar (TiDB-compatible) Handles datetime types (datetime, str, bytes as UTF-8, int, float). This function assumes the input is already decoded (not TiDB-encoded binary). For TiDB compatibility, we properly interpret large numeric values as TiDB packed time format using the official TiDB packed time decoding algorithm. Args: value: Decoded datetime value (datetime, str, bytes as UTF-8, int, float) data_type: Type of datetime column for baseline selection Returns: float: Scalar value compatible with TiDB time encoding Raises: ValueError: If value cannot be parsed as datetime """ if isinstance(value, datetime): dt = value elif isinstance(value, str): dt = _parse_datetime_string(value) elif isinstance(value, bytes): # For bucket bounds: bytes should be simple UTF-8 text try: dt_str = value.decode("utf-8") dt = _parse_datetime_string(dt_str) except UnicodeDecodeError: raise ValueError(f"Invalid datetime value: {value}") elif isinstance(value, (int, float)): # Handle numeric values - check if they look like TiDB packed time numeric_value = ( int(value) if isinstance(value, float) and value.is_integer() else value ) # Check if this is a TiDB packed time format (very large values > 1e15) if isinstance(numeric_value, int) and abs(numeric_value) > 1e15: from utils.tidb_time_decoder import from_packed_uint, is_tidb_packed_time if is_tidb_packed_time(numeric_value): # Decode TiDB packed time format try: dt = from_packed_uint(numeric_value) if dt is None: raise ValueError( f"TiDB packed time decoding returned None for value {numeric_value}" ) except Exception as e: raise ValueError( f"Failed to decode TiDB packed time {numeric_value}: {e}" ) from e else: raise ValueError( f"Numeric datetime value {numeric_value} is unreasonably large (> 1e15) " f"and doesn't appear to be valid TiDB packed time format." ) else: # For reasonable numeric values, try to interpret as Unix timestamp try: dt = datetime.fromtimestamp(float(numeric_value)) except (ValueError, OSError, OverflowError): # If timestamp conversion fails, treat as scalar value (duration offset) return float(numeric_value) else: raise ValueError(f"Invalid datetime value: {value}") # Convert datetime to TiDB-compatible scalar using TiDB time encoding from utils.tidb_time_decoder import tidb_time_to_scalar return tidb_time_to_scalar(dt, data_type) def _parse_datetime_string(dt_str: str) -> datetime: """Parse datetime string to datetime object Args: dt_str: Datetime string in various formats Returns: datetime: Parsed datetime object Raises: ValueError: If string cannot be parsed as datetime """ dt_str = dt_str.strip() # Common datetime formats formats = [ "%Y-%m-%d %H:%M:%S.%f", # 2024-03-19 10:22:42.123456 "%Y-%m-%d %H:%M:%S", # 2024-03-19 10:22:42 "%Y-%m-%d", # 2024-03-19 "%Y/%m/%d %H:%M:%S", # 2024/03/19 10:22:42 "%Y/%m/%d", # 2024/03/19 ] for fmt in formats: try: return datetime.strptime(dt_str, fmt) except ValueError: continue raise ValueError(f"Invalid datetime string: {dt_str}") def _parse_string_value(value: Any) -> float: """Parse string value to scalar float for ordering/sampling (shared by buckets and TopN) Handles string-like types (str, bytes) and numeric values. This function assumes the input is already decoded (not TiDB-encoded binary). Args: value: Decoded value (str, bytes as UTF-8, int, float) Returns: float: Scalar value for ordering (compatible with TextSampler) """ # if isinstance(value, (int, float)): # # Value is already a scalar (common for TextSampler bounds) # return float(value) if isinstance(value, str): # Use TiDB-compatible string-to-scalar conversion from utils.bytes_scalar_converter import convert_bytes_to_scalar return convert_bytes_to_scalar(value.encode("utf-8")) elif isinstance(value, bytes): # For bucket bounds: bytes should be UTF-8 text # Use TiDB-compatible bytes-to-scalar conversion from utils.bytes_scalar_converter import convert_bytes_to_scalar return convert_bytes_to_scalar(value) # elif value is None: # return 0.0 # Allow None for empty bounds # else: # return float(value) # Try to convert to float as fallback else: raise ValueError( f"Invalid string value type: {type(value)}. Expected str or bytes." ) @dataclass class BaseTiDBBucket(ABC): """Base class for TiDB-specific histogram buckets""" lower_bound: Any = None # Raw value from database (usually text) upper_bound: Any = None # Raw value from database (usually text) count: int = 0 repeats: int = 0 ndv: int = 0 bucket_id: Optional[int] = None # Original bucket ID from stats_buckets table column_name: Optional[str] = None # Column name for identification @abstractmethod def to_histogram_bucket(self) -> HistogramBucket: """Convert to pure mathematical HistogramBucket for sampling""" @abstractmethod def parse_bound_value(self, value: Any) -> float: """Parse a raw bound value to scalar float""" @dataclass class TiDBNumericBucket(BaseTiDBBucket): """TiDB bucket for numeric types (INT, BIGINT, FLOAT, DOUBLE, DECIMAL) Type-safe bucket that only handles numeric data types. Raises ValueError for string, datetime, or other non-numeric types. """ data_type: str = "INT" # INT, BIGINT, FLOAT, DOUBLE, DECIMAL precision: Optional[int] = None # For DECIMAL type scale: Optional[int] = None # For DECIMAL type is_unsigned: bool = False # For INT types def parse_bound_value(self, value: Any) -> float: """Parse numeric bound value to float""" return _parse_numeric_value(value) def to_histogram_bucket(self) -> HistogramBucket: """Convert to mathematical HistogramBucket""" lower = self.parse_bound_value(self.lower_bound) upper = self.parse_bound_value(self.upper_bound) def _normalize_raw(value: Any) -> Optional[str]: if value is None: return None if isinstance(value, bytes): try: return value.decode("utf-8") except UnicodeDecodeError: return value.hex() return str(value) return HistogramBucket( lower_bound=lower, upper_bound=upper, count=self.count, repeats=self.repeats, ndv=self.ndv, bucket_id=self.bucket_id, column_name=self.column_name, data_type=self.data_type, raw_lower_bound=_normalize_raw(self.lower_bound), raw_upper_bound=_normalize_raw(self.upper_bound), ) @dataclass class TiDBTimeBucket(BaseTiDBBucket): """TiDB bucket for time types (DATETIME, TIMESTAMP, DATE)""" data_type: str = "DATETIME" # DATETIME, TIMESTAMP, DATE fsp: int = 0 # Fractional seconds precision def parse_bound_value(self, value: Any) -> float: """Parse datetime bound value to duration (seconds) from baseline""" return _parse_datetime_value(value, self.data_type) def to_histogram_bucket(self) -> HistogramBucket: """Convert to mathematical HistogramBucket with duration values""" lower = self.parse_bound_value(self.lower_bound) upper = self.parse_bound_value(self.upper_bound) return HistogramBucket( lower_bound=lower, upper_bound=upper, count=self.count, repeats=self.repeats, ndv=self.ndv, bucket_id=self.bucket_id, column_name=self.column_name, ) @dataclass class TiDBStringBucket(BaseTiDBBucket): """TiDB bucket for string types (CHAR, VARCHAR, TEXT, etc.) Type-safe bucket that only handles string-like data types. Raises ValueError for numeric, datetime, or other non-string types. Note: This is a simplified bucket class that delegates actual string processing to TextSampler. The complex logic like common prefix optimization and bytes-to-scalar conversion is handled by TextSampler and utils functions. """ data_type: str = "VARCHAR" charset: str = "utf8mb4" collation: str = "utf8mb4_bin" character_maximum_length: Optional[int] = None # Legacy fields for backward compatibility with existing TiDBHistogramBucket usage common_prefix_length: int = 0 common_prefix_bytes: bytes = b"" def parse_bound_value(self, value: Any) -> float: """Parse string bound value to scalar float Uses TiDB-compatible string-to-scalar conversion that matches what TextSampler expects for proper distribution matching. """ return _parse_string_value(value) def to_histogram_bucket(self) -> HistogramBucket: """Convert to mathematical HistogramBucket with scalar values Note: The actual scalar conversion is handled by TextSampler during bucket normalization using proper TiDB-compatible algorithms. """ lower = self.parse_bound_value(self.lower_bound) upper = self.parse_bound_value(self.upper_bound) return HistogramBucket( lower_bound=lower, upper_bound=upper, count=self.count, repeats=self.repeats, ndv=self.ndv, bucket_id=self.bucket_id, column_name=self.column_name, ) # Factory function to create appropriate bucket type def create_tidb_bucket(data_type: str, **kwargs) -> BaseTiDBBucket: """Factory function to create appropriate TiDB bucket based on data type""" data_type = data_type.upper() # Numeric types if data_type in [ "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT", "FLOAT", "DOUBLE", "DECIMAL", "NUMERIC", ]: return TiDBNumericBucket(data_type=data_type, **kwargs) # Time types elif data_type in ["DATETIME", "TIMESTAMP", "DATE"]: return TiDBTimeBucket(data_type=data_type, **kwargs) # TIME type (MySQL TIME is time-of-day, treat as time type) elif data_type in ["TIME"]: return TiDBTimeBucket(data_type=data_type, **kwargs) # String types elif data_type in [ "CHAR", "VARCHAR", "TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT", "BINARY", "VARBINARY", "BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", ]: return TiDBStringBucket(data_type=data_type, **kwargs) # Default to string bucket for unknown types else: return TiDBStringBucket(data_type=data_type, **kwargs) # TopN Types (following TiDB's TopNMeta structure) @dataclass class BaseTiDBTopNItem(ABC): """Base class for TiDB-specific TopN items""" encoded: Any = ( None # Raw encoded value from database (like TiDB's TopNMeta.Encoded) ) count: int = 0 # Occurrence count (like TiDB's TopNMeta.Count) @abstractmethod def to_topn_item(self) -> "TopNItem": """Convert to mathematical TopNItem for sampling""" @abstractmethod def parse_encoded_value(self, value: Any) -> float: """Parse encoded value to scalar float for sampling""" @dataclass class TiDBNumericTopNItem(BaseTiDBTopNItem): """TiDB TopN item for numeric types Type-safe TopN item that only handles numeric data types. Raises ValueError for string, datetime, or other non-numeric types. """ data_type: str = "INT" def parse_encoded_value(self, value: Any) -> float: """Parse numeric encoded value to float Handles TiDB-encoded binary data for TopN values. """ # Check if this is TiDB-encoded binary data first if isinstance(value, bytes) and is_tidb_encoded_value(value): decoded_value = _handle_tidb_encoded_bytes(value, (int, float, str)) # Use the shared numeric parsing logic on decoded value return _parse_numeric_value(decoded_value) else: # Use the shared numeric parsing logic directly return _parse_numeric_value(value) def to_topn_item(self) -> "TopNItem": """Convert to mathematical TopNItem""" from .inverse_transform_sampler import TopNItem scalar_value = self.parse_encoded_value(self.encoded) return TopNItem(value=scalar_value, count=self.count) @dataclass class TiDBTimeTopNItem(BaseTiDBTopNItem): """TiDB TopN item for time types""" data_type: str = "DATETIME" def parse_encoded_value(self, value: Any) -> float: """Parse time encoded value to TiDB-compatible scalar Handles TiDB-encoded binary data for TopN values using proper TiDB packed time decoding. """ # Check if this is TiDB-encoded binary data first if isinstance(value, bytes) and is_tidb_encoded_value(value): decoded_value = _handle_tidb_encoded_bytes(value, (int, float, str)) # Use the shared datetime parsing logic which now handles TiDB packed time properly return _parse_datetime_value(decoded_value, self.data_type) else: # Use shared datetime parsing logic directly return _parse_datetime_value(value, self.data_type) def to_topn_item(self) -> "TopNItem": """Convert to mathematical TopNItem""" from .inverse_transform_sampler import TopNItem scalar_value = self.parse_encoded_value(self.encoded) return TopNItem(value=scalar_value, count=self.count) @dataclass class TiDBStringTopNItem(BaseTiDBTopNItem): """TiDB TopN item for string types Type-safe TopN item that only handles string-like data types. Raises ValueError for numeric, datetime, or other non-string types. """ data_type: str = "VARCHAR" def parse_encoded_value(self, value: Any) -> float: """Parse string encoded value to scalar float For TopN items, handles TiDB-encoded binary data first, then converts to scalar. """ # Check if this is TiDB-encoded binary data first if isinstance(value, bytes) and is_tidb_encoded_value(value): decoded_value = _handle_tidb_encoded_bytes(value, (str, bytes)) # Use the shared string parsing logic on decoded value return _parse_string_value(decoded_value) else: # Use the shared string parsing logic directly return _parse_string_value(value) def to_topn_item(self) -> "TopNItem": """Convert to mathematical TopNItem""" from .inverse_transform_sampler import TopNItem scalar_value = self.parse_encoded_value(self.encoded) return TopNItem(value=scalar_value, count=self.count) # Factory function for TopN items def create_tidb_topn_item(data_type: str, **kwargs) -> BaseTiDBTopNItem: """Factory function to create appropriate TiDB TopN item based on data type""" data_type = data_type.upper() # Numeric types if data_type in [ "TINYINT", "SMALLINT", "MEDIUMINT", "INT", "INTEGER", "BIGINT", "FLOAT", "DOUBLE", "DECIMAL", "NUMERIC", ]: return TiDBNumericTopNItem(data_type=data_type, **kwargs) # Time types elif data_type in ["DATETIME", "TIMESTAMP", "DATE"]: return TiDBTimeTopNItem(data_type=data_type, **kwargs) # TIME type (MySQL TIME is time-of-day, treat as time type) elif data_type in ["TIME"]: return TiDBTimeTopNItem(data_type=data_type, **kwargs) # String types elif data_type in [ "CHAR", "VARCHAR", "TEXT", "TINYTEXT", "MEDIUMTEXT", "LONGTEXT", "BINARY", "VARBINARY", "BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", ]: return TiDBStringTopNItem(data_type=data_type, **kwargs) # Default to string TopN for unknown types else: return TiDBStringTopNItem(data_type=data_type, **kwargs)