from .bytes_scalar_converter import convert_bytes_to_scalar def scalar_to_string( target: float, lower: float, upper: float, charset: str = "utf8mb4" ) -> tuple: """ Direct conversion method based on specified charset encoding rules Args: target: target scalar value to reconstruct lower: lower bound for valid range upper: upper bound for valid range charset: character set to use ('utf8mb4', 'utf8', 'latin1', 'ascii', etc.) Returns: (reconstructed string, float value of reconstructed string) """ # Validate charset and dispatch to appropriate handler charset = charset.lower() if charset in ["utf8mb4", "utf8"]: return _scalar_to_utf8_string(target, lower, upper, charset) elif charset == "latin1": raise NotImplementedError(f"Charset '{charset}' is not implemented yet") elif charset == "ascii": raise NotImplementedError(f"Charset '{charset}' is not implemented yet") elif charset in ["gbk", "gb2312"]: raise NotImplementedError(f"Charset '{charset}' is not implemented yet") elif charset in ["binary", "varbinary"]: raise NotImplementedError(f"Charset '{charset}' is not implemented yet") else: raise ValueError( f"Unsupported charset: '{charset}'. Supported charsets: utf8mb4, utf8, latin1, ascii, gbk, gb2312, binary" ) def _scalar_to_utf8_string( target: float, lower: float, upper: float, charset: str ) -> tuple: """ Internal implementation for UTF-8 based charsets Args: target: target scalar value to reconstruct lower: lower bound for valid range upper: upper bound for valid range charset: 'utf8mb4' or 'utf8' Returns: (reconstructed string, float value of reconstructed string) """ target_int = int(target) if target_int == 0: return "", 0.0 result_chars = [] remaining = target_int byte_pos = 0 # initialized to 0, representing the current byte position (0-7) while remaining > 0 and byte_pos < 8: # calculate the weight of the current byte position weight = 256 ** (7 - byte_pos) if weight == 0: break # extract the current byte current_byte = (remaining >> (8 * (7 - byte_pos))) & 0xFF if current_byte == 0: # Global decision point: choose between 0x00 and 0x01 for this position # and terminate (since continuing won't get us closer to target) # Calculate the processed part so far processed_part = target_int - remaining # Two options: option_0_scalar = processed_part # Choose 0x00, rest are zeros option_1_scalar = processed_part + weight # Choose 0x01, rest are zeros # Choose the one closer to target diff_0 = abs(target_int - option_0_scalar) diff_1 = abs(target_int - option_1_scalar) if diff_0 <= diff_1: result_chars.append("\x00") else: result_chars.append("\x01") # Global decision made - break out of loop break # try different length of UTF-8 encoding based on charset char, bytes_consumed = _try_utf8_encoding(remaining, byte_pos, charset) if char: result_chars.append(char) # calculate the consumed value consumed_value = 0 for i in range(bytes_consumed): if byte_pos + i < 8: byte_val = (remaining >> (8 * (7 - byte_pos - i))) & 0xFF consumed_value += byte_val * (256 ** (7 - byte_pos - i)) remaining -= consumed_value byte_pos += bytes_consumed else: # this should not happen with the improved fallback strategy remaining -= current_byte * weight byte_pos += 1 if result_chars: result_string = "".join(result_chars) actual_value = convert_bytes_to_scalar(result_string.encode("utf-8")) if lower <= actual_value <= upper: return result_string, actual_value return None, None def _try_utf8_encoding(remaining: int, start_pos: int, charset: str) -> tuple: """ Try to construct a UTF-8 character from the specified position Args: remaining: remaining integer value start_pos: current byte position (0-7) charset: 'utf8mb4' or 'utf8' Returns: (character, number of bytes consumed) """ if start_pos >= 8: return None, 0 # extract the first byte first_byte = (remaining >> (8 * (7 - start_pos))) & 0xFF # 1 byte UTF-8 (ASCII): 0xxxxxxx (0-127) if first_byte <= 0x7F: try: char = chr(first_byte) return char, 1 except (ValueError, OverflowError, TypeError): pass # 2 bytes UTF-8: 110xxxxx 10xxxxxx if 0xC0 <= first_byte <= 0xDF and start_pos + 1 < 8: second_byte = (remaining >> (8 * (7 - start_pos - 1))) & 0xFF if 0x80 <= second_byte <= 0xBF: try: # construct 2-byte UTF-8 codepoint = ((first_byte & 0x1F) << 6) | (second_byte & 0x3F) if codepoint >= 0x80: # avoid overlong encoding char = chr(codepoint) return char, 2 except (ValueError, OverflowError, TypeError): pass # 3 bytes UTF-8: 1110xxxx 10xxxxxx 10xxxxxx if 0xE0 <= first_byte <= 0xEF and start_pos + 2 < 8: second_byte = (remaining >> (8 * (7 - start_pos - 1))) & 0xFF third_byte = (remaining >> (8 * (7 - start_pos - 2))) & 0xFF if 0x80 <= second_byte <= 0xBF and 0x80 <= third_byte <= 0xBF: try: codepoint = ( ((first_byte & 0x0F) << 12) | ((second_byte & 0x3F) << 6) | (third_byte & 0x3F) ) if codepoint >= 0x800 and not ( 0xD800 <= codepoint <= 0xDFFF ): # avoid overlong encoding and surrogates char = chr(codepoint) return char, 3 except (ValueError, OverflowError, TypeError): pass # 4 bytes UTF-8: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx # Note: utf8 charset in MySQL only supports up to 3 bytes, utf8mb4 supports 4 bytes if charset == "utf8mb4" and 0xF0 <= first_byte <= 0xF7 and start_pos + 3 < 8: bytes_array = [] valid = True for i in range(4): if start_pos + i >= 8: valid = False break byte_val = (remaining >> (8 * (7 - start_pos - i))) & 0xFF bytes_array.append(byte_val) if i == 0: if not (0xF0 <= byte_val <= 0xF7): valid = False break else: if not (0x80 <= byte_val <= 0xBF): valid = False break if valid: try: codepoint = ( ((bytes_array[0] & 0x07) << 18) | ((bytes_array[1] & 0x3F) << 12) | ((bytes_array[2] & 0x3F) << 6) | (bytes_array[3] & 0x3F) ) if 0x10000 <= codepoint <= 0x10FFFF: # valid Unicode range char = chr(codepoint) return char, 4 except (ValueError, OverflowError, TypeError): pass # === NEW: First-byte-based fallback strategy === # Determine expected bytes based on UTF-8 first byte patterns if 0xF0 <= first_byte <= 0xF7: # Should be 4-byte UTF-8 if charset == "utf8mb4" and start_pos + 3 < 8: return chr(0x10FFFF), 4 # Maximum 4-byte UTF-8 character # Fallback to smaller if not enough space elif start_pos + 2 < 8: return chr(0xFFFD), 3 elif start_pos + 1 < 8: return chr(0x07FF), 2 else: return chr(0x7F), 1 elif 0xE0 <= first_byte <= 0xEF: # Should be 3-byte UTF-8 (this is the key fix!) if start_pos + 2 < 8: return chr(0xFFFD), 3 # Use 3-byte fallback, not 4-byte! elif start_pos + 1 < 8: return chr(0x07FF), 2 else: return chr(0x7F), 1 elif 0xC0 <= first_byte <= 0xDF: # Should be 2-byte UTF-8 if start_pos + 1 < 8: return chr(0x07FF), 2 else: return chr(0x7F), 1 else: # Should be 1-byte (0x00-0x7F or invalid bytes 0x80-0xBF) return chr(0x7F), 1