// Copyright 2019 TiKV Project Authors. Licensed under Apache-2.0. // Codes below are from TiKV's index scan executor. However, the struct is not // public so we need to copy the code here. Reference: // https://github.com/tikv/tikv/blob/v8.5.5/components/tidb_query_executors/src/index_scan_executor.rs#L209-L246 // NOTE: Keep changes in this file minimal to reduce divergence from upstream. // TODO: Refactor upstream TiKV so TiCI can reuse its implementation to decode // index data. use itertools::izip; use tidb_query_datatype::{ FieldTypeAccessor, codec::{ Datum, batch::{LazyBatchColumn, LazyBatchColumnVec}, datum::{self, DatumDecoder}, row::v2::{RowSlice, V1CompatibleEncoder}, table, }, expr::EvalContext, }; use tikv_codec::number::NumberDecoder; use tipb::FieldType; /// Common interfaces for table scan and index scan implementations. pub trait ScanExecutorImpl: Send { /// Gets the schema. #[allow(unused)] fn schema(&self) -> &[FieldType]; /// Gets a mutable reference of the executor context. #[allow(unused)] fn mut_context(&mut self) -> &mut EvalContext; fn build_column_vec(&self, scan_rows: usize) -> LazyBatchColumnVec; /// Accepts a key value pair and fills the column vector. /// /// The column vector does not need to be regular when there are errors /// during this process. However if there is no error, the column vector /// must be regular. fn process_kv_pair( &mut self, key: &[u8], value: &[u8], columns: &mut LazyBatchColumnVec, commit_ts: Option, ) -> anyhow::Result<()>; } #[derive(PartialEq, Debug, Copy, Clone)] pub enum DecodeHandleStrategy { NoDecode, DecodeIntHandle, DecodeCommonHandle, } pub struct IndexScanExecutorImpl { /// See `TableScanExecutorImpl`'s `context`. context: EvalContext, /// See `TableScanExecutorImpl`'s `schema`. schema: Vec, /// ID of interested columns (exclude PK handle column). columns_id_without_handle: Vec, columns_id_for_common_handle: Vec, /// If true, also fill the `extra_common_handle_keys` in /// `LazyBatchColumnVec` for each row. fill_extra_common_handle_key: bool, // NOTE: TiCI-specific override for hybrid index keys; upstream TiKV uses // columns_id_without_handle.len() to skip key columns. key_columns_len: usize, // NOTE: TiCI-specific: fulltext/hybrid restore data always includes all index columns. force_restore_index_columns: bool, // NOTE: TiCI-specific: when restore data misses a column, keep key-decoded // value instead of failing hard. fallback_on_missing_restore_data: bool, /// The strategy to decode handles. /// Handle will be always placed in the last column. decode_handle_strategy: DecodeHandleStrategy, /// Number of partition ID columns, now it can only be 0 or 1. /// Must be after all normal columns and handle, but before /// physical_table_id_column pid_column_cnt: usize, /// Number of Physical Table ID columns, can only be 0 or 1. /// Must be last, after pid_column physical_table_id_column_cnt: usize, index_version: i64, } impl ScanExecutorImpl for IndexScanExecutorImpl { #[inline] fn schema(&self) -> &[FieldType] { &self.schema } #[inline] fn mut_context(&mut self) -> &mut EvalContext { &mut self.context } /// Constructs empty columns, with PK containing int handle in decoded /// format and the rest in raw format. /// /// Note: the structure of the constructed column is the same as table scan /// executor but due to different reasons. fn build_column_vec(&self, scan_rows: usize) -> LazyBatchColumnVec { let columns_len = self.schema.len(); let mut columns = Vec::with_capacity(columns_len); for _ in 0..self.columns_id_without_handle.len() { columns.push(LazyBatchColumn::raw_with_capacity(scan_rows)); } match &self.decode_handle_strategy { DecodeHandleStrategy::NoDecode => {} DecodeHandleStrategy::DecodeIntHandle => { columns.push(LazyBatchColumn::decoded_with_capacity_and_tp( scan_rows, tidb_query_datatype::EvalType::Int, )); } DecodeHandleStrategy::DecodeCommonHandle => { for _ in self.columns_id_without_handle.len() ..columns_len - self.pid_column_cnt - self.physical_table_id_column_cnt { columns.push(LazyBatchColumn::raw_with_capacity(scan_rows)); } } } if self.pid_column_cnt > 0 { columns.push(LazyBatchColumn::decoded_with_capacity_and_tp( scan_rows, tidb_query_datatype::EvalType::Int, )); } if self.physical_table_id_column_cnt > 0 { columns.push(LazyBatchColumn::decoded_with_capacity_and_tp( scan_rows, tidb_query_datatype::EvalType::Int, )); } assert_eq!(columns.len(), columns_len); LazyBatchColumnVec::from(columns) } // Value layout: (see https://docs.google.com/document/d/1Co5iMiaxitv3okJmLYLJxZYCNChcjzswJMRr-_45Eqg/edit?usp=sharing) // ```text // +-- IndexValueVersion0 (with restore data, or common handle, or index is global) // | // | Layout: TailLen | Options | Padding | [IntHandle] | [UntouchedFlag] // | Length: 1 | len(options) | len(padding) | 8 | 1 // | // | TailLen: len(padding) + len(IntHandle) + len(UntouchedFlag) // | Options: Encode some value for new features, such as common handle, new collations or global index. // | See below for more information. // | Padding: Ensure length of value always >= 10. (or >= 11 if UntouchedFlag exists.) // | IntHandle: Only exists when table use int handles and index is unique. // | UntouchedFlag: Only exists when index is untouched. // | // +-- Old Encoding (without restore data, integer handle, local) // | // | Layout: [Handle] | [UntouchedFlag] // | Length: 8 | 1 // | // | Handle: Only exists in unique index. // | UntouchedFlag: Only exists when index is untouched. // | // | If neither Handle nor UntouchedFlag exists, value will be one single byte '0' (i.e. []byte{'0'}). // | Length of value <= 9, use to distinguish from the new encoding. // | // +-- IndexValueForClusteredIndexVersion1 // | // | Layout: TailLen | VersionFlag | Version ｜ Options | [UntouchedFlag] // | Length: 1 | 1 | 1 | len(options) | 1 // | // | TailLen: len(UntouchedFlag) // | Options: Encode some value for new features, such as common handle, new collations or global index. // | See below for more information. // | UntouchedFlag: Only exists when index is untouched. // | // | Layout of Options: // | // | Segment: Common Handle | Global Index | New Collation // | Layout: CHandle Flag | CHandle Len | CHandle | PidFlag | PartitionID | restoreData // | Length: 1 | 2 | len(CHandle) | 1 | 8 | len(restoreData) // | // | Common Handle Segment: Exists when unique index used common handles. // | Global Index Segment: Exists when index is global. // | New Collation Segment: Exists when new collation is used and index or handle contains non-binary string. // | In v4.0, restored data contains all the index values. For example, (a int, b char(10)) and index (a, b). // | The restored data contains both the values of a and b. // | In v5.0, restored data contains only non-binary data(except for char and _bin). In the above example, the restored data contains only the value of b. // | Besides, if the collation of b is _bin, then restored data is an integer indicate the spaces are truncated. Then we use sortKey // | and the restored data together to restore original data. // ``` #[inline] fn process_kv_pair( &mut self, key: &[u8], value: &[u8], columns: &mut LazyBatchColumnVec, _commit_ts: Option, ) -> anyhow::Result<()> { table::check_index_key(key)?; let key_payload = &key[table::PREFIX_LEN + table::ID_LEN..]; if self.index_version == -1 { self.index_version = Self::get_index_version(value)? } if value.len() > table::MAX_OLD_ENCODED_VALUE_LEN { self.process_kv_general(key, key_payload, value, columns) } else { self.process_old_collation_kv(key, key_payload, value, columns) } } } #[derive(PartialEq, Debug, Copy, Clone)] enum DecodeHandleOp<'a> { Nop, IntFromKey(&'a [u8]), IntFromValue(&'a [u8]), CommonHandle(&'a [u8]), } #[derive(PartialEq, Debug, Copy, Clone)] enum RestoreData<'a> { NotExists, V4(&'a [u8]), V5(&'a [u8]), } #[derive(PartialEq, Debug, Copy, Clone)] enum DecodePartitionIdOp<'a> { Nop, Pid(&'a [u8]), } #[derive(Clone, Copy, Debug, Default)] pub struct ImportDecodeFlags { /// Number of columns encoded in the index key. TiCI-specific for hybrid /// index keys (sharding columns only). pub key_columns_len: usize, /// Decode index columns from restore data. pub force_restore_index_columns: bool, /// If restore data misses a column, preserve key-decoded value instead of /// returning error. pub fallback_on_missing_restore_data: bool, } impl IndexScanExecutorImpl { /// Build an index scan executor for TiCi import decoding. /// /// - `schema`: output columns schema (index columns plus optional handle /// columns). /// - `columns_id_without_handle`: index column IDs in decode order. /// - `columns_id_for_common_handle`: PK column IDs for common-handle /// decoding. /// - `decode_handle_strategy`: how to decode handle columns, if any. /// - `decode_flags`: import-only decoding flags. /// /// NOTE: This method is added for TiCi import to support hybrid index /// decode, which diverges from upstream TiKV's index scan executor /// behavior. pub fn new_for_import( context: EvalContext, schema: Vec, columns_id_without_handle: Vec, columns_id_for_common_handle: Vec, decode_handle_strategy: DecodeHandleStrategy, decode_flags: ImportDecodeFlags, ) -> Self { let fill_extra_common_handle_key = matches!( decode_handle_strategy, DecodeHandleStrategy::DecodeCommonHandle ); Self { context, schema, columns_id_without_handle, columns_id_for_common_handle, fill_extra_common_handle_key, key_columns_len: decode_flags.key_columns_len, decode_handle_strategy, force_restore_index_columns: decode_flags.force_restore_index_columns, fallback_on_missing_restore_data: decode_flags.fallback_on_missing_restore_data, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, } } #[inline] fn decode_int_handle_from_value(&self, mut value: &[u8]) -> anyhow::Result { // NOTE: it is not `number::decode_i64`. value .read_u64() .map_err(|_| anyhow::anyhow!("Failed to decode handle in value as i64")) .map(|x| x as i64) } /// Decode int handle from index key. /// /// # Key Format /// /// ## Normal format (local index or old global index): /// ```text /// [handle_flag][handle_data] /// ``` /// /// ## V1 Global Index (non-unique, non-clustered): /// ```text /// [INDEX_VALUE_PARTITION_ID_FLAG][partition_id (8 bytes)][handle_flag][handle_data] /// ``` /// /// Where `handle_flag` is `datum::INT_FLAG` or `datum::UINT_FLAG`. /// /// # Note for V1 /// In V1, partition ID appears in both KEY and VALUE: /// - KEY: Contains partition ID prefix (this function skips it) /// - VALUE: Still contains partition ID (extracted by `split_partition_id`) /// /// Future V2 will remove partition ID from VALUE. #[inline] fn decode_int_handle_from_key(&self, key: &[u8]) -> anyhow::Result { let mut val = key; if val.is_empty() { return Err(anyhow::anyhow!("Key is empty, cannot decode handle")); } let first_flag = val[0]; val = &val[1..]; // Support partition handle prefix: PARTITION_ID_FLAG + partition_id(8 bytes) + // inner_handle let handle_flag = if first_flag == table::INDEX_VALUE_PARTITION_ID_FLAG { // Ensure there are at least 8 bytes for partition id + 1 byte for handle flag if val.len() < table::ID_LEN + 1 { return Err(anyhow::anyhow!( "Insufficient data to decode partition ID and handle flag: \ expected at least {} bytes after first flag, got {}", table::ID_LEN + 1, val.len() )); } // Skip the partition id (8 bytes) val = &val[table::ID_LEN..]; // Read the actual handle flag let flag = val[0]; val = &val[1..]; flag } else { // first_flag is already the handle flag first_flag }; // TODO: Better to use `push_datum`. This requires us to allow `push_datum` // receiving optional time zone first. match handle_flag { datum::INT_FLAG => val .read_i64() .map_err(|_| anyhow::anyhow!("Failed to decode handle in key as i64")), datum::UINT_FLAG => val .read_u64() .map_err(|_| anyhow::anyhow!("Failed to decode handle in key as u64")) .map(|x| x as i64), f => Err(anyhow::anyhow!("Unexpected handle flag {}", f)), } } fn extract_columns_from_row_format( &mut self, value: &[u8], columns: &mut LazyBatchColumnVec, ) -> anyhow::Result<()> { let row = RowSlice::from_bytes(value)?; for (idx, col_id) in self.columns_id_without_handle.iter().enumerate() { if let Some((start, offset)) = row.search_in_non_null_ids(*col_id)? { let mut buffer_to_write = columns[idx].mut_raw().begin_concat_extend(); buffer_to_write .write_v2_as_datum(&row.values()[start..offset], &self.schema[idx])?; } else if row.search_in_null_ids(*col_id) { columns[idx].mut_raw().push(datum::DATUM_DATA_NULL); } else { return Err(anyhow::anyhow!("Unexpected missing column {}", col_id)); } } Ok(()) } fn extract_columns_from_datum_format( datum: &mut &[u8], columns: &mut [LazyBatchColumn], ) -> anyhow::Result<()> { for (i, column) in columns.iter_mut().enumerate() { if datum.is_empty() { return Err(anyhow::anyhow!("{}th column is missing value", i)); } let (value, remaining) = datum::split_datum(datum, false)?; column.mut_raw().push(value); *datum = remaining; } Ok(()) } // Process index values that are in old collation, when // `new_collations_enabled_on_first_bootstrap` = true also will access this // function. // NOTE: We should extract the index columns from the key first, // and extract the handles from value if there is no handle in the key. // Otherwise, extract the handles from the key. fn process_old_collation_kv( &mut self, key: &[u8], mut key_payload: &[u8], value: &[u8], columns: &mut LazyBatchColumnVec, ) -> anyhow::Result<()> { if self.physical_table_id_column_cnt > 0 { self.process_physical_table_id_column(key, columns)?; } Self::extract_columns_from_datum_format( &mut key_payload, &mut columns[..self.columns_id_without_handle.len()], )?; match self.decode_handle_strategy { DecodeHandleStrategy::NoDecode => {} // For normal index, it is placed at the end and any columns prior to it are // ensured to be interested. For unique index, it is placed in the value. DecodeHandleStrategy::DecodeIntHandle if key_payload.is_empty() => { // This is a unique index, and we should look up PK int handle in the value. let handle_val = self.decode_int_handle_from_value(value)?; columns[self.columns_id_without_handle.len()] .mut_decoded() .push_int(Some(handle_val)); } DecodeHandleStrategy::DecodeIntHandle => { // This is a normal index, and we should look up PK handle in the key. let handle_val = self.decode_int_handle_from_key(key_payload)?; columns[self.columns_id_without_handle.len()] .mut_decoded() .push_int(Some(handle_val)); } DecodeHandleStrategy::DecodeCommonHandle => { // Otherwise, if the handle is common handle, we extract it from the key. let end_index = columns.columns_len() - self.pid_column_cnt - self.physical_table_id_column_cnt; if self.fill_extra_common_handle_key { columns .mut_extra_common_handle_keys() .push(key_payload.to_vec()); } Self::extract_columns_from_datum_format( &mut key_payload, &mut columns[self.columns_id_without_handle.len()..end_index], )?; } } Ok(()) } // restore_original_data restores the index values whose format is introduced in // TiDB 5.0. Unlike the format in TiDB 4.0, the new format is optimized for // storage space: // - If the index is a composed index, only the non-binary string column's value // need to write to value, not all. // - If a string column's collation is _bin, then we only write the number of // the truncated spaces to value. // - If a string column is char, not varchar, then we use the sortKey directly. // // The whole logic of this function is: // - For each column pass in, check if it needs the restored data to get to // original data. If not, check the next column. // - Skip if the `sort key` is NULL, because the original data must be NULL. // - Depend on the collation if `_bin` or not. Process them differently to get // the correct original data. // - Write the original data into the column, we need to make sure pop() is // called. fn restore_original_data<'a>( &self, restored_values: &[u8], column_iter: impl Iterator, ) -> anyhow::Result<()> { let row = RowSlice::from_bytes(restored_values)?; for (field_type, column_id, column) in column_iter { if !field_type.need_restored_data() { continue; } let is_bin_collation = field_type .collation() .map(|col| col.is_bin_collation()) .unwrap_or(false); assert!(!column.is_empty()); let mut last_value = column.raw().last().unwrap(); let decoded_value = last_value.read_datum()?; if !last_value.is_empty() { return Err(anyhow::anyhow!( "Unexpected extra bytes: {}", tikv_log_wrappers::Value(last_value) )); } if decoded_value == Datum::Null { continue; } let original_data = if is_bin_collation { // _bin collation, we need to combine data from key and value to form the // original data. // Unwrap as checked by `decoded_value.read_datum() == Datum::Null` let truncate_str = decoded_value.as_string()?.unwrap(); let Some(space_num_data) = row.get(*column_id)? else { if self.fallback_on_missing_restore_data { // TiCI-specific: fulltext/hybrid import files may encode // common-handle PK values only in key while omitting them from // restore data. In this case keep the key-decoded value instead // of failing hard. // Reference: https://github.com/pingcap/tidb/blob/ae6361b03225b155615b60fdc6fb1be88fc9ae2b/pkg/tablecodec/tablecodec.go#L876-L880 continue; } return Err(anyhow::anyhow!("Unexpected missing column {}", column_id)); }; let space_num = tidb_query_datatype::codec::row::v2::decode_v2_u64(space_num_data)?; // Form the original data. truncate_str .iter() .cloned() .chain(std::iter::repeat_n( tidb_query_datatype::codec::collation::collator::PADDING_SPACE as _, space_num as _, )) .collect::>() } else { let Some(original_data) = row.get(*column_id)? else { if self.fallback_on_missing_restore_data { // TiCI-specific: if restore data misses this column, fallback to // the value decoded from key/common-handle to stay compatible with // TiDB fulltext/hybrid encoding. // Reference: https://github.com/pingcap/tidb/blob/ae6361b03225b155615b60fdc6fb1be88fc9ae2b/pkg/tablecodec/tablecodec.go#L876-L880 continue; } return Err(anyhow::anyhow!("Unexpected missing column {}", column_id)); }; original_data.to_vec() }; column.mut_raw().pop(); let mut buffer_to_write = column.mut_raw().begin_concat_extend(); buffer_to_write.write_v2_as_datum(&original_data, field_type)?; } Ok(()) } // get_index_version is the same as getIndexVersion() in the TiDB repo. fn get_index_version(value: &[u8]) -> anyhow::Result { if value.len() == 3 || value.len() == 4 { // For the unique index with null value or non-unique index, the length can be 3 // or 4 if <= 9. return Ok(1); } if value.len() <= table::MAX_OLD_ENCODED_VALUE_LEN { return Ok(0); } let tail_len = value[0] as usize; if tail_len >= value.len() { return Err(anyhow::anyhow!("`tail_len`: {} is corrupted", tail_len)); } if (tail_len == 0 || tail_len == 1) && value[1] == table::INDEX_VALUE_VERSION_FLAG { return Ok(value[2] as i64); } Ok(0) } // Process new layout index values in an extensible way, // see https://docs.google.com/document/d/1Co5iMiaxitv3okJmLYLJxZYCNChcjzswJMRr-_45Eqg/edit?usp=sharing fn process_kv_general( &mut self, key: &[u8], key_payload: &[u8], value: &[u8], columns: &mut LazyBatchColumnVec, ) -> anyhow::Result<()> { let (decode_handle, decode_pid, restore_data) = self.build_operations(key_payload, value)?; if self.fill_extra_common_handle_key { match decode_handle { DecodeHandleOp::CommonHandle(key) => { columns.mut_extra_common_handle_keys().push(key.to_vec()); } op => { return Err(anyhow::anyhow!( "invalid op {:?} for fill_extra_common_handle_key", op )); } } } if self.physical_table_id_column_cnt > 0 { match decode_pid { DecodePartitionIdOp::Nop => { self.process_physical_table_id_column(key, columns)?; } // When it's a global index, will return partition id instead of table id. DecodePartitionIdOp::Pid(_) => { self.decode_pid_columns(columns, columns.columns_len() - 1, decode_pid)?; } } } self.decode_index_columns(key_payload, columns, restore_data)?; self.decode_handle_columns(decode_handle, columns, restore_data)?; // Deprecated: Keep this for old tidb version during upgrade. // If need partition id, append partition id to the last column before physical // table id column if exists. if self.pid_column_cnt > 0 { self.decode_pid_columns( columns, columns.columns_len() - self.physical_table_id_column_cnt - 1, decode_pid, )?; } Ok(()) } #[inline] fn build_operations<'a>( &self, mut key_payload: &'a [u8], index_value: &'a [u8], ) -> anyhow::Result<(DecodeHandleOp<'a>, DecodePartitionIdOp<'a>, RestoreData<'a>)> { let tail_len = index_value[0] as usize; if tail_len >= index_value.len() { return Err(anyhow::anyhow!("`tail_len`: {} is corrupted", tail_len)); } let (remaining, tail) = if self.index_version == 1 { // Skip the version segment. index_value[3..].split_at(index_value.len() - 3 - tail_len) } else { index_value[1..].split_at(index_value.len() - 1 - tail_len) }; let (common_handle_bytes, remaining) = Self::split_common_handle(remaining)?; let (decode_handle_op, remaining) = { if !common_handle_bytes.is_empty() && self.decode_handle_strategy != DecodeHandleStrategy::DecodeCommonHandle { return Err(anyhow::anyhow!( "Expect to decode index values with common handles in `DecodeCommonHandle` mode." )); } let dispatcher = match self.decode_handle_strategy { DecodeHandleStrategy::NoDecode => DecodeHandleOp::Nop, DecodeHandleStrategy::DecodeIntHandle if tail_len < 8 => { // This is a non-unique index, we should extract the int handle from the key. // NOTE: TiCi hybrid index keys only encode sharding columns. datum::skip_n(&mut key_payload, self.key_columns_len)?; DecodeHandleOp::IntFromKey(key_payload) } DecodeHandleStrategy::DecodeIntHandle => { // This is a unique index, we should extract the int handle from the value. DecodeHandleOp::IntFromValue(tail) } DecodeHandleStrategy::DecodeCommonHandle if common_handle_bytes.is_empty() => { // This is a non-unique index, we should extract the common handle from the key. // NOTE: TiCi hybrid index keys only encode sharding columns. datum::skip_n(&mut key_payload, self.key_columns_len)?; DecodeHandleOp::CommonHandle(key_payload) } DecodeHandleStrategy::DecodeCommonHandle => { // This is a unique index, we should extract the common handle from the value. DecodeHandleOp::CommonHandle(common_handle_bytes) } }; (dispatcher, remaining) }; let (partition_id_bytes, remaining) = Self::split_partition_id(remaining)?; let decode_pid_op = { if self.pid_column_cnt > 0 && partition_id_bytes.is_empty() { return Err(anyhow::anyhow!( "Expect to decode partition id but payload is empty" )); } else if partition_id_bytes.is_empty() { DecodePartitionIdOp::Nop } else { DecodePartitionIdOp::Pid(partition_id_bytes) } }; let (restore_data, remaining) = Self::split_restore_data(remaining)?; let restore_data = { if restore_data.is_empty() { RestoreData::NotExists } else if self.index_version == 1 { RestoreData::V5(restore_data) } else { RestoreData::V4(restore_data) } }; if !remaining.is_empty() { return Err(anyhow::anyhow!( "Unexpected corrupted extra bytes: {}", tikv_log_wrappers::Value(remaining) )); } Ok((decode_handle_op, decode_pid_op, restore_data)) } #[inline] fn decode_index_columns( &mut self, mut key_payload: &[u8], columns: &mut LazyBatchColumnVec, restore_data: RestoreData<'_>, ) -> anyhow::Result<()> { match restore_data { RestoreData::NotExists => { Self::extract_columns_from_datum_format( &mut key_payload, &mut columns[..self.columns_id_without_handle.len()], )?; } // If there are some restore data, we need to process them to get the original data. RestoreData::V4(rst) => { // 4.0 version format, use the restore data directly. The restore data contain // all the indexed values. self.extract_columns_from_row_format(rst, columns)?; } RestoreData::V5(rst) => { if self.force_restore_index_columns { // TiCI-specific: Fulltext/hybrid indexes store all index columns in restore // data. Decode columns directly from restore data instead of key payload. self.extract_columns_from_row_format(rst, columns)?; } else { // Extract the data from key, then use the restore data to get the original // data. Self::extract_columns_from_datum_format( &mut key_payload, &mut columns[..self.columns_id_without_handle.len()], )?; let limit = self.columns_id_without_handle.len(); self.restore_original_data( rst, izip!( &self.schema[..limit], &self.columns_id_without_handle, &mut columns[..limit], ), )?; } } } Ok(()) } #[inline] fn decode_handle_columns( &mut self, decode_handle: DecodeHandleOp<'_>, columns: &mut LazyBatchColumnVec, restore_data: RestoreData<'_>, ) -> anyhow::Result<()> { match decode_handle { DecodeHandleOp::Nop => {} DecodeHandleOp::IntFromKey(handle) => { let handle = self.decode_int_handle_from_key(handle)?; columns[self.columns_id_without_handle.len()] .mut_decoded() .push_int(Some(handle)); } DecodeHandleOp::IntFromValue(handle) => { let handle = self.decode_int_handle_from_value(handle)?; columns[self.columns_id_without_handle.len()] .mut_decoded() .push_int(Some(handle)); } DecodeHandleOp::CommonHandle(mut handle) => { let end_index = columns.columns_len() - self.pid_column_cnt - self.physical_table_id_column_cnt; Self::extract_columns_from_datum_format( &mut handle, &mut columns[self.columns_id_without_handle.len()..end_index], )?; } } let restore_data_bytes = match restore_data { RestoreData::V5(value) => value, _ => return Ok(()), }; if let DecodeHandleOp::CommonHandle(_) = decode_handle { let skip = self.columns_id_without_handle.len(); let end_index = columns.columns_len() - self.pid_column_cnt - self.physical_table_id_column_cnt; self.restore_original_data( restore_data_bytes, izip!( &self.schema[skip..end_index], &self.columns_id_for_common_handle, &mut columns[skip..end_index], ), )?; } Ok(()) } #[inline] fn process_physical_table_id_column( &mut self, key: &[u8], columns: &mut LazyBatchColumnVec, ) -> anyhow::Result<()> { let table_id = table::decode_table_id(key)?; let col_index = columns.columns_len() - 1; columns[col_index].mut_decoded().push_int(Some(table_id)); Ok(()) } #[inline] fn decode_pid_columns( &mut self, columns: &mut LazyBatchColumnVec, idx: usize, decode_pid: DecodePartitionIdOp<'_>, ) -> anyhow::Result<()> { match decode_pid { DecodePartitionIdOp::Nop => {} DecodePartitionIdOp::Pid(pid) => { let pid = tici_common::codec::number::NumberCodec::decode_i64(pid); columns[idx].mut_decoded().push_int(Some(pid)) } } Ok(()) } #[inline] fn split_common_handle(value: &[u8]) -> anyhow::Result<(&[u8], &[u8])> { if value .first() .is_some_and(|c| *c == table::INDEX_VALUE_COMMON_HANDLE_FLAG) { let handle_len = (&value[1..]).read_u16().map_err(|_| { anyhow::anyhow!( "Fail to read common handle's length from value: {}", tikv_log_wrappers::Value::value(value) ) })? as usize; let handle_end_offset = 3 + handle_len; if handle_end_offset > value.len() { return Err(anyhow::anyhow!("`handle_len` is corrupted: {}", handle_len)); } Ok(value[3..].split_at(handle_len)) } else { Ok(value.split_at(0)) } } #[inline] fn split_partition_id(value: &[u8]) -> anyhow::Result<(&[u8], &[u8])> { if value .first() .is_some_and(|c| *c == table::INDEX_VALUE_PARTITION_ID_FLAG) { if value.len() < 9 { return Err(anyhow::anyhow!( "Remaining len {} is too short to decode partition ID", value.len() )); } Ok(value[1..].split_at(8)) } else { Ok(value.split_at(0)) } } #[inline] fn split_restore_data(value: &[u8]) -> anyhow::Result<(&[u8], &[u8])> { Ok( if value .first() .is_some_and(|c| *c == table::INDEX_VALUE_RESTORED_DATA_FLAG) { (value, &value[value.len()..]) } else { (&value[..0], value) }, ) } } #[cfg(test)] mod tests { use tidb_query_datatype::{Collation, FieldTypeTp, codec::Datum}; use super::*; // These following tests in TiKV rely on `BatchIndexScanExecutor` which is not // ported to this file. So skip porting the tests in TiCI // fn test_basic() // fn test_unique_common_handle_index() // fn test_old_collation_non_unique_common_handle_index() // fn test_new_collation_unique_int_handle_index() // fn test_new_collation_non_unique_int_handle_index() // fn test_new_collation_non_unique_common_handle_index() // fn test_unique_common_handle_global_index() #[test] fn test_int_handle_char_index() { use tidb_query_datatype::builder::FieldTypeBuilder; // Schema: create table t(a int, b char(10) collate utf8mb4_bin, c char(10) // collate utf8mb4_unicode_ci, key i_a(a), key i_b(b), key i_c(c), key i_abc(a, // b, c), unique key i_ua(a), unique key i_ub(b), unique key i_uc(c), // unique key i_uabc(a,b,c)); insert into t values (1, "a ", "A "); // i_a and i_ua let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![FieldTypeTp::Long.into(), FieldTypeTp::LongLong.into()], columns_id_without_handle: vec![1], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x30], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_b and i_ub idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![2], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x30], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, ], &[0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_c and i_uc idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![3], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x0, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x1, 0x0, 0x41], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x8, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_abc and i_uabc idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![1, 2, 3], columns_id_for_common_handle: vec![], key_columns_len: 3, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x0, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x2, 0x0, 0x3, 0x0, 0x1, 0x61, 0x41, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!(columns[3].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x8, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x2, 0x0, 0x3, 0x0, 0x1, 0x61, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!(columns[3].decoded().to_int_vec().last().unwrap(), &Some(1)); } #[test] fn test_int_handle_varchar_index() { use tidb_query_datatype::builder::FieldTypeBuilder; // Schema: create table t(a int, b varchar(10) collate utf8mb4_bin, c // varchar(10) collate utf8mb4_unicode_ci, key i_a(a), key i_b(b), key i_c(c), // key i_abc(a, b, c), unique key i_ua(a), unique key i_ub(b), unique // key i_uc(c), unique key i_uabc(a,b,c)); insert into t values (1, "a // ", "A "); // i_a and i_ua let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![FieldTypeTp::Long.into(), FieldTypeTp::LongLong.into()], columns_id_without_handle: vec![1], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x30], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x35, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_b and i_ub idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![2], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x0, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x2, 0x0, 0x61, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, ], &[ 0x8, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x2, 0x0, 0x61, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_c and i_uc idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![3], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x0, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x2, 0x0, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x8, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x3, 0x2, 0x0, 0x41, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!(columns[1].decoded().to_int_vec().last().unwrap(), &Some(1)); // i_abc and i_uabc idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::LongLong.into(), ], columns_id_without_handle: vec![1, 2, 3], columns_id_for_common_handle: vec![], key_columns_len: 3, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x0, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x3, 0x0, 0x5, 0x0, 0x1, 0x61, 0x20, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!(columns[3].decoded().to_int_vec().last().unwrap(), &Some(1)); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3d, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x8, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x1, 0x2, 0x3, 0x1, 0x0, 0x3, 0x0, 0x5, 0x0, 0x1, 0x61, 0x20, 0x41, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!(columns[3].decoded().to_int_vec().last().unwrap(), &Some(1)); } #[test] fn test_common_handle_index() { use tidb_query_datatype::builder::FieldTypeBuilder; // create table t(a int, b char(10) collate utf8mb4_bin, c char(10) collate // utf8mb4_unicode_ci, d varchar(10) collate utf8mb4_bin, e varchar(10) collate // utf8mb4_general_ci , primary key(a, b, c, d, e), key i_a(a), key // i_b(b), key i_c(c), key i_d(d), key i_e(e), key i_abcde(a, b, c, d, e), // unique key i_ua(a), unique key i_ub(b), unique key i_uc( c), unique // key i_ud(d), unique key i_ue(e), unique key i_uabcde(a,b,c, d, e)); // // CREATE TABLE `t` ( // `a` int(11) NOT NULL, // `b` char(10) NOT NULL, // `c` char(10) COLLATE utf8mb4_unicode_ci NOT NULL, // `d` varchar(10) NOT NULL, // `e` varchar(10) COLLATE utf8mb4_general_ci NOT NULL, // PRIMARY KEY (`a`,`b`,`c`,`d`,`e`), // KEY `i_a` (`a`), // KEY `i_b` (`b`), // KEY `i_c` (`c`), // KEY `i_d` (`d`), // KEY `i_e` (`e`), // KEY `i_abcde` (`a`,`b`,`c`,`d`,`e`), // UNIQUE KEY `i_ua` (`a`), // UNIQUE KEY `i_ub` (`b`), // UNIQUE KEY `i_uc` (`c`), // UNIQUE KEY `i_ud` (`d`), // UNIQUE KEY `i_ue` (`e`), // UNIQUE KEY `i_uabcde` (`a`,`b`,`c`,`d`,`e`) // ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin // // insert into t values (1, "a ", "A ", "a ", "A "); // i_a and i_ua let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![1], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); // i_b and i_ub idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![2], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); // i_c and i_uc idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![3], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); // i_d and i_ud idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![4], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); // i_e and i_ue idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![5], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); // i_abcde and i_uabcde idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Utf8Mb4UnicodeCi) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4Bin) .into(), FieldTypeBuilder::new() .tp(FieldTypeTp::VarChar) .collation(Collation::Utf8Mb4UnicodeCi) .into(), ], columns_id_without_handle: vec![1, 2, 3, 4, 5], columns_id_for_common_handle: vec![1, 2, 3, 4, 5], key_columns_len: 5, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[6].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[7].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[8].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[9].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x40, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, ], &[ 0x0, 0x7d, 0x1, 0x7f, 0x0, 0x31, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0xe, 0x33, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x1, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x1, 0x0, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf9, 0x80, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, 0x4, 0x5, 0x1, 0x0, 0x2, 0x0, 0x4, 0x0, 0x41, 0x1, 0x41, 0x20, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); assert_eq!( columns[5].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[6].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a".as_bytes().to_vec()) ); assert_eq!( columns[7].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A".as_bytes().to_vec()) ); assert_eq!( columns[8].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("a ".as_bytes().to_vec()) ); assert_eq!( columns[9].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("A ".as_bytes().to_vec()) ); } #[test] fn test_common_handle_with_physical_table_id() { // CREATE TABLE `tcommonhash` ( // `a` int(11) NOT NULL, // `b` int(11) DEFAULT NULL, // `c` int(11) NOT NULL, // `d` int(11) NOT NUL, // PRIMARY KEY (`a`,`c`,`d`) /*T![clustered_index] CLUSTERED */, // KEY `idx_bc` (`b`,`c`) // ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin // insert into tcommonhash values (1, 2, 3, 1); // idx_bc let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeTp::Long.into(), FieldTypeTp::Long.into(), FieldTypeTp::Long.into(), FieldTypeTp::Long.into(), // EXTRA_PHYSICAL_TABLE_ID_COL FieldTypeTp::Long.into(), ], columns_id_without_handle: vec![2, 3], columns_id_for_common_handle: vec![1, 3, 4], key_columns_len: 2, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 1, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[0x0, 0x7d, 0x1], &mut columns, None, ) .unwrap(); assert_eq!( columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(2) ); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::I64(3) ); assert_eq!( columns[2].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( columns[3].raw().last().unwrap().read_datum().unwrap(), Datum::I64(3) ); assert_eq!( columns[4].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( // physical table id columns[5].mut_decoded().to_int_vec()[0].unwrap(), 92 ); } #[test] fn test_global_index_with_physical_table_id() { // CREATE TABLE `t` ( // `a` int(11) NOT NULL, // `b` int(11) NOT NUL, // UNIQUE KEY uidx_a(`a`), // ) partition by hash(b) partitions 5; // insert into t values (1, 2); // uidx_a let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ // column `a` FieldTypeTp::Long.into(), // _tidb_rowid FieldTypeTp::Long.into(), // EXTRA_PHYSICAL_TABLE_ID_COL FieldTypeTp::Long.into(), ], columns_id_without_handle: vec![1], columns_id_for_common_handle: vec![], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeIntHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 1, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(10); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7b, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ], &[ 0x8, 0x7e, // INDEX_VALUE_PARTITION_ID_FLAG 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5c, // partition id 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // _tidb_rowid ], &mut columns, None, ) .unwrap(); assert_eq!( // column a columns[0].raw().last().unwrap().read_datum().unwrap(), Datum::I64(1) ); assert_eq!( // _tidb_rowid columns[1].mut_decoded().to_int_vec()[0].unwrap(), 1 ); assert_eq!( // partition id columns[2].mut_decoded().to_int_vec()[0].unwrap(), 92 ); } #[test] fn test_common_handle_index_latin1_bin() { use tidb_query_datatype::builder::FieldTypeBuilder; // create table t(c1 varchar(200) CHARACTER SET latin1 COLLATE latin1_bin, c2 // int, primary key(c1) clustered, key kk(c2)); idx_exec for index // kk(c2), its columns will be let mut idx_exe = IndexScanExecutorImpl { context: Default::default(), schema: vec![ FieldTypeTp::Long.into(), FieldTypeBuilder::new() .tp(FieldTypeTp::String) .collation(Collation::Latin1Bin) .into(), ], columns_id_without_handle: vec![2], columns_id_for_common_handle: vec![1], key_columns_len: 1, force_restore_index_columns: false, fallback_on_missing_restore_data: false, decode_handle_strategy: DecodeHandleStrategy::DecodeCommonHandle, pid_column_cnt: 0, physical_table_id_column_cnt: 0, index_version: -1, fill_extra_common_handle_key: false, }; let mut columns = idx_exe.build_column_vec(1); idx_exe .process_kv_pair( &[ 0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x37, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x3, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x1, 0x6a, 0x6f, 0x76, 0x69, 0x61, 0x6c, 0x65, 0x64, 0xff, 0x69, 0x73, 0x6f, 0x6e, 0x0, 0x0, 0x0, 0x0, 0xfb, ], &[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x1, 0x0, 0x0, ], &mut columns, None, ) .unwrap(); assert_eq!( columns[1].raw().last().unwrap().read_datum().unwrap(), Datum::Bytes("jovialedison".as_bytes().to_vec()) ); } #[test] fn test_index_version() { assert_eq!( IndexScanExecutorImpl::get_index_version(&[0x0, 0x7d, 0x1]).unwrap(), 1 ); assert_eq!( IndexScanExecutorImpl::get_index_version(&[0x1, 0x7d, 0x1, 0x31]).unwrap(), 1 ); assert_eq!( IndexScanExecutorImpl::get_index_version(&[ 0x0, 0x7d, 0x1, 0x80, 0x0, 0x2, 0x0, 0x0, 0x0, 0x1, 0x2, 0x1, 0x0, 0x2, 0x0, 0x61, 0x31 ]) .unwrap(), 1 ); assert_eq!( IndexScanExecutorImpl::get_index_version(&[ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x31 ]) .unwrap(), 0 ); assert_eq!( IndexScanExecutorImpl::get_index_version(&[0x30]).unwrap(), 0 ); } }