// Copyright 2025 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #pragma once #include #include #include #include namespace DB::DM::InvertedIndex { enum class Version { Invalid = 0, V1 = 1, }; // InvertedIndex file format (V1): // | VERSION | Block 0 (compressed) | Block 1 (compressed) | ... | Block N (compressed) | Meta | Meta size | Magic flag | // Block format: // | number of values | value | row_ids size | value | row_ids size | ... | value | row_ids size | row_ids | row_ids | ... | row_ids | // Meta format: // | size of T | number of blocks | offset | size | min | max | offset | size | min | max | ... | offset | size | min | max | using RowID = UInt32; using RowIDs = std::vector; // A block is a minimal unit of IO, it will be as small as possible, but >= 64KB. static constexpr size_t BlockSize = 64 * 1024; // 64 KB // template struct BlockEntry { T value; RowIDs row_ids; }; template struct Block { Block() = default; std::vector> entries; void serialize(WriteBuffer & write_buf) const; static void deserialize(Block & block, ReadBuffer & read_buf); static void search(BitmapFilterPtr & bitmap_filter, ReadBuffer & read_buf, T key); static void searchRange(BitmapFilterPtr & bitmap_filter, ReadBuffer & read_buf, T begin, T end); }; template struct MetaEntry { UInt32 offset; // offset in the file UInt32 size; // block size T min; T max; void serialize(WriteBuffer & write_buf) const; static void deserialize(MetaEntry & entry, ReadBuffer & read_buf); }; template struct Meta { Meta() = default; std::vector> entries; void serialize(WriteBuffer & write_buf) const; static void deserialize(Meta & meta, ReadBuffer & read_buf); }; static std::string_view constexpr MagicFlag = "INVE"; static UInt32 constexpr MagicFlagLength = MagicFlag.size(); // Get the size of the block in bytes. template constexpr size_t getBlockSize(UInt32 entry_size, UInt32 row_ids_size) { return sizeof(UInt32) + entry_size * (sizeof(T) + sizeof(UInt32)) + row_ids_size * sizeof(RowID); } } // namespace DB::DM::InvertedIndex