// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include namespace DB { using CFModifyFlag = RecordKVFormat::CFModifyFlag; template const TiKVKey & RegionCFDataBase::getTiKVKey(const Value & val) { return *std::get<0>(val); } template const std::shared_ptr & getTiKVValuePtr(const Value & val) { return std::get<1>(val); } template const TiKVValue & RegionCFDataBase::getTiKVValue(const Value & val) { return *getTiKVValuePtr(val); } template RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value, DupCheck mode) { const auto & raw_key = RecordKVFormat::decodeTiKVKey(key); auto kv_pair = Trait::genKVPair(std::move(key), raw_key, std::move(value)); if (!kv_pair) return 0; return insert(std::move(*kv_pair), mode); } template <> RegionDataRes RegionCFDataBase::insert(TiKVKey && key, TiKVValue && value, DupCheck mode) { UNUSED(mode); RegionDataRes added_size = calcTiKVKeyValueSize(key, value); Pair kv_pair = RegionLockCFDataTrait::genKVPair(std::move(key), std::move(value)); const auto & decoded = std::get<2>(kv_pair.second); bool is_large_txn = decoded->isLargeTxn(); { auto iter = data.find(kv_pair.first); if (iter != data.end()) { added_size -= calcTiKVKeyValueSize(iter->second); data.erase(iter); } else { if unlikely (is_large_txn) { GET_METRIC(tiflash_raft_process_keys, type_large_txn_lock_put).Increment(1); } } } // according to the process of pessimistic lock, just overwrite. data.emplace(std::move(kv_pair.first), std::move(kv_pair.second)); return added_size; } template RegionDataRes RegionCFDataBase::insert(std::pair && kv_pair, DupCheck mode) { auto & map = data; TiKVValue prev_value; if (mode == DupCheck::AllowSame) { prev_value = TiKVValue::copyFrom(getTiKVValue(kv_pair.second)); } auto [it, ok] = map.emplace(std::move(kv_pair)); // We support duplicated kv pairs if they are the same in snapshot. // This is because kvs in raftstore v2's snapshot may be overlapped. // However, we still not permit duplicated kvs from raft cmd. if (!ok) { if (mode == DupCheck::Deny) { throw Exception( "Found existing key in hex: " + getTiKVKey(it->second).toDebugString(), ErrorCodes::LOGICAL_ERROR); } else if (mode == DupCheck::AllowSame) { if (prev_value != getTiKVValue(it->second)) { throw Exception( "Found existing key in hex and val differs: " + getTiKVKey(it->second).toDebugString() + " prev_val: " + getTiKVValue(it->second).toDebugString() + " new_val: " + prev_value.toDebugString(), ErrorCodes::LOGICAL_ERROR); } // duplicated key is ignored return 0; } else { throw Exception( "Found existing key in hex: " + getTiKVKey(it->second).toDebugString(), ErrorCodes::LOGICAL_ERROR); } } return calcTiKVKeyValueSize(it->second); } template size_t RegionCFDataBase::calcTiKVKeyValueSize(const Value & value) { return calcTiKVKeyValueSize(getTiKVKey(value), getTiKVValue(value)); } template size_t RegionCFDataBase::calcTiKVKeyValueSize(const TiKVKey & key, const TiKVValue & value) { if constexpr (std::is_same::value) { // We start to count size of Lock Cf. return key.dataSize() + value.dataSize(); } else { return key.dataSize() + value.dataSize(); } } template bool RegionCFDataBase::shouldIgnoreRemove(const RegionCFDataBase::Value &) { return false; } template <> bool RegionCFDataBase::shouldIgnoreRemove(const RegionCFDataBase::Value & value) { return RegionWriteCFDataTrait::getWriteType(value) == CFModifyFlag::DelFlag; } template size_t RegionCFDataBase::remove(const Key & key, bool quiet) { auto & map = data; if (auto it = map.find(key); it != map.end()) { const Value & value = it->second; if (shouldIgnoreRemove(value)) return 0; size_t size = calcTiKVKeyValueSize(value); if constexpr (std::is_same::value) { if unlikely (std::get<2>(value)->isLargeTxn()) { GET_METRIC(tiflash_raft_process_keys, type_large_txn_lock_del).Increment(1); } } map.erase(it); return size; } else if (!quiet) throw Exception("Key not found", ErrorCodes::LOGICAL_ERROR); return 0; } template bool RegionCFDataBase::cmp(const Map & a, const Map & b) { if (a.size() != b.size()) return false; for (const auto & [key, value] : a) { if (auto it = b.find(key); it != b.end()) { if (getTiKVKey(value) != getTiKVKey(it->second) || getTiKVValue(value) != getTiKVValue(it->second)) return false; } else return false; } return true; } template bool RegionCFDataBase::operator==(const RegionCFDataBase & cf) const { return cmp(cf.data, data); } template size_t RegionCFDataBase::getSize() const { return data.size(); } template RegionCFDataBase::RegionCFDataBase(RegionCFDataBase && region) : data(std::move(region.data)) {} template RegionCFDataBase & RegionCFDataBase::operator=(RegionCFDataBase && region) { data = std::move(region.data); return *this; } template std::string getTraitName() { if constexpr (std::is_same_v) { return "write"; } else if constexpr (std::is_same_v) { return "default"; } else if constexpr (std::is_same_v) { return "lock"; } else { return "unknown"; } } template size_t RegionCFDataBase::mergeFrom(const RegionCFDataBase & ori_region_data) { size_t size_changed = 0; const auto & ori_map = ori_region_data.data; auto & tar_map = data; size_t ori_key_count = ori_region_data.getSize(); for (auto it = ori_map.begin(); it != ori_map.end(); it++) { size_changed += calcTiKVKeyValueSize(it->second); auto ok = tar_map.emplace(*it).second; if (!ok) throw Exception( ErrorCodes::LOGICAL_ERROR, "{}: got duplicate key {}, cf={}, ori_key_count={}", __PRETTY_FUNCTION__, getTiKVKey(it->second).toDebugString(), getTraitName(), ori_key_count); } return size_changed; } template size_t RegionCFDataBase::splitInto(const RegionRange & range, RegionCFDataBase & new_region_data) { const auto & [start_key, end_key] = range; size_t size_changed = 0; { auto & ori_map = data; auto & tar_map = new_region_data.data; for (auto it = ori_map.begin(); it != ori_map.end();) { const auto & key = getTiKVKey(it->second); if (start_key.compare(key) <= 0 && end_key.compare(key) > 0) { size_changed += calcTiKVKeyValueSize(it->second); tar_map.insert(std::move(*it)); it = ori_map.erase(it); } else ++it; } } return size_changed; } template size_t RegionCFDataBase::serialize(WriteBuffer & buf) const { size_t total_size = 0; size_t size = getSize(); total_size += writeBinary2(size, buf); for (const auto & ele : data) { const auto & key = getTiKVKey(ele.second); const auto & value = getTiKVValue(ele.second); total_size += key.serialize(buf); total_size += value.serialize(buf); } return total_size; } template size_t RegionCFDataBase::deserialize(ReadBuffer & buf, RegionCFDataBase & new_region_data) { auto size = readBinary2(buf); size_t cf_data_size = 0; for (size_t i = 0; i < size; ++i) { auto key = TiKVKey::deserialize(buf); auto value = TiKVValue::deserialize(buf); cf_data_size += new_region_data.insert(std::move(key), std::move(value)); } return cf_data_size; } template const typename RegionCFDataBase::Data & RegionCFDataBase::getData() const { return data; } template typename RegionCFDataBase::Data & RegionCFDataBase::getDataMut() { return data; } template struct RegionCFDataBase; template struct RegionCFDataBase; template struct RegionCFDataBase; } // namespace DB