// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include namespace DB { FileChecker::FileChecker(const std::string & file_info_path_) { setPath(file_info_path_); } void FileChecker::setPath(const std::string & file_info_path_) { files_info_path = file_info_path_; Poco::Path path(files_info_path); tmp_files_info_path = path.parent().toString() + "tmp_" + path.getFileName(); } void FileChecker::update(const Poco::File & file) { initialize(); updateImpl(file); save(); } void FileChecker::update(const Files::const_iterator & begin, const Files::const_iterator & end) { initialize(); for (auto it = begin; it != end; ++it) updateImpl(*it); save(); } bool FileChecker::check() const { /** Read the files again every time you call `check` - so as not to violate the constancy. * `check` method is rarely called. */ Map local_map; load(local_map); if (local_map.empty()) return true; for (const auto & name_size : local_map) { Poco::File file(Poco::Path(files_info_path).parent().toString() + "/" + name_size.first); if (!file.exists()) { LOG_ERROR(log, "File {} doesn't exist", file.path()); return false; } size_t real_size = file.getSize(); if (real_size != name_size.second) { LOG_ERROR( log, "Size of {} is wrong. Size is {} but should be {}", file.path(), real_size, name_size.second); return false; } } return true; } void FileChecker::initialize() { if (initialized) return; load(map); initialized = true; } void FileChecker::updateImpl(const Poco::File & file) { map[Poco::Path(file.path()).getFileName()] = file.getSize(); } void FileChecker::save() const { { WriteBufferFromFile out(tmp_files_info_path); /// So complex JSON structure - for compatibility with the old format. writeCString("{\"yandex\":{", out); for (auto it = map.begin(); it != map.end(); ++it) { if (it != map.begin()) writeString(",", out); /// `escapeForFileName` is not really needed. But it is left for compatibility with the old code. writeJSONString(escapeForFileName(it->first), out); writeString(R"(:{"size":")", out); writeIntText(it->second, out); writeString("\"}", out); } writeCString("}}", out); out.next(); } Poco::File current_file(files_info_path); if (current_file.exists()) { std::string old_file_name = files_info_path + ".old"; current_file.renameTo(old_file_name); Poco::File(tmp_files_info_path).renameTo(files_info_path); Poco::File(old_file_name).remove(); } else Poco::File(tmp_files_info_path).renameTo(files_info_path); } void FileChecker::load(Map & map) const { map.clear(); if (!Poco::File(files_info_path).exists()) return; ReadBufferFromFile in(files_info_path); WriteBufferFromOwnString out; /// The JSON library does not support whitespace. We delete them. Inefficient. while (!in.eof()) { char c; readChar(c, in); if (!isspace(c)) writeChar(c, out); } JSON json(out.str()); JSON files = json["yandex"]; // Note: loop variable 'name_value' should always be a copy because the return type of iterator of type 'JSON' is not a reference. for (const auto name_value : files) map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt(); } } // namespace DB