diff --git a/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp b/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp index c0b51d8a63..4115e985d3 100644 --- a/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp +++ b/benchmark/local_infinity/fulltext/fulltext_benchmark.cpp @@ -44,7 +44,7 @@ import match_expr; import function_expr; import search_expr; import column_expr; -import virtual_storage; +import virtual_store; using namespace infinity; @@ -123,7 +123,7 @@ void BenchmarkImport(SharedPtr infinity, const String &db_name, const String &table_name, const String &import_from) { - if (!VirtualStorage::ExistsLocal(import_from)) { + if (!LocalStore::Exists(import_from)) { LOG_ERROR(fmt::format("Data file doesn't exist: {}", import_from)); return; } diff --git a/benchmark/local_infinity/infinity_benchmark.cpp b/benchmark/local_infinity/infinity_benchmark.cpp index aba397bf41..880f18f4d9 100644 --- a/benchmark/local_infinity/infinity_benchmark.cpp +++ b/benchmark/local_infinity/infinity_benchmark.cpp @@ -42,7 +42,7 @@ import knn_expr; import column_def; import statement_common; import data_type; -import virtual_storage; +import virtual_store; using namespace infinity; @@ -84,7 +84,7 @@ int main() { String path = "/var/infinity"; - VirtualStorage::CleanupDirectoryLocal(path); + LocalStore::CleanupDirectory(path); Infinity::LocalInit(path); @@ -373,7 +373,7 @@ int main() { auto r2 = infinity->CreateTable(db_name, table_name, std::move(column_defs), std::vector{}, std::move(create_tb_options)); std::string sift_base_path = std::string(test_data_path()) + "/benchmark/sift/base.fvecs"; - if (!VirtualStorage::ExistsLocal(sift_base_path)) { + if (!LocalStore::Exists(sift_base_path)) { std::cout << "File: " << sift_base_path << " doesn't exist" << std::endl; break; } diff --git a/benchmark/local_infinity/knn/knn_import_benchmark.cpp b/benchmark/local_infinity/knn/knn_import_benchmark.cpp index f33a6e8a70..e897c106c6 100644 --- a/benchmark/local_infinity/knn/knn_import_benchmark.cpp +++ b/benchmark/local_infinity/knn/knn_import_benchmark.cpp @@ -37,7 +37,7 @@ import knn_expr; import column_def; import statement_common; import data_type; -import virtual_storage; +import virtual_store; using namespace infinity; @@ -59,10 +59,10 @@ int main(int argc, char *argv[]) { data_path = std::string(argv[3]); } - if (VirtualStorage::ExistsLocal(data_path)) { + if (LocalStore::Exists(data_path)) { std::cout << "Data path: " << data_path << " is already existed." << std::endl; } else { - VirtualStorage::MakeDirectoryLocal(data_path); + LocalStore::MakeDirectory(data_path); std::cout << "Data path: " << data_path << " is created." << std::endl; } @@ -113,7 +113,7 @@ int main(int argc, char *argv[]) { // auto [ table, status2 ] = data_base->GetTable(table_name); - if (!VirtualStorage::ExistsLocal(base_path)) { + if (!LocalStore::Exists(base_path)) { std::cout << "File: " << base_path << " doesn't exist" << std::endl; break; } diff --git a/src/executor/operator/physical_export.cpp b/src/executor/operator/physical_export.cpp index 0c7be6e225..206d76bebd 100644 --- a/src/executor/operator/physical_export.cpp +++ b/src/executor/operator/physical_export.cpp @@ -44,6 +44,9 @@ import status; import buffer_manager; import default_values; import internal_types; +import virtual_store; +import local_file_handle; +import abstract_file_handle; namespace infinity { @@ -100,11 +103,11 @@ SizeT PhysicalExport::ExportToCSV(QueryContext *query_context, ExportOperatorSta SizeT select_column_count = select_columns.size(); LocalFileSystem fs; - auto [file_handler, status] = fs.OpenFile(file_path_, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kWriteLock); + auto [file_handle, status] = fs.OpenFile(file_path_, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kWriteLock); if (!status.ok()) { RecoverableError(status); } - DeferFn file_defer([&]() { fs.Close(*file_handler); }); + DeferFn file_defer([&]() { fs.Close(*file_handle); }); if (header_) { // Output CSV header @@ -135,7 +138,7 @@ SizeT PhysicalExport::ExportToCSV(QueryContext *query_context, ExportOperatorSta header += '\n'; } } - fs.Write(*file_handler, header.c_str(), header.size()); + fs.Write(*file_handle, header.c_str(), header.size()); } SizeT offset = offset_; @@ -217,16 +220,16 @@ SizeT PhysicalExport::ExportToCSV(QueryContext *query_context, ExportOperatorSta if (row_count > 0 && this->row_limit_ != 0 && (row_count % this->row_limit_) == 0) { ++file_no_; - fs.Close(*file_handler); + fs.Close(*file_handle); String new_file_path = fmt::format("{}.part{}", file_path_, file_no_); auto result = fs.OpenFile(new_file_path, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kWriteLock); if (!result.second.ok()) { RecoverableError(result.second); } - file_handler = std::move(result.first); + file_handle = std::move(result.first); } - fs.Write(*file_handler, line.c_str(), line.size()); + fs.Write(*file_handle, line.c_str(), line.size()); ++row_count; if (limit_ != 0 && row_count == limit_) { @@ -257,12 +260,11 @@ SizeT PhysicalExport::ExportToJSONL(QueryContext *query_context, ExportOperatorS SizeT select_column_count = select_columns.size(); - LocalFileSystem fs; - auto [file_handler, status] = fs.OpenFile(file_path_, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kWriteLock); + auto [file_handle, status] = LocalStore::Open(file_path_, FileAccessMode::kWrite); if (!status.ok()) { RecoverableError(status); } - DeferFn file_defer([&]() { fs.Close(*file_handler); }); + DeferFn file_defer([&]() { file_handle->Close(); }); SizeT offset = offset_; SizeT row_count{0}; @@ -347,18 +349,18 @@ SizeT PhysicalExport::ExportToJSONL(QueryContext *query_context, ExportOperatorS } if (row_count > 0 && this->row_limit_ != 0 && (row_count % this->row_limit_) == 0) { ++file_no_; - fs.Close(*file_handler); + file_handle->Close(); String new_file_path = fmt::format("{}.part{}", file_path_, file_no_); - auto result = fs.OpenFile(new_file_path, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kWriteLock); - if (!result.second.ok()) { - RecoverableError(result.second); + auto [part_file_handle, part_status] = LocalStore::Open(new_file_path, FileAccessMode::kWrite); + if (!part_status.ok()) { + RecoverableError(part_status); } - file_handler = std::move(result.first); + file_handle = std::move(part_file_handle); } // LOG_DEBUG(line_json.dump()); String to_write = line_json.dump() + "\n"; - fs.Write(*file_handler, to_write.c_str(), to_write.size()); + file_handle->Append(to_write.c_str(), to_write.size()); ++row_count; if (limit_ != 0 && row_count == limit_) { return row_count; diff --git a/src/main/cluster_manager.cpp b/src/main/cluster_manager.cpp index 7ad3ca85e8..99fede231d 100644 --- a/src/main/cluster_manager.cpp +++ b/src/main/cluster_manager.cpp @@ -230,7 +230,12 @@ void ClusterManager::CheckHeartBeat() { } void ClusterManager::CheckHeartBeatInner() { - auto hb_interval = std::chrono::milliseconds(leader_node_->heartbeat_interval_); + if (this_node_->node_role_ != NodeRole::kLeader) { + String error_message = "Invalid node role."; + UnrecoverableError(error_message); + } + // this_node_ is the leader; + auto hb_interval = std::chrono::milliseconds(this_node_->heartbeat_interval_); while (true) { std::unique_lock hb_lock(this->hb_mutex_); this->hb_cv_.wait_for(hb_lock, hb_interval, [&] { return !this->hb_running_; }); @@ -245,7 +250,7 @@ void ClusterManager::CheckHeartBeatInner() { for (auto &[node_name, node_info] : other_node_map_) { if (node_info->node_status_ == NodeStatus::kAlive) { - if (node_info->last_update_ts_ + 2 * leader_node_->heartbeat_interval_ < this_node_->last_update_ts_) { + if (node_info->last_update_ts_ + 2 * this_node_->heartbeat_interval_ < this_node_->last_update_ts_) { node_info->node_status_ = NodeStatus::kTimeout; LOG_INFO(fmt::format("Node {} is timeout", node_name)); } diff --git a/src/main/config.cpp b/src/main/config.cpp index 3abcab418c..50b7b4299e 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -25,7 +25,7 @@ import boost; import compilation_config; import default_values; import logger; -import virtual_storage; +import virtual_store; import utility; import status; import options; @@ -116,7 +116,7 @@ Status Config::ParseTimeInfo(const String &time_info, i64 &time_seconds) { Status Config::Init(const SharedPtr &config_path, DefaultConfig *default_config) { toml::table config_toml{}; - if (config_path.get() == nullptr || config_path->empty() || !VirtualStorage::ExistsLocal(std::filesystem::absolute(*config_path))) { + if (config_path.get() == nullptr || config_path->empty() || !LocalStore::Exists(std::filesystem::absolute(*config_path))) { if (config_path.get() == nullptr || config_path->empty()) { fmt::print("No config file is given, use default configs.\n"); } else { diff --git a/src/main/infinity.cpp b/src/main/infinity.cpp index e900e02e53..bf4bab8721 100644 --- a/src/main/infinity.cpp +++ b/src/main/infinity.cpp @@ -23,7 +23,7 @@ import config; import resource_manager; import task_scheduler; import storage; -import local_file_system; +import virtual_store; import third_party; import query_options; import query_result; @@ -57,6 +57,7 @@ import query_options; import extra_ddl_info; import drop_index_info; import drop_table_info; +import third_party; import infinity_exception; import third_party; @@ -68,10 +69,9 @@ u64 Infinity::GetSessionId() { return session_->session_id(); } void Infinity::Hello() { fmt::print("hello infinity\n"); } void Infinity::LocalInit(const String &path) { - LocalFileSystem fs; SharedPtr config_path = MakeShared(std::filesystem::absolute(path + "/infinity_conf.toml")); - if (fs.Exists(*config_path)) { + if (LocalStore::Exists(*config_path)) { InfinityContext::instance().Init(config_path); } else { UniquePtr default_config = MakeUnique(); diff --git a/src/main/infinity_context.cpp b/src/main/infinity_context.cpp index adca3085b4..3562cbc3ca 100644 --- a/src/main/infinity_context.cpp +++ b/src/main/infinity_context.cpp @@ -59,10 +59,8 @@ void InfinityContext::Init(const SharedPtr &config_path, bool admin_flag config_ = MakeUnique(); auto status = config_->Init(config_path, default_config); if (!status.ok()) { - fmt::print("Error: {}", *status.msg_); std::exit(static_cast(status.code())); } - Logger::Initialize(config_.get()); resource_manager_ = MakeUnique(config_->CPULimit(), 0); @@ -74,7 +72,6 @@ void InfinityContext::Init(const SharedPtr &config_path, bool admin_flag UnrecoverableError(status.message()); return; } - if (admin_flag or config_->ServerMode() == "cluster") { // Admin mode or cluster start phase return; diff --git a/src/planner/logical_planner.cpp b/src/planner/logical_planner.cpp index b143bca2f4..9ff55d303e 100644 --- a/src/planner/logical_planner.cpp +++ b/src/planner/logical_planner.cpp @@ -64,7 +64,7 @@ import logical_command; import explain_logical_plan; import explain_ast; -import local_file_system; +import virtual_store; import status; import default_values; @@ -918,10 +918,8 @@ Status LogicalPlanner::BuildExport(const CopyStatement *statement, SharedPtrfile_path_)) { + if (LocalStore::Exists(statement->file_path_)) { Status status = Status::DuplicatedFile(statement->file_path_); RecoverableError(status); } @@ -1080,10 +1078,8 @@ Status LogicalPlanner::BuildImport(const CopyStatement *statement, SharedPtrfile_path_)) { + if (!LocalStore::Exists(statement->file_path_)) { RecoverableError(Status::FileNotFound(statement->file_path_)); } diff --git a/src/storage/buffer/file_worker/bmp_index_file_worker.cpp b/src/storage/buffer/file_worker/bmp_index_file_worker.cpp index 9e43c4bb56..d12c5d29fe 100644 --- a/src/storage/buffer/file_worker/bmp_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/bmp_index_file_worker.cpp @@ -49,10 +49,10 @@ BMPIndexFileWorker::BMPIndexFileWorker(SharedPtr data_dir, LocalFileSystem fs; String index_path = GetFilePath(); - auto [file_handler, status] = fs.OpenFile(index_path, FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = fs.OpenFile(index_path, FileFlags::READ_FLAG, FileLockType::kNoLock); if (status.ok()) { // When replay by full checkpoint, the data is deleted, but catalog is recovered. Do not read file in recovery. - index_size = fs.GetFileSize(*file_handler); + index_size = fs.GetFileSize(*file_handle); } } index_size_ = index_size; @@ -104,7 +104,7 @@ bool BMPIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, c if constexpr (std::is_same_v) { UnrecoverableError("Invalid index type."); } else { - index->Save(*file_handler_); + index->Save(*file_handle_); } }, *bmp_index); @@ -125,7 +125,7 @@ void BMPIndexFileWorker::ReadFromFileImpl(SizeT file_size) { UnrecoverableError("Invalid index type."); } else { using IndexT = std::decay_t; - index = new IndexT(IndexT::Load(*file_handler_)); + index = new IndexT(IndexT::Load(*file_handle_)); } }, *bmp_index); diff --git a/src/storage/buffer/file_worker/data_file_worker.cpp b/src/storage/buffer/file_worker/data_file_worker.cpp index e75e1394b1..b89bef300c 100644 --- a/src/storage/buffer/file_worker/data_file_worker.cpp +++ b/src/storage/buffer/file_worker/data_file_worker.cpp @@ -14,6 +14,8 @@ module; +#include + module data_file_worker; import stl; @@ -72,28 +74,24 @@ bool DataFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const // - footer: checksum u64 magic_number = 0x00dd3344; - u64 nbytes = fs.Write(*file_handler_, &magic_number, sizeof(magic_number)); - if (nbytes != sizeof(magic_number)) { - Status status = Status::DataIOError(fmt::format("Write magic number which length is {}.", nbytes)); + Status status = file_handle_->Append(&magic_number, sizeof(magic_number)); + if(!status.ok()) { RecoverableError(status); } - nbytes = fs.Write(*file_handler_, const_cast(&buffer_size_), sizeof(buffer_size_)); - if (nbytes != sizeof(buffer_size_)) { - Status status = Status::DataIOError(fmt::format("Write buffer length field which length is {}.", nbytes)); + status = file_handle_->Append(const_cast(&buffer_size_), sizeof(buffer_size_)); + if(!status.ok()) { RecoverableError(status); } - nbytes = fs.Write(*file_handler_, data_, buffer_size_); - if (nbytes != buffer_size_) { - Status status = Status::DataIOError(fmt::format("Expect to write buffer with size: {}, but {} bytes is written", buffer_size_, nbytes)); + status = file_handle_->Append(data_, buffer_size_); + if(!status.ok()) { RecoverableError(status); } u64 checksum{}; - nbytes = fs.Write(*file_handler_, &checksum, sizeof(checksum)); - if (nbytes != sizeof(checksum)) { - Status status = Status::DataIOError(fmt::format("Write buffer length field which length is {}.", nbytes)); + status = file_handle_->Append(&checksum, sizeof(checksum)); + if(!status.ok()) { RecoverableError(status); } prepare_success = true; // Not run defer_fn @@ -110,22 +108,26 @@ void DataFileWorker::ReadFromFileImpl(SizeT file_size) { // file header: magic number, buffer_size u64 magic_number{0}; - u64 nbytes = fs.Read(*file_handler_, &magic_number, sizeof(magic_number)); - if (nbytes != sizeof(magic_number)) { - Status status = Status::DataIOError(fmt::format("Read magic number which length isn't {}.", nbytes)); + auto [nbytes1, status1] = file_handle_->Read(&magic_number, sizeof(magic_number)); + if(!status1.ok()) { + RecoverableError(status1); + } + if (nbytes1 != sizeof(magic_number)) { + Status status = Status::DataIOError(fmt::format("Read magic number which length isn't {}.", nbytes1)); RecoverableError(status); } if (magic_number != 0x00dd3344) { - Status status = Status::DataIOError(fmt::format("Read magic number which length isn't {}.", nbytes)); + Status status = Status::DataIOError(fmt::format("Read magic number which length isn't {}.", nbytes1)); RecoverableError(status); } u64 buffer_size_{}; - nbytes = fs.Read(*file_handler_, &buffer_size_, sizeof(buffer_size_)); - if (nbytes != sizeof(buffer_size_)) { - Status status = Status::DataIOError(fmt::format("Unmatched buffer length: {} / {}", nbytes, buffer_size_)); - RecoverableError(status); + auto [nbytes2, status2] = file_handle_->Read(&buffer_size_, sizeof(buffer_size_)); + if (nbytes2 != sizeof(buffer_size_)) { + Status status = Status::DataIOError(fmt::format("Unmatched buffer length: {} / {}", nbytes2, buffer_size_)); + RecoverableError(status2); } + if (file_size != buffer_size_ + 3 * sizeof(u64)) { Status status = Status::DataIOError(fmt::format("File size: {} isn't matched with {}.", file_size, buffer_size_ + 3 * sizeof(u64))); RecoverableError(status); @@ -133,17 +135,17 @@ void DataFileWorker::ReadFromFileImpl(SizeT file_size) { // file body data_ = static_cast(new char[buffer_size_]); - nbytes = fs.Read(*file_handler_, data_, buffer_size_); - if (nbytes != buffer_size_) { - Status status = Status::DataIOError(fmt::format("Expect to read buffer with size: {}, but {} bytes is read", buffer_size_, nbytes)); + auto [nbytes3, status3] = file_handle_->Read(data_, buffer_size_); + if (nbytes3 != buffer_size_) { + Status status = Status::DataIOError(fmt::format("Expect to read buffer with size: {}, but {} bytes is read", buffer_size_, nbytes3)); RecoverableError(status); } // file footer: checksum u64 checksum{0}; - nbytes = fs.Read(*file_handler_, &checksum, sizeof(checksum)); - if (nbytes != sizeof(checksum)) { - Status status = Status::DataIOError(fmt::format("Incorrect file checksum length: {}.", nbytes)); + auto [nbytes4, status4] = file_handle_->Read(&checksum, sizeof(checksum)); + if (nbytes4 != sizeof(checksum)) { + Status status = Status::DataIOError(fmt::format("Incorrect file checksum length: {}.", nbytes4)); RecoverableError(status); } } diff --git a/src/storage/buffer/file_worker/emvb_index_file_worker.cpp b/src/storage/buffer/file_worker/emvb_index_file_worker.cpp index 3bebf4252f..ea6e672223 100644 --- a/src/storage/buffer/file_worker/emvb_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/emvb_index_file_worker.cpp @@ -88,7 +88,7 @@ void EMVBIndexFileWorker::FreeInMemory() { bool EMVBIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const FileWorkerSaveCtx &ctx) { auto *index = static_cast(data_); - index->SaveIndexInner(*file_handler_); + index->SaveIndexInner(*file_handle_); prepare_success = true; return true; } @@ -104,7 +104,7 @@ void EMVBIndexFileWorker::ReadFromFileImpl(SizeT file_size) { const auto residual_pq_subspace_bits = index_emvb->residual_pq_subspace_bits_; auto *index = new EMVBIndex(start_segment_offset_, column_embedding_dim, residual_pq_subspace_num, residual_pq_subspace_bits); data_ = static_cast(index); - index->ReadIndexInner(*file_handler_); + index->ReadIndexInner(*file_handle_); } const EmbeddingInfo *EMVBIndexFileWorker::GetEmbeddingInfo() const { return static_cast(column_def_->type()->type_info().get()); } diff --git a/src/storage/buffer/file_worker/file_worker.cpp b/src/storage/buffer/file_worker/file_worker.cpp index af8eb6f61f..47e40b87b3 100644 --- a/src/storage/buffer/file_worker/file_worker.cpp +++ b/src/storage/buffer/file_worker/file_worker.cpp @@ -14,21 +14,24 @@ module; +#include + module file_worker; import stl; import utility; import infinity_exception; -import local_file_system; +import local_file_handle; import third_party; import file_system_type; import defer_op; import status; -import local_file_system; +import virtual_store; import persistence_manager; import infinity_context; import logger; import persist_result_handler; +import abstract_file_handle; namespace infinity { @@ -40,32 +43,28 @@ bool FileWorker::WriteToFile(bool to_spill, const FileWorkerSaveCtx &ctx) { UnrecoverableError(error_message); } - LocalFileSystem fs; - if(persistence_manager_ != nullptr && !to_spill) { String write_dir = *file_dir_; String write_path = Path(*data_dir_) / write_dir / *file_name_; String tmp_write_path = Path(*temp_dir_) / StringTransform(write_path, "/", "_"); - u8 flags = FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG; - auto [file_handler, status] = fs.OpenFile(tmp_write_path, flags, FileLockType::kWriteLock); + auto [file_handle, status] = LocalStore::Open(tmp_write_path, FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(status.message()); } - file_handler_ = std::move(file_handler); + file_handle_ = std::move(file_handle); bool prepare_success = false; - DeferFn defer_fn([&]() { - fs.Close(*file_handler_); - file_handler_ = nullptr; + file_handle_->Close(); + file_handle_ = nullptr; }); bool all_save = WriteToFileImpl(to_spill, prepare_success, ctx); if (prepare_success) { - fs.SyncFile(*file_handler_); + file_handle_->Sync(); } - fs.SyncFile(*file_handler_); + file_handle_->Sync(); PersistResultHandler handler(persistence_manager_); PersistWriteResult persist_result = persistence_manager_->Persist(write_path, tmp_write_path); @@ -76,27 +75,25 @@ bool FileWorker::WriteToFile(bool to_spill, const FileWorkerSaveCtx &ctx) { return all_save; } else { String write_dir = ChooseFileDir(to_spill); - if (!fs.Exists(write_dir)) { - fs.CreateDirectory(write_dir); + if (!LocalStore::Exists(write_dir)) { + LocalStore::MakeDirectory(write_dir); } String write_path = fmt::format("{}/{}", write_dir, *file_name_); - u8 flags = FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG; - auto [file_handler, status] = fs.OpenFile(write_path, flags, FileLockType::kWriteLock); + auto [file_handle, status] = LocalStore::Open(write_path, FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(status.message()); } - file_handler_ = std::move(file_handler); + file_handle_ = std::move(file_handle); if (to_spill) { - auto local_file_handle = static_cast(file_handler_.get()); - LOG_TRACE(fmt::format("Open spill file: {}, fd: {}", write_path, local_file_handle->fd_)); + LOG_TRACE(fmt::format("Open spill file: {}, fd: {}", write_path, file_handle_->FileDescriptor())); } bool prepare_success = false; DeferFn defer_fn([&]() { - fs.Close(*file_handler_); - file_handler_ = nullptr; + file_handle_->Close(); + file_handle_ = nullptr; }); bool all_save = WriteToFileImpl(to_spill, prepare_success, ctx); @@ -104,14 +101,13 @@ bool FileWorker::WriteToFile(bool to_spill, const FileWorkerSaveCtx &ctx) { if (to_spill) { LOG_TRACE(fmt::format("Write to spill file {} finished. success {}", write_path, prepare_success)); } - fs.SyncFile(*file_handler_); + file_handle_->Sync(); } return all_save; } } void FileWorker::ReadFromFile(bool from_spill) { - LocalFileSystem fs; bool use_object_cache = !from_spill && persistence_manager_ != nullptr; PersistResultHandler handler; if (use_object_cache) { @@ -128,21 +124,20 @@ void FileWorker::ReadFromFile(bool from_spill) { read_path = persistence_manager_->GetObjPath(obj_addr_.obj_key_); } SizeT file_size = 0; - u8 flags = FileFlags::READ_FLAG; - auto [file_handler, status] = fs.OpenFile(read_path, flags, FileLockType::kReadLock); + auto [file_handle, status] = LocalStore::Open(read_path, FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(status.message()); } if (use_object_cache) { - fs.Seek(*file_handler, obj_addr_.part_offset_); + file_handle->Seek(obj_addr_.part_offset_); file_size = obj_addr_.part_size_; } else { - file_size = fs.GetFileSize(*file_handler); + file_size = file_handle->FileSize(); } - file_handler_ = std::move(file_handler); + file_handle_ = std::move(file_handle); DeferFn defer_fn([&]() { - file_handler_->Close(); - file_handler_ = nullptr; + file_handle_->Close(); + file_handle_ = nullptr; if (use_object_cache && obj_addr_.Valid()) { String read_path = fmt::format("{}/{}", ChooseFileDir(from_spill), *file_name_); PersistWriteResult res = persistence_manager_->PutObjCache(read_path); @@ -153,23 +148,21 @@ void FileWorker::ReadFromFile(bool from_spill) { } void FileWorker::MoveFile() { - LocalFileSystem fs; - String src_path = fmt::format("{}/{}", ChooseFileDir(true), *file_name_); String dest_dir = ChooseFileDir(false); String dest_path = fmt::format("{}/{}", dest_dir, *file_name_); if (persistence_manager_ == nullptr) { - if (!fs.Exists(src_path)) { + if (!LocalStore::Exists(src_path)) { Status status = Status::FileNotFound(src_path); RecoverableError(status); } - if (!fs.Exists(dest_dir)) { - fs.CreateDirectory(dest_dir); + if (!LocalStore::Exists(dest_dir)) { + LocalStore::MakeDirectory(dest_dir); } // if (fs.Exists(dest_path)) { // UnrecoverableError(fmt::format("File {} was already been created before.", dest_path)); // } - fs.Rename(src_path, dest_path); + LocalStore::Rename(src_path, dest_path); } else { PersistResultHandler handler(persistence_manager_); PersistWriteResult persist_result = persistence_manager_->Persist(dest_path, src_path); @@ -195,20 +188,18 @@ void FileWorker::CleanupFile() const { handler.HandleWriteResult(result); return; } - LocalFileSystem fs; String path = fmt::format("{}/{}", ChooseFileDir(false), *file_name_); - if (fs.Exists(path)) { - fs.DeleteFile(path); + if (LocalStore::Exists(path)) { + LocalStore::DeleteFile(path); LOG_INFO(fmt::format("Cleaned file: {}", path)); } } void FileWorker::CleanupTempFile() const { - LocalFileSystem fs; String path = fmt::format("{}/{}", ChooseFileDir(true), *file_name_); - if (fs.Exists(path)) { - fs.DeleteFile(path); + if (LocalStore::Exists(path)) { + LocalStore::DeleteFile(path); LOG_INFO(fmt::format("Cleaned file: {}", path)); } else { String error_message = fmt::format("Cleanup: File {} not found for deletion", path); diff --git a/src/storage/buffer/file_worker/file_worker.cppm b/src/storage/buffer/file_worker/file_worker.cppm index fda47fb690..4904914f0f 100644 --- a/src/storage/buffer/file_worker/file_worker.cppm +++ b/src/storage/buffer/file_worker/file_worker.cppm @@ -19,7 +19,7 @@ module; export module file_worker; import stl; -import file_system; +import local_file_handle; import third_party; import file_worker_type; import persistence_manager; @@ -81,6 +81,6 @@ public: protected: void *data_{nullptr}; - UniquePtr file_handler_{nullptr}; + UniquePtr file_handle_{nullptr}; }; } // namespace infinity \ No newline at end of file diff --git a/src/storage/buffer/file_worker/hnsw_file_worker.cpp b/src/storage/buffer/file_worker/hnsw_file_worker.cpp index 758daf8035..36af2cf582 100644 --- a/src/storage/buffer/file_worker/hnsw_file_worker.cpp +++ b/src/storage/buffer/file_worker/hnsw_file_worker.cpp @@ -56,10 +56,10 @@ HnswFileWorker::HnswFileWorker(SharedPtr data_dir, LocalFileSystem fs; String index_path = GetFilePath(); - auto [file_handler, status] = fs.OpenFile(index_path, FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = fs.OpenFile(index_path, FileFlags::READ_FLAG, FileLockType::kNoLock); if (status.ok()) { // When replay by full checkpoint, the data is deleted, but catalog is recovered. Do not read file in recovery. - index_size = fs.GetFileSize(*file_handler); + index_size = fs.GetFileSize(*file_handle); } } index_size_ = index_size; @@ -112,7 +112,7 @@ bool HnswFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const if constexpr (std::is_same_v) { UnrecoverableError("Invalid index type."); } else { - index->Save(*file_handler_); + index->Save(*file_handle_); } }, *hnsw_index); @@ -133,7 +133,7 @@ void HnswFileWorker::ReadFromFileImpl(SizeT file_size) { UnrecoverableError("Invalid index type."); } else { using IndexT = std::decay_t; - index = IndexT::Load(*file_handler_).release(); + index = IndexT::Load(*file_handle_).release(); } }, *hnsw_index); diff --git a/src/storage/buffer/file_worker/ivf_index_file_worker.cpp b/src/storage/buffer/file_worker/ivf_index_file_worker.cpp index 150171baca..bd21464b31 100644 --- a/src/storage/buffer/file_worker/ivf_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/ivf_index_file_worker.cpp @@ -57,7 +57,7 @@ void IVFIndexFileWorker::FreeInMemory() { bool IVFIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const FileWorkerSaveCtx &ctx) { if (data_) [[likely]] { auto index = static_cast(data_); - index->SaveIndexInner(*file_handler_); + index->SaveIndexInner(*file_handle_); prepare_success = true; LOG_TRACE("Finished WriteToFileImpl(bool &prepare_success)."); } else { @@ -69,7 +69,7 @@ bool IVFIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, c void IVFIndexFileWorker::ReadFromFileImpl(SizeT file_size) { if (!data_) [[likely]] { auto index = IVFIndexInChunk::GetNewIVFIndexInChunk(index_base_.get(), column_def_.get()); - index->ReadIndexInner(*file_handler_); + index->ReadIndexInner(*file_handle_); data_ = static_cast(index); LOG_TRACE("Finished ReadFromFileImpl()."); } else { diff --git a/src/storage/buffer/file_worker/raw_file_worker.cpp b/src/storage/buffer/file_worker/raw_file_worker.cpp index f66444c740..174520cc35 100644 --- a/src/storage/buffer/file_worker/raw_file_worker.cpp +++ b/src/storage/buffer/file_worker/raw_file_worker.cpp @@ -13,13 +13,15 @@ // limitations under the License. module; + #include +#include module raw_file_worker; import stl; import infinity_exception; -import local_file_system; +import local_file_handle; import third_party; import status; import logger; @@ -64,10 +66,8 @@ void RawFileWorker::FreeInMemory() { bool RawFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const FileWorkerSaveCtx &ctx) { assert(data_ != nullptr && buffer_size_ > 0); - LocalFileSystem fs; - i64 nbytes = fs.Write(*file_handler_, data_, buffer_size_); - if (nbytes != (i64)buffer_size_) { - Status status = Status::DataIOError(fmt::format("Expect to write buffer with size: {}, but {} bytes is written", buffer_size_, nbytes)); + auto status = file_handle_->Append(data_, buffer_size_); + if(!status.ok()) { RecoverableError(status); } prepare_success = true; // Not run defer_fn @@ -75,11 +75,13 @@ bool RawFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const } void RawFileWorker::ReadFromFileImpl(SizeT file_size) { - LocalFileSystem fs; - buffer_size_ = fs.GetFileSize(*file_handler_); + buffer_size_ = file_handle_->FileSize(); data_ = static_cast(new char[buffer_size_]); - i64 nbytes = fs.Read(*file_handler_, data_, buffer_size_); - if (nbytes != (i64)buffer_size_) { + auto [nbytes, status1] = file_handle_->Read(data_, buffer_size_); + if(!status1.ok()) { + RecoverableError(status1); + } + if (nbytes != buffer_size_) { Status status = Status::DataIOError(fmt::format("Expect to read buffer with size: {}, but {} bytes is read", buffer_size_, nbytes)); RecoverableError(status); } diff --git a/src/storage/buffer/file_worker/secondary_index_file_worker.cpp b/src/storage/buffer/file_worker/secondary_index_file_worker.cpp index 303c1ecdf3..6321ba05b2 100644 --- a/src/storage/buffer/file_worker/secondary_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/secondary_index_file_worker.cpp @@ -66,7 +66,7 @@ void SecondaryIndexFileWorker::FreeInMemory() { bool SecondaryIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const FileWorkerSaveCtx &ctx) { if (data_) [[likely]] { auto index = static_cast(data_); - index->SaveIndexInner(*file_handler_); + index->SaveIndexInner(*file_handle_); prepare_success = true; LOG_TRACE("Finished WriteToFileImpl(bool &prepare_success)."); } else { @@ -79,7 +79,7 @@ bool SecondaryIndexFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_succ void SecondaryIndexFileWorker::ReadFromFileImpl(SizeT file_size) { if (!data_) [[likely]] { auto index = GetSecondaryIndexData(column_def_->type(), row_count_, false); - index->ReadIndexInner(*file_handler_); + index->ReadIndexInner(*file_handle_); data_ = static_cast(index); LOG_TRACE("Finished ReadFromFileImpl()."); } else { @@ -145,7 +145,7 @@ void SecondaryIndexFileWorkerParts::FreeInMemory() { bool SecondaryIndexFileWorkerParts::WriteToFileImpl(bool to_spill, bool &prepare_success, const FileWorkerSaveCtx &ctx) { if (data_) [[likely]] { - file_handler_->Write(data_, part_row_count_ * data_pair_size_); + file_handle_->Append(data_, part_row_count_ * data_pair_size_); prepare_success = true; LOG_TRACE("Finished WriteToFileImpl(bool &prepare_success)."); } else { @@ -163,7 +163,7 @@ void SecondaryIndexFileWorkerParts::ReadFromFileImpl(SizeT file_size) { if (!data_) [[likely]] { const u32 read_bytes = part_row_count_ * data_pair_size_; data_ = static_cast(new char[read_bytes]); - file_handler_->Read(data_, read_bytes); + file_handle_->Read(data_, read_bytes); LOG_TRACE("Finished ReadFromFileImpl()."); } else { String error_message = "ReadFromFileImpl: data_ is not nullptr"; diff --git a/src/storage/buffer/file_worker/var_file_worker.cpp b/src/storage/buffer/file_worker/var_file_worker.cpp index 266d23ecee..62979e7dfd 100644 --- a/src/storage/buffer/file_worker/var_file_worker.cpp +++ b/src/storage/buffer/file_worker/var_file_worker.cpp @@ -14,14 +14,17 @@ module; +#include + module var_file_worker; import stl; import infinity_exception; import var_buffer; import third_party; -import local_file_system; +import local_file_handle; import persistence_manager; +import status; namespace infinity { @@ -79,11 +82,9 @@ bool VarFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, const char *ptr = buffer_data.get(); buffer->Write(ptr); - LocalFileSystem fs; - u64 nbytes = fs.Write(*file_handler_, buffer_data.get(), data_size); - if (nbytes != data_size) { - String error_message = fmt::format("Write {} bytes to file failed, only {} bytes written.", data_size, nbytes); - UnrecoverableError(error_message); + Status status = file_handle_->Append(buffer_data.get(), data_size); + if(!status.ok()) { + UnrecoverableError(status.message()); } prepare_success = true; buffer_size_ = data_size; @@ -100,9 +101,11 @@ void VarFileWorker::ReadFromFileImpl(SizeT file_size) { UnrecoverableError(error_message); } - LocalFileSystem fs; auto buffer = MakeUnique(buffer_size_); - u64 nbytes = fs.Read(*file_handler_, buffer.get(), buffer_size_); + auto [nbytes, status] = file_handle_->Read(buffer.get(), buffer_size_); + if(!status.ok()) { + UnrecoverableError(status.message()); + } if (nbytes != buffer_size_) { String error_message = fmt::format("Read {} bytes from file failed, only {} bytes read.", buffer_size_, nbytes); UnrecoverableError(error_message); diff --git a/src/storage/buffer/file_worker/version_file_worker.cpp b/src/storage/buffer/file_worker/version_file_worker.cpp index ac26b885cd..430ceb93ad 100644 --- a/src/storage/buffer/file_worker/version_file_worker.cpp +++ b/src/storage/buffer/file_worker/version_file_worker.cpp @@ -76,11 +76,11 @@ bool VersionFileWorker::WriteToFileImpl(bool to_spill, bool &prepare_success, co TxnTimeStamp latest_change_ts = data->latest_change_ts(); if (to_spill) { - data->SpillToFile(*file_handler_); + data->SpillToFile(file_handle_.get()); return true; } else { const auto &ctx = static_cast(base_ctx); - data->SaveToFile(ctx.checkpoint_ts_, *file_handler_); + data->SaveToFile(ctx.checkpoint_ts_, *file_handle_); return ctx.checkpoint_ts_ >= latest_change_ts; } } @@ -90,7 +90,7 @@ void VersionFileWorker::ReadFromFileImpl(SizeT file_size) { String error_message = "Data is already allocated."; UnrecoverableError(error_message); } - auto *data = BlockVersion::LoadFromFile(*file_handler_).release(); + auto *data = BlockVersion::LoadFromFile(file_handle_.get()).release(); data_ = static_cast(data); } diff --git a/src/storage/invertedindex/column_index_iterator.cpp b/src/storage/invertedindex/column_index_iterator.cpp index 967e0e2588..accbc05cfc 100644 --- a/src/storage/invertedindex/column_index_iterator.cpp +++ b/src/storage/invertedindex/column_index_iterator.cpp @@ -12,7 +12,6 @@ import posting_list_format; import index_defines; import term_meta; import dict_reader; -import local_file_system; import third_party; import infinity_context; import persistence_manager; diff --git a/src/storage/invertedindex/column_index_merger.cpp b/src/storage/invertedindex/column_index_merger.cpp index 28d5921e69..3027a67476 100644 --- a/src/storage/invertedindex/column_index_merger.cpp +++ b/src/storage/invertedindex/column_index_merger.cpp @@ -103,18 +103,18 @@ void ColumnIndexMerger::Merge(const Vector &base_names, const VectorGetObjPath(obj_addr.obj_key_); } - auto [file_handler, status] = fs_.OpenFile(column_len_file, FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = fs_.OpenFile(column_len_file, FileFlags::READ_FLAG, FileLockType::kNoLock); if (!status.ok()) { UnrecoverableError(status.message()); } - const u32 file_size = fs_.GetFileSize(*file_handler); + const u32 file_size = fs_.GetFileSize(*file_handle); u32 file_read_array_len = file_size / sizeof(u32); if (unsafe_column_lengths.size() < id_offset + file_read_array_len) { unsafe_column_lengths.resize(id_offset + file_read_array_len); } - const i64 read_count = fs_.Read(*file_handler, unsafe_column_lengths.data() + id_offset, file_size); - file_handler->Close(); + const i64 read_count = fs_.Read(*file_handle, unsafe_column_lengths.data() + id_offset, file_size); + file_handle->Close(); if (read_count != file_size) { String error_message = "ColumnIndexMerger: when loading column length file, read_count != file_size"; UnrecoverableError(error_message); @@ -127,12 +127,12 @@ void ColumnIndexMerger::Merge(const Vector &base_names, const Vector &column_length_array = column_lengths_.UnsafeVec(); - fs.Write(*file_handler, &column_length_array[0], sizeof(column_length_array[0]) * column_length_array.size()); - fs.Close(*file_handler); + fs.Write(*file_handle, &column_length_array[0], sizeof(column_length_array[0]) * column_length_array.size()); + fs.Close(*file_handle); if (use_object_cache) { PersistResultHandler handler(pm); PersistWriteResult result1 = pm->Persist(posting_file, tmp_posting_file, false); @@ -356,15 +358,15 @@ void MemoryIndexer::Load() { } String column_length_file = index_prefix + LENGTH_SUFFIX + SPILL_SUFFIX; - auto [file_handler, status] = fs.OpenFile(column_length_file, FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(column_length_file, FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(status.message()); } Vector &column_lengths = column_lengths_.UnsafeVec(); column_lengths.resize(doc_count_); - fs.Read(*file_handler, &column_lengths[0], sizeof(column_lengths[0]) * column_lengths.size()); - fs.Close(*file_handler); + file_handle->Read(&column_lengths[0], sizeof(column_lengths[0]) * column_lengths.size()); + file_handle->Close(); u32 column_length_sum = column_lengths_.Sum(); column_length_sum_.store(column_length_sum); diff --git a/src/storage/io/local_file_handle.cpp b/src/storage/io/local_file_handle.cpp new file mode 100644 index 0000000000..9801e4050d --- /dev/null +++ b/src/storage/io/local_file_handle.cpp @@ -0,0 +1,151 @@ +// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +#include +#include +#include +#include +#include + +module local_file_handle; + +import third_party; +import status; +import virtual_store; +import infinity_exception; +import logger; +import abstract_file_handle; + +namespace infinity { + +LocalFileHandle::~LocalFileHandle() { + if(access_mode_ == FileAccessMode::kWrite) { + if(!sync_) { + UnrecoverableError("Not sync before destruction."); + } + } + if(fd_ != -1) { + UnrecoverableError("File isn't close before destruction."); + } +} + +Status LocalFileHandle::Close() { + if(access_mode_ == FileAccessMode::kWrite) { + if(!sync_) { + Status status = Sync(); + if(!status.ok()) { + return status; + } + } + } + if(fd_ == -1) { + UnrecoverableError("File was closed before"); + } + close(fd_); + fd_ = -1; + path_.clear(); + sync_ = false; + access_mode_ = FileAccessMode::kInvalid; + return Status::OK(); +} + +Status LocalFileHandle::Append(const void *buffer, u64 nbytes) { + if(access_mode_ != FileAccessMode::kWrite) { + String error_message = fmt::format("File: {} isn't open.", path_); + UnrecoverableError(error_message); + } + i64 written = 0; + while (written < (i64)nbytes) { + i64 write_count = write(fd_, (char*)buffer + written, nbytes - written); + if (write_count == -1) { + String error_message = fmt::format("Can't write file: {}: {}. fd: {}", path_, strerror(errno), fd_); + UnrecoverableError(error_message); + } + written += write_count; + } + return Status::OK(); +} + +Status LocalFileHandle::Append(const String &buffer, u64 nbytes) { + return Append(buffer.data(), nbytes); +} + +Tuple LocalFileHandle::Read(void *buffer, u64 nbytes) { + i64 read_n = 0; + while (read_n < (i64)nbytes) { + SizeT a = nbytes - read_n; + i64 read_count = read(fd_, (char *)buffer + read_n, a); + if (read_count == 0) { + break; + } + if (read_count == -1) { + String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); + UnrecoverableError(error_message); + } + read_n += read_count; + } + return {read_n, Status::OK()}; +} + +Tuple LocalFileHandle::Read(String &buffer, u64 nbytes) { + i64 read_n = 0; + while (read_n < (i64)nbytes) { + SizeT a = nbytes - read_n; + i64 read_count = read(fd_, buffer.data() + read_n, a); + if (read_count == 0) { + break; + } + if (read_count == -1) { + String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); + UnrecoverableError(error_message); + } + read_n += read_count; + } + return {read_n, Status::OK()}; +} + +Status LocalFileHandle::Seek(u64 nbytes) { + if ((off_t)-1 == lseek(fd_, nbytes, SEEK_SET)) { + String error_message = fmt::format("Can't seek file: {}: {}", path_, strerror(errno)); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +SizeT LocalFileHandle::FileSize() { + struct stat s {}; + if (fstat(fd_, &s) == -1) { + return -1; + } + return s.st_size; +} + +Tuple LocalFileHandle::MmapRead(const String &name) { return {nullptr, 0, Status::OK()}; } + +Status LocalFileHandle::Unmmap(const String &name) { return Status::OK(); } + +Status LocalFileHandle::Sync() { + if(access_mode_ != FileAccessMode::kWrite) { + return Status::InvalidCommand("Non-write access mode, shouldn't call Sync()"); + } + if(!sync_) { + sync_ = true; + fsync(fd_); + } + return Status::OK(); +} + +} // namespace infinity diff --git a/src/storage/io/local_file_handle.cppm b/src/storage/io/local_file_handle.cppm new file mode 100644 index 0000000000..687e4f3777 --- /dev/null +++ b/src/storage/io/local_file_handle.cppm @@ -0,0 +1,57 @@ +// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +export module local_file_handle; + +import stl; +import status; +import abstract_file_handle; + +namespace infinity { + +export class LocalFileHandle { +public: + LocalFileHandle(i32 fd, const String &path, FileAccessMode file_access_mode) : fd_(fd), path_(path), access_mode_(file_access_mode) {} + ~LocalFileHandle(); + + Status Close(); + Status Append(const void *buffer, u64 nbytes); + Status Append(const String &buffer, u64 nbytes); + Tuple Read(void *buffer, u64 nbytes); + Tuple Read(String &buffer, u64 nbytes); + Status Seek(u64 nbytes); + SizeT FileSize(); + Tuple MmapRead(const String &name); + Status Unmmap(const String &name); + Status Sync(); + +public: + i32 FileDescriptor() const { + return fd_; + } + + String Path() const { + return path_; + } + +private: + i32 fd_{-1}; + String path_{}; + FileAccessMode access_mode_{FileAccessMode::kInvalid}; + atomic_bool sync_{false}; +}; + +} // namespace infinity \ No newline at end of file diff --git a/src/storage/io/virtual_store.cpp b/src/storage/io/virtual_store.cpp new file mode 100644 index 0000000000..f4f58be0b1 --- /dev/null +++ b/src/storage/io/virtual_store.cpp @@ -0,0 +1,309 @@ +// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +#include +#include +#include +#include +#include +#include + +module virtual_store; + +import stl; +import third_party; +import virtual_storage_type; +import logger; +import infinity_exception; +import default_values; +import abstract_file_handle; + +namespace infinity { + +Status RemoteStore::Init(StorageType storage_type, Map &config) { + // Init remote filesystem and local disk cache + storage_type_ = storage_type; + switch (storage_type) { + case StorageType::kMinio: { + auto iter = config.find("url"); + if (iter == config.end()) { + return Status::InvalidConfig("Missing MINIO 'URL'"); + } + String url = iter->second; + + iter = config.find("access_key"); + if (iter == config.end()) { + return Status::InvalidConfig("Missing MINIO 'access_key'"); + } + String access_key = iter->second; + + iter = config.find("secret_key"); + if (iter == config.end()) { + return Status::InvalidConfig("Missing MINIO 'secret_key'"); + } + String secret_key = iter->second; + + iter = config.find("enable_https"); + if (iter == config.end()) { + return Status::InvalidConfig("Missing MINIO 'enable_https'"); + } + String enable_https_str = iter->second; + bool enable_https{false}; + if (enable_https_str == "true") { + enable_https = true; + } else if (enable_https_str == "false") { + enable_https = false; + } else { + return Status::InvalidConfig(fmt::format("Invalid MINIO 'enable_https' value: {}", enable_https_str)); + } + + minio_base_url_ = MakeUnique(url, enable_https); + minio_provider_ = MakeUnique(access_key, secret_key); + minio_client_ = MakeUnique(*minio_base_url_, minio_provider_.get()); + break; + } + default: { + return Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type))); + } + } + return Status::OK(); +} + +Status RemoteStore::UnInit() { + switch (storage_type_) { + case StorageType::kMinio: { + minio_base_url_.reset(); + minio_provider_.reset(); + minio_client_.reset(); + break; + } + default: { + return Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type_))); + } + } + return Status::OK(); +} + +Tuple, Status> LocalStore::Open(const String& path, FileAccessMode access_mode) { + i32 fd = -1; + switch (access_mode) { + case FileAccessMode::kRead: { + fd = open(path.c_str(), O_RDONLY, 0666); + break; + } + case FileAccessMode::kWrite: { + fd = open(path.c_str(), O_RDWR | O_CREAT, 0666); + break; + } + case FileAccessMode::kMmapRead: { + UnrecoverableError("Unsupported now."); + break; + } + case FileAccessMode::kInvalid: { + break; + } + } + if(fd == -1) { + String error_message = fmt::format("File open failed: {}", strerror(errno)); + return {nullptr, Status::IOError(error_message)}; + } + return {MakeUnique(fd, path, access_mode), Status::OK()}; +} + + +// For local disk filesystem, such as temp file, disk cache and WAL +bool LocalStore::Exists(const String &path) { + if (!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + std::error_code error_code; + Path p{path}; + bool is_exists = std::filesystem::exists(p, error_code); + if (error_code.value() == 0) { + return is_exists; + } else { + String error_message = fmt::format("{} exists exception: {}", path, strerror(errno)); + UnrecoverableError(error_message); + } + return false; +} + +Status LocalStore::DeleteFile(const String &file_name) { + if (!std::filesystem::path(file_name).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", file_name); + UnrecoverableError(error_message); + } + std::error_code error_code; + Path p{file_name}; + bool is_deleted = std::filesystem::remove(p, error_code); + if (error_code.value() == 0) { + if (!is_deleted) { + String error_message = fmt::format("Failed to delete file: {}: {}", file_name, strerror(errno)); + LOG_WARN(error_message); + Status status = Status::IOError(error_message); + return status; + } + } else { + String error_message = fmt::format("Delete file {} exception: {}", file_name, strerror(errno)); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::MakeDirectory(const String &path) { + if (!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + std::error_code error_code; + Path p{path}; + std::filesystem::create_directories(p, error_code); + if (error_code.value() != 0) { + String error_message = fmt::format("{} create exception: {}", path, strerror(errno)); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::RemoveDirectory(const String &path) { + if (!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + std::error_code error_code; + Path p{path}; + std::filesystem::remove_all(p, error_code); + if (error_code.value() != 0) { + String error_message = fmt::format("Delete directory {} exception: {}", path, error_code.message()); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::CleanupDirectory(const String &path) { + if (!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + std::error_code error_code; + Path p{path}; + if (!std::filesystem::exists(p)) { + std::filesystem::create_directories(p, error_code); + if (error_code.value() != 0) { + String error_message = fmt::format("CleanupDirectory create {} exception: {}", path, error_code.message()); + UnrecoverableError(error_message); + } + return Status::OK(); + } + try { + std::ranges::for_each(std::filesystem::directory_iterator{path}, [&](const auto &dir_entry) { std::filesystem::remove_all(dir_entry); }); + } catch (const std::filesystem::filesystem_error &e) { + String error_message = fmt::format("CleanupDirectory cleanup {} exception: {}", path, e.what()); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::Rename(const String &old_path, const String &new_path) { + if (!std::filesystem::path(old_path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", old_path); + UnrecoverableError(error_message); + } + if (!std::filesystem::path(new_path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", new_path); + UnrecoverableError(error_message); + } + if (rename(old_path.c_str(), new_path.c_str()) != 0) { + String error_message = fmt::format("Can't rename file: {}, {}", old_path, strerror(errno)); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::Truncate(const String &file_name, SizeT new_length) { + if (!std::filesystem::path(file_name).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", file_name); + UnrecoverableError(error_message); + } + std::error_code error_code; + std::filesystem::resize_file(file_name, new_length, error_code); + if (error_code.value() != 0) { + String error_message = fmt::format("Failed to truncate {} to size {}", file_name, strerror(errno)); + UnrecoverableError(error_message); + } + return Status::OK(); +} + +Status LocalStore::Merge(const String &dst_path, const String &src_path) { + if (!std::filesystem::path(dst_path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", dst_path); + UnrecoverableError(error_message); + } + if (!std::filesystem::path(src_path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", src_path); + UnrecoverableError(error_message); + } + Path dst{dst_path}; + Path src{src_path}; + std::ifstream srcFile(src, std::ios::binary); + if (!srcFile.is_open()) { + String error_message = fmt::format("Failed to open source file {}", src_path); + UnrecoverableError(error_message); + return Status::OK(); + } + std::ofstream dstFile(dst, std::ios::binary | std::ios::app); + if (!dstFile.is_open()) { + String error_message = fmt::format("Failed to open destination file {}", dst_path); + UnrecoverableError(error_message); + return Status::OK(); + } + char buffer[DEFAULT_READ_BUFFER_SIZE]; + while (srcFile.read(buffer, DEFAULT_READ_BUFFER_SIZE)) { + dstFile.write(buffer, srcFile.gcount()); + } + dstFile.write(buffer, srcFile.gcount()); + srcFile.close(); + dstFile.close(); + return Status::OK(); +} + +Tuple>, Status> LocalStore::ListDirectory(const String &path) { + if (!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + Path dir_path(path); + if (!is_directory(dir_path)) { + String error_message = fmt::format("{} isn't a directory", path); + UnrecoverableError(error_message); + } + + Vector> file_array; + std::ranges::for_each(std::filesystem::directory_iterator{path}, + [&](const auto &dir_entry) { file_array.emplace_back(MakeShared(dir_entry)); }); + return {file_array, Status::OK()}; +} + +SizeT LocalStore::GetFileSize(const String& path) { + if(!std::filesystem::path(path).is_absolute()) { + String error_message = fmt::format("{} isn't absolute path.", path); + UnrecoverableError(error_message); + } + return std::filesystem::file_size(path); +} + +} // namespace infinity diff --git a/src/storage/io/virtual_store.cppm b/src/storage/io/virtual_store.cppm new file mode 100644 index 0000000000..310ead4894 --- /dev/null +++ b/src/storage/io/virtual_store.cppm @@ -0,0 +1,59 @@ +// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module; + +export module virtual_store; + +import stl; +import status; +import third_party; +import local_file_handle; +import virtual_storage_type; +import abstract_file_handle; + +namespace infinity { + +// Only one instance; +export class RemoteStore { +public: + Status Init(StorageType storage_type, Map &config); + Status UnInit(); + +private: + // Using by minio + StorageType storage_type_{StorageType::kLocal}; + + UniquePtr minio_base_url_{}; + UniquePtr minio_provider_{}; + UniquePtr minio_client_{}; +}; + +export class LocalStore { +public: + static Tuple, Status> Open(const String& path, FileAccessMode access_mode); + static bool IsRegularFile(const String& path); + static bool Exists(const String& path); + static Status DeleteFile(const String& path); + static Status MakeDirectory(const String& path); + static Status RemoveDirectory(const String& path); + static Status CleanupDirectory(const String& path); + static Status Rename(const String& old_path, const String& new_path); + static Status Truncate(const String& file_name, SizeT new_length); + static Status Merge(const String& dst_file, const String& src_file); + static Tuple>, Status> ListDirectory(const String& path); + static SizeT GetFileSize(const String& path); +}; + +} diff --git a/src/storage/knn_index/emvb/emvb_index.cpp b/src/storage/knn_index/emvb/emvb_index.cpp index 2fb6f13069..d5c1ccf438 100644 --- a/src/storage/knn_index/emvb/emvb_index.cpp +++ b/src/storage/knn_index/emvb/emvb_index.cpp @@ -482,113 +482,113 @@ EMVBQueryResultType EMVBIndex::GetQueryResultT(const f32 *query_ptr, const u32 q } template -void Serialize(FileHandler &file_handler, const Vector &val) { +void Serialize(LocalFileHandle &file_handle, const Vector &val) { const u32 size = val.size(); - file_handler.Write(&size, sizeof(size)); - file_handler.Write(val.data(), size * sizeof(T)); + file_handle.Append(&size, sizeof(size)); + file_handle.Append(val.data(), size * sizeof(T)); } template -void DeSerialize(FileHandler &file_handler, Vector &val) { +void DeSerialize(LocalFileHandle &file_handle, Vector &val) { u32 size = 0; - file_handler.Read(&size, sizeof(size)); + file_handle.Read(&size, sizeof(size)); val.resize(size); - file_handler.Read(val.data(), size * sizeof(T)); + file_handle.Read(val.data(), size * sizeof(T)); } -void Serialize(FileHandler &file_handler, const EMVBSharedVec &val, const u32 expect_element_num) { +void Serialize(LocalFileHandle &file_handle, const EMVBSharedVec &val, const u32 expect_element_num) { const auto [shared_u32_ptr, size] = val.GetData(); if (size != expect_element_num) { const auto error_msg = fmt::format("EMVBSharedVec size mismatch: expect {}, got {}.", expect_element_num, size); UnrecoverableError(error_msg); } - file_handler.Write(&expect_element_num, sizeof(expect_element_num)); - file_handler.Write(shared_u32_ptr.get(), expect_element_num * sizeof(u32)); + file_handle.Append(&expect_element_num, sizeof(expect_element_num)); + file_handle.Append(shared_u32_ptr.get(), expect_element_num * sizeof(u32)); } -void DeSerialize(FileHandler &file_handler, EMVBSharedVec &val, const u32 expect_element_num) { +void DeSerialize(LocalFileHandle &file_handle, EMVBSharedVec &val, const u32 expect_element_num) { if (const auto [_, old_size] = val.GetData(); old_size > 0) { const auto error_msg = fmt::format("EMVBSharedVec size mismatch: expect 0, got {}.", old_size); UnrecoverableError(error_msg); } u32 size = 0; - file_handler.Read(&size, sizeof(size)); + file_handle.Read(&size, sizeof(size)); if (size != expect_element_num) { const auto error_msg = fmt::format("EMVBSharedVec size mismatch: expect {}, got {}.", expect_element_num, size); UnrecoverableError(error_msg); } const auto tmp_buffer = MakeUniqueForOverwrite(expect_element_num); - file_handler.Read(tmp_buffer.get(), expect_element_num * sizeof(u32)); + file_handle.Read(tmp_buffer.get(), expect_element_num * sizeof(u32)); val.PushBack(tmp_buffer.get(), tmp_buffer.get() + expect_element_num); } -void Serialize(FileHandler &file_handler, const EMVBSharedVec &val) { +void Serialize(LocalFileHandle &file_handle, const EMVBSharedVec &val) { const auto [shared_u32_ptr, size] = val.GetData(); const u32 element_num = size; - file_handler.Write(&element_num, sizeof(element_num)); - file_handler.Write(shared_u32_ptr.get(), size * sizeof(u32)); + file_handle.Append(&element_num, sizeof(element_num)); + file_handle.Append(shared_u32_ptr.get(), size * sizeof(u32)); } -void DeSerialize(FileHandler &file_handler, EMVBSharedVec &val) { +void DeSerialize(LocalFileHandle &file_handle, EMVBSharedVec &val) { if (const auto [_, old_size] = val.GetData(); old_size > 0) { const auto error_msg = fmt::format("EMVBSharedVec size mismatch: expect 0, got {}.", old_size); UnrecoverableError(error_msg); } u32 element_num = 0; - file_handler.Read(&element_num, sizeof(element_num)); + file_handle.Read(&element_num, sizeof(element_num)); const auto tmp_buffer = MakeUniqueForOverwrite(element_num); - file_handler.Read(tmp_buffer.get(), element_num * sizeof(u32)); + file_handle.Read(tmp_buffer.get(), element_num * sizeof(u32)); val.PushBack(tmp_buffer.get(), tmp_buffer.get() + element_num); } -void EMVBIndex::SaveIndexInner(FileHandler &file_handler) { +void EMVBIndex::SaveIndexInner(LocalFileHandle &file_handle) { std::unique_lock lock(rw_mutex_); // write index data - file_handler.Write(&start_segment_offset_, sizeof(start_segment_offset_)); - file_handler.Write(&embedding_dimension_, sizeof(embedding_dimension_)); - file_handler.Write(&residual_pq_subspace_num_, sizeof(residual_pq_subspace_num_)); - file_handler.Write(&residual_pq_subspace_bits_, sizeof(residual_pq_subspace_bits_)); - file_handler.Write(&n_centroids_, sizeof(n_centroids_)); - Serialize(file_handler, centroids_data_); - Serialize(file_handler, centroid_norms_neg_half_); + file_handle.Append(&start_segment_offset_, sizeof(start_segment_offset_)); + file_handle.Append(&embedding_dimension_, sizeof(embedding_dimension_)); + file_handle.Append(&residual_pq_subspace_num_, sizeof(residual_pq_subspace_num_)); + file_handle.Append(&residual_pq_subspace_bits_, sizeof(residual_pq_subspace_bits_)); + file_handle.Append(&n_centroids_, sizeof(n_centroids_)); + Serialize(file_handle, centroids_data_); + Serialize(file_handle, centroid_norms_neg_half_); const u32 n_docs = n_docs_; - file_handler.Write(&n_docs, sizeof(n_docs)); - file_handler.Write(&n_total_embeddings_, sizeof(n_total_embeddings_)); - Serialize(file_handler, doc_lens_, n_docs); - Serialize(file_handler, doc_offsets_, n_docs); - Serialize(file_handler, centroid_id_assignments_, n_total_embeddings_); + file_handle.Append(&n_docs, sizeof(n_docs)); + file_handle.Append(&n_total_embeddings_, sizeof(n_total_embeddings_)); + Serialize(file_handle, doc_lens_, n_docs); + Serialize(file_handle, doc_offsets_, n_docs); + Serialize(file_handle, centroid_id_assignments_, n_total_embeddings_); for (u32 i = 0; i < n_centroids_; ++i) { - Serialize(file_handler, centroids_to_docid_[i]); + Serialize(file_handle, centroids_to_docid_[i]); } // write product quantizer - product_quantizer_->Save(file_handler); + product_quantizer_->Save(file_handle); } -void EMVBIndex::ReadIndexInner(FileHandler &file_handler) { +void EMVBIndex::ReadIndexInner(LocalFileHandle &file_handle) { std::unique_lock lock(rw_mutex_); // read index data { // check start_segment_offset_, embedding_dimension_, residual_pq_subspace_num_, residual_pq_subspace_bits_ u32 tmp_u32 = 0; - file_handler.Read(&tmp_u32, sizeof(tmp_u32)); + file_handle.Read(&tmp_u32, sizeof(tmp_u32)); if (tmp_u32 != start_segment_offset_) { const auto error_msg = fmt::format("EMVBIndex::ReadIndexInner: start_segment_offset_ mismatch: expect {}, got {}.", start_segment_offset_, tmp_u32); UnrecoverableError(error_msg); } - file_handler.Read(&tmp_u32, sizeof(tmp_u32)); + file_handle.Read(&tmp_u32, sizeof(tmp_u32)); if (tmp_u32 != embedding_dimension_) { const auto error_msg = fmt::format("EMVBIndex::ReadIndexInner: embedding_dimension_ mismatch: expect {}, got {}.", embedding_dimension_, tmp_u32); UnrecoverableError(error_msg); } - file_handler.Read(&tmp_u32, sizeof(tmp_u32)); + file_handle.Read(&tmp_u32, sizeof(tmp_u32)); if (tmp_u32 != residual_pq_subspace_num_) { const auto error_msg = fmt::format("EMVBIndex::ReadIndexInner: residual_pq_subspace_num_ mismatch: expect {}, got {}.", residual_pq_subspace_num_, tmp_u32); UnrecoverableError(error_msg); } - file_handler.Read(&tmp_u32, sizeof(tmp_u32)); + file_handle.Read(&tmp_u32, sizeof(tmp_u32)); if (tmp_u32 != residual_pq_subspace_bits_) { const auto error_msg = fmt::format("EMVBIndex::ReadIndexInner: residual_pq_subspace_bits_ mismatch: expect {}, got {}.", residual_pq_subspace_bits_, @@ -596,22 +596,22 @@ void EMVBIndex::ReadIndexInner(FileHandler &file_handler) { UnrecoverableError(error_msg); } } - file_handler.Read(&n_centroids_, sizeof(n_centroids_)); - DeSerialize(file_handler, centroids_data_); - DeSerialize(file_handler, centroid_norms_neg_half_); + file_handle.Read(&n_centroids_, sizeof(n_centroids_)); + DeSerialize(file_handle, centroids_data_); + DeSerialize(file_handle, centroid_norms_neg_half_); u32 n_docs = 0; - file_handler.Read(&n_docs, sizeof(n_docs)); + file_handle.Read(&n_docs, sizeof(n_docs)); n_docs_ = n_docs; - file_handler.Read(&n_total_embeddings_, sizeof(n_total_embeddings_)); - DeSerialize(file_handler, doc_lens_, n_docs); - DeSerialize(file_handler, doc_offsets_, n_docs); - DeSerialize(file_handler, centroid_id_assignments_, n_total_embeddings_); + file_handle.Read(&n_total_embeddings_, sizeof(n_total_embeddings_)); + DeSerialize(file_handle, doc_lens_, n_docs); + DeSerialize(file_handle, doc_offsets_, n_docs); + DeSerialize(file_handle, centroid_id_assignments_, n_total_embeddings_); centroids_to_docid_ = MakeUnique[]>(n_centroids_); for (u32 i = 0; i < n_centroids_; ++i) { - DeSerialize(file_handler, centroids_to_docid_[i]); + DeSerialize(file_handle, centroids_to_docid_[i]); } // read product quantizer - product_quantizer_->Load(file_handler); + product_quantizer_->Load(file_handle); } EMVBIndex &EMVBIndex::operator=(EMVBIndex &&other) { diff --git a/src/storage/knn_index/emvb/emvb_index.cppm b/src/storage/knn_index/emvb/emvb_index.cppm index ef10558c80..be2bfd5592 100644 --- a/src/storage/knn_index/emvb/emvb_index.cppm +++ b/src/storage/knn_index/emvb/emvb_index.cppm @@ -23,7 +23,7 @@ namespace infinity { extern template class EMVBSharedVec; class EMVBProductQuantizer; -class FileHandler; +class LocalFileHandle; struct RowID; struct SegmentEntry; class ColumnDef; @@ -79,9 +79,9 @@ public: u32 out_second_stage, f32 threshold_final) const; - void SaveIndexInner(FileHandler &file_handler); + void SaveIndexInner(LocalFileHandle &file_handle); - void ReadIndexInner(FileHandler &file_handler); + void ReadIndexInner(LocalFileHandle &file_handler); u32 GetDocNum() const; diff --git a/src/storage/knn_index/emvb/product_quantizer.cpp b/src/storage/knn_index/emvb/product_quantizer.cpp index c78789d3f4..b680fc990f 100644 --- a/src/storage/knn_index/emvb/product_quantizer.cpp +++ b/src/storage/knn_index/emvb/product_quantizer.cpp @@ -27,7 +27,7 @@ import index_base; import third_party; import logger; import infinity_exception; -import file_system; +import local_file_handle; import eigen_svd; namespace infinity { @@ -356,7 +356,7 @@ void PQ::GetMultipleIPDistance(const u32 em } template -void OPQ::Save(FileHandler &file_handler) { +void OPQ::Save(LocalFileHandle &file_handle) { std::unique_lock lock(this->rw_mutex_); const u32 subspace_centroid_data_size = this->subspace_centroid_num_ * this->subspace_dimension_; for (const auto ¢roids_v : this->subspace_centroids_) { @@ -364,10 +364,10 @@ void OPQ::Save(FileHandler &file_handler) { const auto error_info = fmt::format("centroids size {} not equal to expected size {}", centroids_v.size(), subspace_centroid_data_size); UnrecoverableError(error_info); } - file_handler.Write(centroids_v.data(), subspace_centroid_data_size * sizeof(typename std::decay_t::value_type)); + file_handle.Append(centroids_v.data(), subspace_centroid_data_size * sizeof(typename std::decay_t::value_type)); } for (const auto &norms_v : this->subspace_centroid_norms_neg_half) { - file_handler.Write(norms_v.data(), norms_v.size() * sizeof(typename std::decay_t::value_type)); + file_handle.Append(norms_v.data(), norms_v.size() * sizeof(typename std::decay_t::value_type)); } const u32 encoded_embedding_data_size = this->encoded_embedding_data_.size(); if (encoded_embedding_data_size != this->next_embedding_id_) { @@ -375,42 +375,42 @@ void OPQ::Save(FileHandler &file_handler) { fmt::format("encoded_embedding_data size {} not equal to expected size {}", encoded_embedding_data_size, this->next_embedding_id_); UnrecoverableError(error_info); } - file_handler.Write(&encoded_embedding_data_size, sizeof(encoded_embedding_data_size)); + file_handle.Append(&encoded_embedding_data_size, sizeof(encoded_embedding_data_size)); for (const auto &encoded_embedding : this->encoded_embedding_data_) { - file_handler.Write(encoded_embedding.data(), + file_handle.Append(encoded_embedding.data(), encoded_embedding.size() * sizeof(typename std::decay_t::value_type)); } - file_handler.Write(&this->next_embedding_id_, sizeof(this->next_embedding_id_)); + file_handle.Append(&this->next_embedding_id_, sizeof(this->next_embedding_id_)); // save matrix R - file_handler.Write(matrix_R_.get(), this->dimension_ * this->dimension_ * sizeof(typename decltype(matrix_R_)::element_type)); + file_handle.Append(matrix_R_.get(), this->dimension_ * this->dimension_ * sizeof(typename decltype(matrix_R_)::element_type)); } template -void OPQ::Load(FileHandler &file_handler) { +void OPQ::Load(LocalFileHandle &file_handle) { std::unique_lock lock(this->rw_mutex_); const u32 subspace_centroid_data_size = this->subspace_centroid_num_ * this->subspace_dimension_; for (auto ¢roids_v : this->subspace_centroids_) { centroids_v.resize(subspace_centroid_data_size); - file_handler.Read(centroids_v.data(), subspace_centroid_data_size * sizeof(typename std::decay_t::value_type)); + file_handle.Read(centroids_v.data(), subspace_centroid_data_size * sizeof(typename std::decay_t::value_type)); } for (auto &norms_v : this->subspace_centroid_norms_neg_half) { - file_handler.Read(norms_v.data(), norms_v.size() * sizeof(typename std::decay_t::value_type)); + file_handle.Read(norms_v.data(), norms_v.size() * sizeof(typename std::decay_t::value_type)); } u32 encoded_embedding_data_size = 0; - file_handler.Read(&encoded_embedding_data_size, sizeof(encoded_embedding_data_size)); + file_handle.Read(&encoded_embedding_data_size, sizeof(encoded_embedding_data_size)); this->encoded_embedding_data_.resize(encoded_embedding_data_size); for (auto &encoded_embedding : this->encoded_embedding_data_) { - file_handler.Read(encoded_embedding.data(), + file_handle.Read(encoded_embedding.data(), encoded_embedding.size() * sizeof(typename std::decay_t::value_type)); } - file_handler.Read(&this->next_embedding_id_, sizeof(this->next_embedding_id_)); + file_handle.Read(&this->next_embedding_id_, sizeof(this->next_embedding_id_)); if (encoded_embedding_data_size != this->next_embedding_id_) { const auto error_info = fmt::format("encoded_embedding_data size {} not equal to expected size {}", encoded_embedding_data_size, this->next_embedding_id_); UnrecoverableError(error_info); } // load matrix R - file_handler.Read(matrix_R_.get(), this->dimension_ * this->dimension_ * sizeof(typename decltype(matrix_R_)::element_type)); + file_handle.Read(matrix_R_.get(), this->dimension_ * this->dimension_ * sizeof(typename decltype(matrix_R_)::element_type)); } constexpr u32 current_max_subspace_num = 128; diff --git a/src/storage/knn_index/emvb/product_quantizer.cppm b/src/storage/knn_index/emvb/product_quantizer.cppm index a66b070cde..830db751bc 100644 --- a/src/storage/knn_index/emvb/product_quantizer.cppm +++ b/src/storage/knn_index/emvb/product_quantizer.cppm @@ -18,7 +18,7 @@ export module emvb_product_quantization; import stl; namespace infinity { -class FileHandler; +class LocalFileHandle; export class EMVBProductQuantizer { public: @@ -29,8 +29,8 @@ public: virtual f32 GetSingleIPDistance(u32 embedding_id, u32 query_id, u32 query_num, const f32 *ip_table) const = 0; virtual void GetMultipleIPDistance(u32 embedding_offset, u32 embedding_num, u32 query_id, u32 query_num, const f32 *ip_table, f32 *output_ptr) const = 0; - virtual void Save(FileHandler &file_handler) = 0; - virtual void Load(FileHandler &file_handler) = 0; + virtual void Save(LocalFileHandle &file_handle) = 0; + virtual void Load(LocalFileHandle &file_handle) = 0; }; template @@ -76,9 +76,9 @@ public: UniquePtr GetIPDistanceTable(const f32 *query_data, u32 query_num) const override; - void Save(FileHandler &file_handler) override; + void Save(LocalFileHandle &file_handle) override; - void Load(FileHandler &file_handler) override; + void Load(LocalFileHandle &file_handle) override; }; export UniquePtr GetEMVBOPQ(u32 pq_subspace_num, u32 pq_subspace_bits, u32 embedding_dimension); diff --git a/src/storage/knn_index/knn_hnsw/data_store/data_store.cppm b/src/storage/knn_index/knn_hnsw/data_store/data_store.cppm index 6dd7984bd4..f6df8fefa6 100644 --- a/src/storage/knn_index/knn_hnsw/data_store/data_store.cppm +++ b/src/storage/knn_index/knn_hnsw/data_store/data_store.cppm @@ -22,7 +22,7 @@ export module data_store; import stl; import hnsw_common; -import file_system; +import local_file_handle; import vec_store_type; import graph_store; import infinity_exception; @@ -143,36 +143,36 @@ public: return size; } - void Save(FileHandler &file_handler) const { + void Save(LocalFileHandle &file_handle) const { SizeT cur_vec_num = this->cur_vec_num(); auto [chunk_num, last_chunk_size] = ChunkInfo(cur_vec_num); - file_handler.Write(&chunk_size_, sizeof(chunk_size_)); - file_handler.Write(&max_chunk_n_, sizeof(max_chunk_n_)); + file_handle.Append(&chunk_size_, sizeof(chunk_size_)); + file_handle.Append(&max_chunk_n_, sizeof(max_chunk_n_)); - file_handler.Write(&cur_vec_num, sizeof(cur_vec_num)); - vec_store_meta_.Save(file_handler); - graph_store_meta_.Save(file_handler); + file_handle.Append(&cur_vec_num, sizeof(cur_vec_num)); + vec_store_meta_.Save(file_handle); + graph_store_meta_.Save(file_handle); for (SizeT i = 0; i < chunk_num; ++i) { SizeT chunk_size = (i < chunk_num - 1) ? chunk_size_ : last_chunk_size; - inners_[i].Save(file_handler, chunk_size, vec_store_meta_, graph_store_meta_); + inners_[i].Save(file_handle, chunk_size, vec_store_meta_, graph_store_meta_); } } - static This Load(FileHandler &file_handler, SizeT max_chunk_n = 0) { + static This Load(LocalFileHandle &file_handle, SizeT max_chunk_n = 0) { SizeT chunk_size; - file_handler.Read(&chunk_size, sizeof(chunk_size)); + file_handle.Read(&chunk_size, sizeof(chunk_size)); SizeT max_chunk_n1; - file_handler.Read(&max_chunk_n1, sizeof(max_chunk_n1)); + file_handle.Read(&max_chunk_n1, sizeof(max_chunk_n1)); if (max_chunk_n == 0) { max_chunk_n = max_chunk_n1; } assert(max_chunk_n >= max_chunk_n1); SizeT cur_vec_num; - file_handler.Read(&cur_vec_num, sizeof(cur_vec_num)); - VecStoreMeta vec_store_meta = VecStoreMeta::Load(file_handler); - GraphStoreMeta graph_store_meta = GraphStoreMeta::Load(file_handler); + file_handle.Read(&cur_vec_num, sizeof(cur_vec_num)); + VecStoreMeta vec_store_meta = VecStoreMeta::Load(file_handle); + GraphStoreMeta graph_store_meta = GraphStoreMeta::Load(file_handle); This ret = This(chunk_size, max_chunk_n, std::move(vec_store_meta), std::move(graph_store_meta)); ret.cur_vec_num_ = cur_vec_num; @@ -181,7 +181,7 @@ public: auto [chunk_num, last_chunk_size] = ret.ChunkInfo(cur_vec_num); for (SizeT i = 0; i < chunk_num; ++i) { SizeT cur_chunk_size = (i < chunk_num - 1) ? chunk_size : last_chunk_size; - ret.inners_[i] = Inner::Load(file_handler, cur_chunk_size, chunk_size, ret.vec_store_meta_, ret.graph_store_meta_, mem_usage); + ret.inners_[i] = Inner::Load(file_handle, cur_chunk_size, chunk_size, ret.vec_store_meta_, ret.graph_store_meta_, mem_usage); } ret.mem_usage_.store(mem_usage); return ret; @@ -409,22 +409,22 @@ public: return size; } - void Save(FileHandler &file_handler, SizeT cur_vec_num, const VecStoreMeta &vec_store_meta, const GraphStoreMeta &graph_store_meta) const { - vec_store_inner_.Save(file_handler, cur_vec_num, vec_store_meta); - graph_store_inner_.Save(file_handler, cur_vec_num, graph_store_meta); - file_handler.Write(labels_.get(), sizeof(LabelType) * cur_vec_num); + void Save(LocalFileHandle &file_handle, SizeT cur_vec_num, const VecStoreMeta &vec_store_meta, const GraphStoreMeta &graph_store_meta) const { + vec_store_inner_.Save(file_handle, cur_vec_num, vec_store_meta); + graph_store_inner_.Save(file_handle, cur_vec_num, graph_store_meta); + file_handle.Append(labels_.get(), sizeof(LabelType) * cur_vec_num); } - static This Load(FileHandler &file_handler, + static This Load(LocalFileHandle &file_handle, SizeT cur_vec_num, SizeT chunk_size, VecStoreMeta &vec_store_meta, GraphStoreMeta &graph_store_meta, SizeT &mem_usage) { - auto vec_store_inner = VecStoreInner::Load(file_handler, cur_vec_num, chunk_size, vec_store_meta, mem_usage); - auto graph_store_iner = GraphStoreInner::Load(file_handler, cur_vec_num, chunk_size, graph_store_meta, mem_usage); + auto vec_store_inner = VecStoreInner::Load(file_handle, cur_vec_num, chunk_size, vec_store_meta, mem_usage); + auto graph_store_iner = GraphStoreInner::Load(file_handle, cur_vec_num, chunk_size, graph_store_meta, mem_usage); This ret(chunk_size, std::move(vec_store_inner), std::move(graph_store_iner)); - file_handler.Read(ret.labels_.get(), sizeof(LabelType) * cur_vec_num); + file_handle.Read(ret.labels_.get(), sizeof(LabelType) * cur_vec_num); return ret; } diff --git a/src/storage/knn_index/knn_hnsw/data_store/graph_store.cppm b/src/storage/knn_index/knn_hnsw/data_store/graph_store.cppm index f3a6d92c8a..0c0c48a689 100644 --- a/src/storage/knn_index/knn_hnsw/data_store/graph_store.cppm +++ b/src/storage/knn_index/knn_hnsw/data_store/graph_store.cppm @@ -21,7 +21,7 @@ export module graph_store; import stl; import hnsw_common; -import file_system; +import local_file_handle; namespace infinity { @@ -68,24 +68,24 @@ public: SizeT GetSizeInBytes() const { return sizeof(Mmax0_) + sizeof(Mmax_) + sizeof(max_layer_) + sizeof(enterpoint_); } - void Save(FileHandler &file_handler) const { - file_handler.Write(&Mmax0_, sizeof(Mmax0_)); - file_handler.Write(&Mmax_, sizeof(Mmax_)); + void Save(LocalFileHandle &file_handle) const { + file_handle.Append(&Mmax0_, sizeof(Mmax0_)); + file_handle.Append(&Mmax_, sizeof(Mmax_)); - file_handler.Write(&max_layer_, sizeof(max_layer_)); - file_handler.Write(&enterpoint_, sizeof(enterpoint_)); + file_handle.Append(&max_layer_, sizeof(max_layer_)); + file_handle.Append(&enterpoint_, sizeof(enterpoint_)); } - static GraphStoreMeta Load(FileHandler &file_handler) { + static GraphStoreMeta Load(LocalFileHandle &file_handle) { SizeT Mmax0, Mmax; - file_handler.Read(&Mmax0, sizeof(Mmax0)); - file_handler.Read(&Mmax, sizeof(Mmax)); + file_handle.Read(&Mmax0, sizeof(Mmax0)); + file_handle.Read(&Mmax, sizeof(Mmax)); GraphStoreMeta meta(Mmax0, Mmax); i32 max_layer; VertexType enterpoint; - file_handler.Read(&max_layer, sizeof(max_layer)); - file_handler.Read(&enterpoint, sizeof(enterpoint)); + file_handle.Read(&max_layer, sizeof(max_layer)); + file_handle.Read(&enterpoint, sizeof(enterpoint)); meta.max_layer_ = max_layer; meta.enterpoint_ = enterpoint; return meta; @@ -167,36 +167,36 @@ public: return size; } - void Save(FileHandler &file_handler, SizeT cur_vertex_n, const GraphStoreMeta &meta) const { + void Save(LocalFileHandle &file_handle, SizeT cur_vertex_n, const GraphStoreMeta &meta) const { SizeT layer_sum = 0; for (VertexType vertex_i = 0; vertex_i < (VertexType)cur_vertex_n; ++vertex_i) { layer_sum += GetLevel0(vertex_i, meta)->layer_n_; } - file_handler.Write(&layer_sum, sizeof(layer_sum)); - file_handler.Write(graph_.get(), cur_vertex_n * meta.level0_size()); + file_handle.Append(&layer_sum, sizeof(layer_sum)); + file_handle.Append(graph_.get(), cur_vertex_n * meta.level0_size()); for (VertexType vertex_i = 0; vertex_i < (VertexType)cur_vertex_n; ++vertex_i) { const VertexL0 *v = GetLevel0(vertex_i, meta); if (v->layer_n_) { - file_handler.Write(v->layers_p_, meta.levelx_size() * v->layer_n_); + file_handle.Append(v->layers_p_, meta.levelx_size() * v->layer_n_); } } } - static GraphStoreInner Load(FileHandler &file_handler, SizeT cur_vertex_n, SizeT max_vertex, const GraphStoreMeta &meta, SizeT &mem_usage) { + static GraphStoreInner Load(LocalFileHandle &file_handle, SizeT cur_vertex_n, SizeT max_vertex, const GraphStoreMeta &meta, SizeT &mem_usage) { assert(cur_vertex_n <= max_vertex); SizeT layer_sum; - file_handler.Read(&layer_sum, sizeof(layer_sum)); + file_handle.Read(&layer_sum, sizeof(layer_sum)); GraphStoreInner graph_store(max_vertex, meta, cur_vertex_n); - file_handler.Read(graph_store.graph_.get(), cur_vertex_n * meta.level0_size()); + file_handle.Read(graph_store.graph_.get(), cur_vertex_n * meta.level0_size()); auto loaded_layers = MakeUnique(meta.levelx_size() * layer_sum); char *loaded_layers_p = loaded_layers.get(); for (VertexType vertex_i = 0; vertex_i < (VertexType)cur_vertex_n; ++vertex_i) { VertexL0 *v = graph_store.GetLevel0(vertex_i, meta); if (v->layer_n_) { - file_handler.Read(loaded_layers_p, meta.levelx_size() * v->layer_n_); + file_handle.Read(loaded_layers_p, meta.levelx_size() * v->layer_n_); v->layers_p_ = loaded_layers_p; loaded_layers_p += meta.levelx_size() * v->layer_n_; } else { diff --git a/src/storage/knn_index/knn_hnsw/data_store/lvq_vec_store.cppm b/src/storage/knn_index/knn_hnsw/data_store/lvq_vec_store.cppm index fa621639f1..54e0f6dd6b 100644 --- a/src/storage/knn_index/knn_hnsw/data_store/lvq_vec_store.cppm +++ b/src/storage/knn_index/knn_hnsw/data_store/lvq_vec_store.cppm @@ -26,7 +26,7 @@ module; export module lvq_vec_store; import stl; -import file_system; +import local_file_handle; import hnsw_common; namespace infinity { @@ -98,18 +98,18 @@ public: SizeT GetSizeInBytes() const { return sizeof(dim_) + sizeof(MeanType) * dim_ + sizeof(GlobalCacheType); } - void Save(FileHandler &file_handler) const { - file_handler.Write(&dim_, sizeof(dim_)); - file_handler.Write(mean_.get(), sizeof(MeanType) * dim_); - file_handler.Write(&global_cache_, sizeof(GlobalCacheType)); + void Save(LocalFileHandle &file_handle) const { + file_handle.Append(&dim_, sizeof(dim_)); + file_handle.Append(mean_.get(), sizeof(MeanType) * dim_); + file_handle.Append(&global_cache_, sizeof(GlobalCacheType)); } - static This Load(FileHandler &file_handler) { + static This Load(LocalFileHandle &file_handle) { SizeT dim; - file_handler.Read(&dim, sizeof(dim)); + file_handle.Read(&dim, sizeof(dim)); This meta(dim); - file_handler.Read(meta.mean_.get(), sizeof(MeanType) * dim); - file_handler.Read(&meta.global_cache_, sizeof(GlobalCacheType)); + file_handle.Read(meta.mean_.get(), sizeof(MeanType) * dim); + file_handle.Read(&meta.global_cache_, sizeof(GlobalCacheType)); return meta; } @@ -265,14 +265,14 @@ public: SizeT GetSizeInBytes(SizeT cur_vec_num, const Meta &meta) const { return cur_vec_num * meta.compress_data_size(); } - void Save(FileHandler &file_handler, SizeT cur_vec_num, const Meta &meta) const { - file_handler.Write(ptr_.get(), cur_vec_num * meta.compress_data_size()); + void Save(LocalFileHandle &file_handle, SizeT cur_vec_num, const Meta &meta) const { + file_handle.Append(ptr_.get(), cur_vec_num * meta.compress_data_size()); } - static This Load(FileHandler &file_handler, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { + static This Load(LocalFileHandle &file_handle, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { assert(cur_vec_num <= max_vec_num); This ret(max_vec_num, meta); - file_handler.Read(ret.ptr_.get(), cur_vec_num * meta.compress_data_size()); + file_handle.Read(ret.ptr_.get(), cur_vec_num * meta.compress_data_size()); mem_usage += max_vec_num * meta.compress_data_size(); return ret; } diff --git a/src/storage/knn_index/knn_hnsw/data_store/plain_vec_store.cppm b/src/storage/knn_index/knn_hnsw/data_store/plain_vec_store.cppm index e7d358b54f..e5d56b4724 100644 --- a/src/storage/knn_index/knn_hnsw/data_store/plain_vec_store.cppm +++ b/src/storage/knn_index/knn_hnsw/data_store/plain_vec_store.cppm @@ -25,7 +25,7 @@ module; export module plain_vec_store; import stl; -import file_system; +import local_file_handle; import hnsw_common; namespace infinity { @@ -56,11 +56,11 @@ public: SizeT GetSizeInBytes() const { return sizeof(SizeT); } - void Save(FileHandler &file_handler) const { file_handler.Write(&dim_, sizeof(dim_)); } + void Save(LocalFileHandle &file_handle) const { file_handle.Append(&dim_, sizeof(dim_)); } - static This Load(FileHandler &file_handler) { + static This Load(LocalFileHandle &file_handle) { SizeT dim; - file_handler.Read(&dim, sizeof(dim)); + file_handle.Read(&dim, sizeof(dim)); return This(dim); } @@ -94,14 +94,14 @@ public: SizeT GetSizeInBytes(SizeT cur_vec_num, const Meta &meta) const { return sizeof(DataType) * cur_vec_num * meta.dim(); } - void Save(FileHandler &file_handler, SizeT cur_vec_num, const Meta &meta) const { - file_handler.Write(ptr_.get(), sizeof(DataType) * cur_vec_num * meta.dim()); + void Save(LocalFileHandle &file_handle, SizeT cur_vec_num, const Meta &meta) const { + file_handle.Append(ptr_.get(), sizeof(DataType) * cur_vec_num * meta.dim()); } - static This Load(FileHandler &file_handler, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { + static This Load(LocalFileHandle &file_handle, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { assert(cur_vec_num <= max_vec_num); This ret(max_vec_num, meta); - file_handler.Read(ret.ptr_.get(), sizeof(DataType) * cur_vec_num * meta.dim()); + file_handle.Read(ret.ptr_.get(), sizeof(DataType) * cur_vec_num * meta.dim()); mem_usage += sizeof(DataType) * max_vec_num * meta.dim(); return ret; } diff --git a/src/storage/knn_index/knn_hnsw/data_store/sparse_vec_store.cppm b/src/storage/knn_index/knn_hnsw/data_store/sparse_vec_store.cppm index 54a06774f8..b55ed93293 100644 --- a/src/storage/knn_index/knn_hnsw/data_store/sparse_vec_store.cppm +++ b/src/storage/knn_index/knn_hnsw/data_store/sparse_vec_store.cppm @@ -24,7 +24,7 @@ module; export module sparse_vec_store; import stl; -import file_system; +import local_file_handle; import hnsw_common; import sparse_util; @@ -46,11 +46,11 @@ public: static This Make(SizeT max_dim) { return This(max_dim); } static This Make(SizeT max_dim, bool) { return This(max_dim); } - void Save(FileHandler &file_handler) const { file_handler.Write(&max_dim_, sizeof(max_dim_)); } + void Save(LocalFileHandle &file_handle) const { file_handle.Append(&max_dim_, sizeof(max_dim_)); } - static This Load(FileHandler &file_handler) { + static This Load(LocalFileHandle &file_handle) { SizeT max_dim; - file_handler.Read(&max_dim, sizeof(max_dim)); + file_handle.Read(&max_dim, sizeof(max_dim)); return This(max_dim); } @@ -85,12 +85,12 @@ public: return ret; } - void Save(FileHandler &file_handler, SizeT cur_vec_num, const Meta &meta) const { + void Save(LocalFileHandle &file_handle, SizeT cur_vec_num, const Meta &meta) const { SizeT nnz = 0; for (SizeT i = 0; i < cur_vec_num; ++i) { nnz += vecs_[i].nnz_; } - file_handler.Write(&nnz, sizeof(nnz)); + file_handle.Append(&nnz, sizeof(nnz)); auto indptr = MakeUniqueForOverwrite(cur_vec_num + 1); indptr[0] = 0; auto indice = MakeUniqueForOverwrite(nnz); @@ -101,20 +101,20 @@ public: Copy(vec.data_.get(), vec.data_.get() + vec.nnz_, data.get() + indptr[i]); indptr[i + 1] = indptr[i] + vec.nnz_; } - file_handler.Write(indptr.get(), sizeof(i32) * (cur_vec_num + 1)); - file_handler.Write(indice.get(), sizeof(IdxType) * nnz); - file_handler.Write(data.get(), sizeof(DataType) * nnz); + file_handle.Append(indptr.get(), sizeof(i32) * (cur_vec_num + 1)); + file_handle.Append(indice.get(), sizeof(IdxType) * nnz); + file_handle.Append(data.get(), sizeof(DataType) * nnz); } - static This Load(FileHandler &file_handler, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { + static This Load(LocalFileHandle &file_handle, SizeT cur_vec_num, SizeT max_vec_num, const Meta &meta, SizeT &mem_usage) { SizeT nnz = 0; - file_handler.Read(&nnz, sizeof(nnz)); + file_handle.Read(&nnz, sizeof(nnz)); auto indptr = MakeUniqueForOverwrite(cur_vec_num + 1); - file_handler.Read(indptr.get(), sizeof(i32) * (cur_vec_num + 1)); + file_handle.Read(indptr.get(), sizeof(i32) * (cur_vec_num + 1)); auto indice = MakeUniqueForOverwrite(nnz); - file_handler.Read(indice.get(), sizeof(IdxType) * nnz); + file_handle.Read(indice.get(), sizeof(IdxType) * nnz); auto data = MakeUniqueForOverwrite(nnz); - file_handler.Read(data.get(), sizeof(DataType) * nnz); + file_handle.Read(data.get(), sizeof(DataType) * nnz); This ret(max_vec_num, meta); mem_usage += sizeof(SparseVecEle) * max_vec_num; diff --git a/src/storage/knn_index/knn_hnsw/hnsw_alg.cppm b/src/storage/knn_index/knn_hnsw/hnsw_alg.cppm index 98c1aaf8be..6238170811 100644 --- a/src/storage/knn_index/knn_hnsw/hnsw_alg.cppm +++ b/src/storage/knn_index/knn_hnsw/hnsw_alg.cppm @@ -20,7 +20,7 @@ module; export module hnsw_alg; import stl; -import file_system; +import local_file_handle; import file_system_type; import infinity_exception; import knn_result_handler; @@ -102,19 +102,19 @@ public: SizeT GetSizeInBytes() const { return sizeof(M_) + sizeof(ef_construction_) + data_store_.GetSizeInBytes(); } - void Save(FileHandler &file_handler) { - file_handler.Write(&M_, sizeof(M_)); - file_handler.Write(&ef_construction_, sizeof(ef_construction_)); - data_store_.Save(file_handler); + void Save(LocalFileHandle &file_handle) { + file_handle.Append(&M_, sizeof(M_)); + file_handle.Append(&ef_construction_, sizeof(ef_construction_)); + data_store_.Save(file_handle); } - static UniquePtr Load(FileHandler &file_handler) { + static UniquePtr Load(LocalFileHandle &file_handle) { SizeT M; - file_handler.Read(&M, sizeof(M)); + file_handle.Read(&M, sizeof(M)); SizeT ef_construction; - file_handler.Read(&ef_construction, sizeof(ef_construction)); + file_handle.Read(&ef_construction, sizeof(ef_construction)); - auto data_store = DataStore::Load(file_handler); + auto data_store = DataStore::Load(file_handle); Distance distance(data_store.dim()); return MakeUnique(M, ef_construction, std::move(data_store), std::move(distance), 0); diff --git a/src/storage/knn_index/knn_ivf/ivf_index_data.cpp b/src/storage/knn_index/knn_ivf/ivf_index_data.cpp index 0f3f3a8043..d19a63598d 100644 --- a/src/storage/knn_index/knn_ivf/ivf_index_data.cpp +++ b/src/storage/knn_index/knn_ivf/ivf_index_data.cpp @@ -250,9 +250,9 @@ void IVFIndexInChunk::BuildIVFIndexT(const RowID base_rowid, } } -void IVFIndexInChunk::SaveIndexInner(FileHandler &file_handler) const { IVF_Index_Storage::Save(file_handler); } +void IVFIndexInChunk::SaveIndexInner(LocalFileHandle &file_handle) const { IVF_Index_Storage::Save(file_handle); } -void IVFIndexInChunk::ReadIndexInner(FileHandler &file_handler) { IVF_Index_Storage::Load(file_handler); } +void IVFIndexInChunk::ReadIndexInner(LocalFileHandle &file_handle) { IVF_Index_Storage::Load(file_handle); } IVFIndexInChunk *IVFIndexInChunk::GetNewIVFIndexInChunk(const IndexBase *ivf_index, const ColumnDef *column_def) { const auto *ivf_index_ptr = static_cast(ivf_index); diff --git a/src/storage/knn_index/knn_ivf/ivf_index_data.cppm b/src/storage/knn_index/knn_ivf/ivf_index_data.cppm index 3814fc981a..7713e97a37 100644 --- a/src/storage/knn_index/knn_ivf/ivf_index_data.cppm +++ b/src/storage/knn_index/knn_ivf/ivf_index_data.cppm @@ -23,6 +23,7 @@ import column_def; import embedding_info; import internal_types; import logical_type; +import local_file_handle; namespace infinity { @@ -45,9 +46,9 @@ public: const SharedPtr &column_def, BufferManager *buffer_mgr); - void SaveIndexInner(FileHandler &file_handler) const; + void SaveIndexInner(LocalFileHandle &file_handle) const; - void ReadIndexInner(FileHandler &file_handler); + void ReadIndexInner(LocalFileHandle &file_handle); static IVFIndexInChunk *GetNewIVFIndexInChunk(const IndexBase *ivf_index, const ColumnDef *column_def); diff --git a/src/storage/knn_index/knn_ivf/ivf_index_storage.cpp b/src/storage/knn_index/knn_ivf/ivf_index_storage.cpp index e878d41b9f..d9ab922483 100644 --- a/src/storage/knn_index/knn_ivf/ivf_index_storage.cpp +++ b/src/storage/knn_index/knn_ivf/ivf_index_storage.cpp @@ -41,38 +41,38 @@ IVF_Centroids_Storage::IVF_Centroids_Storage(const u32 embedding_dimension, cons assert(centroids_data_.size() == embedding_dimension_ * centroids_num_); } -void IVF_Centroids_Storage::Save(FileHandler &file_handler) const { - file_handler.Write(&embedding_dimension_, sizeof(embedding_dimension_)); - file_handler.Write(¢roids_num_, sizeof(centroids_num_)); +void IVF_Centroids_Storage::Save(LocalFileHandle &file_handle) const { + file_handle.Append(&embedding_dimension_, sizeof(embedding_dimension_)); + file_handle.Append(¢roids_num_, sizeof(centroids_num_)); const auto data_bytes = centroids_num_ * embedding_dimension_ * sizeof(f32); - file_handler.Write(centroids_data_.data(), data_bytes); + file_handle.Append(centroids_data_.data(), data_bytes); } -void IVF_Centroids_Storage::Load(FileHandler &file_handler) { - file_handler.Read(&embedding_dimension_, sizeof(embedding_dimension_)); - file_handler.Read(¢roids_num_, sizeof(centroids_num_)); +void IVF_Centroids_Storage::Load(LocalFileHandle &file_handle) { + file_handle.Read(&embedding_dimension_, sizeof(embedding_dimension_)); + file_handle.Read(¢roids_num_, sizeof(centroids_num_)); const auto vec_size = centroids_num_ * embedding_dimension_; centroids_data_.resize(vec_size); - file_handler.Read(centroids_data_.data(), vec_size * sizeof(f32)); + file_handle.Read(centroids_data_.data(), vec_size * sizeof(f32)); } // IVF_Part_Storage -void IVF_Part_Storage::Save(FileHandler &file_handler) const { - file_handler.Write(&part_id_, sizeof(part_id_)); - file_handler.Write(&embedding_dimension_, sizeof(embedding_dimension_)); - file_handler.Write(&embedding_num_, sizeof(embedding_num_)); +void IVF_Part_Storage::Save(LocalFileHandle &file_handle) const { + file_handle.Append(&part_id_, sizeof(part_id_)); + file_handle.Append(&embedding_dimension_, sizeof(embedding_dimension_)); + file_handle.Append(&embedding_num_, sizeof(embedding_num_)); static_assert(std::is_same_v); assert(embedding_num_ == embedding_segment_offsets_.size()); - file_handler.Write(embedding_segment_offsets_.data(), embedding_num_ * sizeof(SegmentOffset)); + file_handle.Append(embedding_segment_offsets_.data(), embedding_num_ * sizeof(SegmentOffset)); } -void IVF_Part_Storage::Load(FileHandler &file_handler) { - file_handler.Read(&part_id_, sizeof(part_id_)); - file_handler.Read(&embedding_dimension_, sizeof(embedding_dimension_)); - file_handler.Read(&embedding_num_, sizeof(embedding_num_)); +void IVF_Part_Storage::Load(LocalFileHandle &file_handle) { + file_handle.Read(&part_id_, sizeof(part_id_)); + file_handle.Read(&embedding_dimension_, sizeof(embedding_dimension_)); + file_handle.Read(&embedding_num_, sizeof(embedding_num_)); embedding_segment_offsets_.resize(embedding_num_); - file_handler.Read(embedding_segment_offsets_.data(), embedding_num_ * sizeof(SegmentOffset)); + file_handle.Read(embedding_segment_offsets_.data(), embedding_num_ * sizeof(SegmentOffset)); } template @@ -89,17 +89,17 @@ class IVF_Part_Storage_Plain final : public IVF_Part_Storage { public: IVF_Part_Storage_Plain(const u32 part_id, const u32 embedding_dimension) : IVF_Part_Storage(part_id, embedding_dimension) {} - void Save(FileHandler &file_handler) const override { - IVF_Part_Storage::Save(file_handler); + void Save(LocalFileHandle &file_handle) const override { + IVF_Part_Storage::Save(file_handle); const u32 element_cnt = embedding_num() * embedding_dimension(); assert(element_cnt == data_.size()); - file_handler.Write(data_.data(), element_cnt * sizeof(StorageDataT)); + file_handle.Append(data_.data(), element_cnt * sizeof(StorageDataT)); } - void Load(FileHandler &file_handler) override { - IVF_Part_Storage::Load(file_handler); + void Load(LocalFileHandle &file_handle) override { + IVF_Part_Storage::Load(file_handle); const u32 element_cnt = embedding_num() * embedding_dimension(); data_.resize(element_cnt); - file_handler.Read(data_.data(), element_cnt * sizeof(StorageDataT)); + file_handle.Read(data_.data(), element_cnt * sizeof(StorageDataT)); } void AppendOneEmbedding(const void *embedding_ptr, const SegmentOffset segment_offset, const IVF_Centroids_Storage *) override { @@ -227,24 +227,24 @@ IVF_Part_Storage::Make(u32 part_id, u32 embedding_dimension, EmbeddingDataType e // IVF_Index_Storage -void IVF_Index_Storage::Save(FileHandler &file_handler) const { - file_handler.Write(&row_count_, sizeof(row_count_)); - file_handler.Write(&embedding_count_, sizeof(embedding_count_)); - ivf_centroids_storage_.Save(file_handler); +void IVF_Index_Storage::Save(LocalFileHandle &file_handle) const { + file_handle.Append(&row_count_, sizeof(row_count_)); + file_handle.Append(&embedding_count_, sizeof(embedding_count_)); + ivf_centroids_storage_.Save(file_handle); assert(ivf_centroids_storage_.centroids_num() == ivf_part_storages_.size()); for (u32 part_id = 0; part_id < ivf_centroids_storage_.centroids_num(); ++part_id) { - ivf_part_storages_[part_id]->Save(file_handler); + ivf_part_storages_[part_id]->Save(file_handle); } } -void IVF_Index_Storage::Load(FileHandler &file_handler) { - file_handler.Read(&row_count_, sizeof(row_count_)); - file_handler.Read(&embedding_count_, sizeof(embedding_count_)); - ivf_centroids_storage_.Load(file_handler); +void IVF_Index_Storage::Load(LocalFileHandle &file_handle) { + file_handle.Read(&row_count_, sizeof(row_count_)); + file_handle.Read(&embedding_count_, sizeof(embedding_count_)); + ivf_centroids_storage_.Load(file_handle); ivf_part_storages_.resize(ivf_centroids_storage_.centroids_num()); for (u32 part_id = 0; part_id < ivf_centroids_storage_.centroids_num(); ++part_id) { ivf_part_storages_[part_id] = IVF_Part_Storage::Make(part_id, embedding_dimension_, embedding_data_type_, ivf_option_.storage_option_); - ivf_part_storages_[part_id]->Load(file_handler); + ivf_part_storages_[part_id]->Load(file_handle); } } diff --git a/src/storage/knn_index/knn_ivf/ivf_index_storage.cppm b/src/storage/knn_index/knn_ivf/ivf_index_storage.cppm index 7a2211e252..ee286b4994 100644 --- a/src/storage/knn_index/knn_ivf/ivf_index_storage.cppm +++ b/src/storage/knn_index/knn_ivf/ivf_index_storage.cppm @@ -24,6 +24,7 @@ import data_type; namespace infinity { +class LocalFileHandle; class FileHandler; template @@ -74,8 +75,8 @@ public: const f32 *data() const { return centroids_data_.data(); } u32 embedding_dimension() const { return embedding_dimension_; } u32 centroids_num() const { return centroids_num_; } - void Save(FileHandler &file_handler) const; - void Load(FileHandler &file_handler); + void Save(LocalFileHandle &file_handle) const; + void Load(LocalFileHandle &file_handle); }; class IVF_Part_Storage { @@ -96,8 +97,8 @@ public: u32 embedding_num() const { return embedding_num_; } SegmentOffset embedding_segment_offset(const u32 embedding_index) const { return embedding_segment_offsets_[embedding_index]; } - virtual void Save(FileHandler &file_handler) const; - virtual void Load(FileHandler &file_handler); + virtual void Save(LocalFileHandle &file_handle) const; + virtual void Load(LocalFileHandle &file_handle); virtual void AppendOneEmbedding(const void *embedding_ptr, SegmentOffset segment_offset, const IVF_Centroids_Storage *ivf_centroids_storage) = 0; @@ -134,8 +135,8 @@ public: void AddMultiVector(SegmentOffset segment_offset, const void *multi_vector_ptr, u32 embedding_num); void GetMemData(IVF_Index_Storage &&mem_data); - void Save(FileHandler &file_handler) const; - void Load(FileHandler &file_handler); + void Save(LocalFileHandle &file_handle) const; + void Load(LocalFileHandle &file_handle); private: template diff --git a/src/storage/knn_index/sparse/bmp_alg.cppm b/src/storage/knn_index/sparse/bmp_alg.cppm index 472e35b9c4..f98892a49a 100644 --- a/src/storage/knn_index/sparse/bmp_alg.cppm +++ b/src/storage/knn_index/sparse/bmp_alg.cppm @@ -18,7 +18,7 @@ export module bmp_alg; import stl; import sparse_util; -import file_system; +import local_file_handle; import bm_posting; import bmp_util; import hnsw_common; @@ -145,9 +145,9 @@ public: Pair, Vector> SearchKnn(const SparseVecRef &query, i32 topk, const BmpSearchOptions &options, const Filter &filter) const; - void Save(FileHandler &file_handler) const; + void Save(LocalFileHandle &file_handle) const; - static BMPAlg Load(FileHandler &file_handler); + static BMPAlg Load(LocalFileHandle &file_handle); SizeT GetSizeInBytes() const; diff --git a/src/storage/knn_index/sparse/bmp_alg_serialize.cpp b/src/storage/knn_index/sparse/bmp_alg_serialize.cpp index e260e4266d..e598f6e33f 100644 --- a/src/storage/knn_index/sparse/bmp_alg_serialize.cpp +++ b/src/storage/knn_index/sparse/bmp_alg_serialize.cpp @@ -158,7 +158,7 @@ template class BlockFwd; // --------------------------BMPAlg-------------------------- template -void BMPAlg::Save(FileHandler &file_handler) const { +void BMPAlg::Save(LocalFileHandle &file_handle) const { auto size = GetSizeInBytes(); auto buffer = MakeUnique(sizeof(size) + size); char *p = buffer.get(); @@ -167,15 +167,15 @@ void BMPAlg::Save(FileHandler &file_handler) co if (SizeT write_n = p - buffer.get(); write_n != sizeof(size) + size) { UnrecoverableError(fmt::format("BMPAlg::Save: write_n != sizeof(size) + size: {} != {}", write_n, sizeof(size) + size)); } - file_handler.Write(buffer.get(), sizeof(size) + size); + file_handle.Append(buffer.get(), sizeof(size) + size); } template -BMPAlg BMPAlg::Load(FileHandler &file_handler) { +BMPAlg BMPAlg::Load(LocalFileHandle &file_handle) { SizeT size; - file_handler.Read(&size, sizeof(size)); + file_handle.Read(&size, sizeof(size)); auto buffer = MakeUnique(size); - file_handler.Read(buffer.get(), size); + file_handle.Read(buffer.get(), size); const char *p = buffer.get(); return ReadAdv(p); } diff --git a/src/storage/knn_index/sparse/linscan_alg.cppm b/src/storage/knn_index/sparse/linscan_alg.cppm index 944fd9718d..9e4bd0f919 100644 --- a/src/storage/knn_index/sparse/linscan_alg.cppm +++ b/src/storage/knn_index/sparse/linscan_alg.cppm @@ -21,7 +21,7 @@ export module linscan_alg; import stl; import sparse_util; -import file_system; +import local_file_handle; import knn_result_handler; import serialize; import infinity_exception; @@ -134,20 +134,20 @@ public: u32 row_num() const { return row_num_; } - void Save(FileHandler &file_handler) const { + void Save(LocalFileHandle &file_handle) const { SizeT bytes = GetSizeInBytes(); auto buffer = MakeUnique(sizeof(bytes) + bytes); char *p = buffer.get(); WriteBufAdv(p, bytes); WriteAdv(p); - file_handler.Write(buffer.get(), sizeof(bytes) + bytes); + file_handle.Append(buffer.get(), sizeof(bytes) + bytes); } - static LinScan Load(FileHandler &file_handler) { + static LinScan Load(LocalFileHandle &file_handle) { SizeT bytes; - file_handler.Read(&bytes, sizeof(bytes)); + file_handle.Read(&bytes, sizeof(bytes)); auto buffer = MakeUnique(bytes); - file_handler.Read(buffer.get(), bytes); + file_handle.Read(buffer.get(), bytes); const char *buffer_ptr = buffer.get(); return ReadAdv(buffer_ptr); } diff --git a/src/storage/knn_index/sparse/sparse_util.cppm b/src/storage/knn_index/sparse/sparse_util.cppm index 339d36cd2b..15ac202241 100644 --- a/src/storage/knn_index/sparse/sparse_util.cppm +++ b/src/storage/knn_index/sparse/sparse_util.cppm @@ -22,9 +22,9 @@ export module sparse_util; import stl; import sparse_vector_distance; import knn_result_handler; -import file_system; import infinity_exception; import third_party; +import local_file_handle; namespace infinity { @@ -79,22 +79,22 @@ public: return SparseVecRef(nnz, indices, data); } - static SparseMatrix Load(FileHandler &file_handler) { + static SparseMatrix Load(LocalFileHandle &file_handle) { i64 nrow = 0; i64 ncol = 0; i64 nnz = 0; - file_handler.Read(&nrow, sizeof(nrow)); - file_handler.Read(&ncol, sizeof(ncol)); - file_handler.Read(&nnz, sizeof(nnz)); + file_handle.Read(&nrow, sizeof(nrow)); + file_handle.Read(&ncol, sizeof(ncol)); + file_handle.Read(&nnz, sizeof(nnz)); auto indptr = MakeUnique(nrow + 1); - file_handler.Read(indptr.get(), sizeof(i64) * (nrow + 1)); + file_handle.Read(indptr.get(), sizeof(i64) * (nrow + 1)); if (indptr[nrow] != nnz) { UnrecoverableError("Invalid indptr."); } auto indices = MakeUnique(nnz); - file_handler.Read(indices.get(), sizeof(IdxT) * nnz); + file_handle.Read(indices.get(), sizeof(IdxT) * nnz); // assert all element in indices >= 0 and < ncol { bool check = std::all_of(indices.get(), indices.get() + nnz, [ncol](IdxT ele) { return ele >= 0 && ele < ncol; }); @@ -104,17 +104,17 @@ public: } auto data = MakeUnique(nnz); - file_handler.Read(data.get(), sizeof(DataT) * nnz); + file_handle.Read(data.get(), sizeof(DataT) * nnz); return {std::move(data), std::move(indices), std::move(indptr), nrow, ncol, nnz}; } - void Save(FileHandler &file_handler) const { - file_handler.Write(&nrow_, sizeof(nrow_)); - file_handler.Write(&ncol_, sizeof(ncol_)); - file_handler.Write(&nnz_, sizeof(nnz_)); - file_handler.Write(indptr_.get(), sizeof(i64) * (nrow_ + 1)); - file_handler.Write(indices_.get(), sizeof(IdxT) * nnz_); - file_handler.Write(data_.get(), sizeof(DataT) * nnz_); + void Save(LocalFileHandle &file_handle) const { + file_handle.Append(&nrow_, sizeof(nrow_)); + file_handle.Append(&ncol_, sizeof(ncol_)); + file_handle.Append(&nnz_, sizeof(nnz_)); + file_handle.Append(indptr_.get(), sizeof(i64) * (nrow_ + 1)); + file_handle.Append(indices_.get(), sizeof(IdxT) * nnz_); + file_handle.Append(data_.get(), sizeof(DataT) * nnz_); } void Clear() { diff --git a/src/storage/meta/entry/block_version.cpp b/src/storage/meta/entry/block_version.cpp index 181dc8d750..ca858a5677 100644 --- a/src/storage/meta/entry/block_version.cpp +++ b/src/storage/meta/entry/block_version.cpp @@ -26,26 +26,26 @@ import default_values; import column_vector; import serialize; import local_file_system; -import abstract_file_handle; +import local_file_handle; import status; namespace infinity { // deprecated -void CreateField::SaveToFile(FileHandler &file_handler) const { - file_handler.Write(&create_ts_, sizeof(create_ts_)); - file_handler.Write(&row_count_, sizeof(row_count_)); +void CreateField::SaveToFile(FileHandler &file_handle) const { + file_handle.Write(&create_ts_, sizeof(create_ts_)); + file_handle.Write(&row_count_, sizeof(row_count_)); } // deprecated -CreateField CreateField::LoadFromFile(FileHandler &file_handler) { +CreateField CreateField::LoadFromFile(FileHandler &file_handle) { CreateField create_field; - file_handler.Read(&create_field.create_ts_, sizeof(create_field.create_ts_)); - file_handler.Read(&create_field.row_count_, sizeof(create_field.row_count_)); + file_handle.Read(&create_field.create_ts_, sizeof(create_field.create_ts_)); + file_handle.Read(&create_field.row_count_, sizeof(create_field.row_count_)); return create_field; } -void CreateField::SaveToFile(AbstractFileHandle *file_handle) const { +void CreateField::SaveToFile(LocalFileHandle *file_handle) const { Status status = file_handle->Append((char*)(&create_ts_), sizeof(create_ts_)); if(!status.ok()) { UnrecoverableError(status.message()); @@ -56,7 +56,7 @@ void CreateField::SaveToFile(AbstractFileHandle *file_handle) const { } } -CreateField CreateField::LoadFromFile(AbstractFileHandle *file_handle) { +CreateField CreateField::LoadFromFile(LocalFileHandle *file_handle) { CreateField create_field; auto [size1, status1] = file_handle->Read(&create_field.create_ts_, sizeof(create_field.create_ts_)); if(!status1.ok()) { @@ -94,27 +94,27 @@ i32 BlockVersion::GetRowCount(TxnTimeStamp begin_ts) const { return iter->row_count_; } -void BlockVersion::SaveToFile(TxnTimeStamp checkpoint_ts, FileHandler &file_handler) const { +void BlockVersion::SaveToFile(TxnTimeStamp checkpoint_ts, LocalFileHandle &file_handle) const { BlockOffset create_size = created_.size(); while (create_size > 0 && created_[create_size - 1].create_ts_ > checkpoint_ts) { --create_size; } - file_handler.Write(&create_size, sizeof(create_size)); + file_handle.Append(&create_size, sizeof(create_size)); for (SizeT j = 0; j < create_size; ++j) { - created_[j].SaveToFile(file_handler); + created_[j].SaveToFile(&file_handle); } BlockOffset capacity = deleted_.size(); - file_handler.Write(&capacity, sizeof(capacity)); + file_handle.Append(&capacity, sizeof(capacity)); TxnTimeStamp dump_ts = 0; u32 deleted_row_count = 0; for (const auto &ts : deleted_) { if (ts <= checkpoint_ts) { - file_handler.Write(&ts, sizeof(ts)); + file_handle.Append(&ts, sizeof(ts)); } else { ++ deleted_row_count; - file_handler.Write(&dump_ts, sizeof(dump_ts)); + file_handle.Append(&dump_ts, sizeof(dump_ts)); } } LOG_TRACE(fmt::format("Flush block version, ckp ts: {}, write create: {}, 0 delete {}", checkpoint_ts, create_size, deleted_row_count)); @@ -132,7 +132,7 @@ void BlockVersion::SpillToFile(FileHandler &file_handler) const { file_handler.Write(deleted_.data(), capacity * sizeof(TxnTimeStamp)); } -void BlockVersion::SpillToFile(AbstractFileHandle *file_handle) const { +void BlockVersion::SpillToFile(LocalFileHandle *file_handle) const { BlockOffset create_size = created_.size(); Status status = file_handle->Append(&create_size, sizeof(create_size)); if(!status.ok()) { @@ -173,7 +173,7 @@ UniquePtr BlockVersion::LoadFromFile(FileHandler &file_handler) { } -UniquePtr BlockVersion::LoadFromFile(AbstractFileHandle *file_handle) { +UniquePtr BlockVersion::LoadFromFile(LocalFileHandle *file_handle) { auto block_version = MakeUnique(); BlockOffset create_size; diff --git a/src/storage/meta/entry/block_version.cppm b/src/storage/meta/entry/block_version.cppm index 6c54f44e93..a2bc52bdb5 100644 --- a/src/storage/meta/entry/block_version.cppm +++ b/src/storage/meta/entry/block_version.cppm @@ -18,7 +18,7 @@ export module block_version; import stl; import file_system; -import abstract_file_handle; +import local_file_handle; namespace infinity { @@ -38,8 +38,8 @@ struct CreateField { void SaveToFile(FileHandler &file_handler) const; static CreateField LoadFromFile(FileHandler &file_handler); - void SaveToFile(AbstractFileHandle *file_handle) const; - static CreateField LoadFromFile(AbstractFileHandle *file_handle); + void SaveToFile(LocalFileHandle *file_handle) const; + static CreateField LoadFromFile(LocalFileHandle *file_handle); }; export struct BlockVersion { @@ -55,14 +55,14 @@ export struct BlockVersion { i32 GetRowCount(TxnTimeStamp begin_ts) const; - void SaveToFile(TxnTimeStamp checkpoint_ts, FileHandler &file_handler) const; + void SaveToFile(TxnTimeStamp checkpoint_ts, LocalFileHandle &file_handler) const; // deprecated void SpillToFile(FileHandler &file_handler) const; static UniquePtr LoadFromFile(FileHandler &file_handler); - void SpillToFile(AbstractFileHandle *file_handle) const; - static UniquePtr LoadFromFile(AbstractFileHandle *file_handle); + void SpillToFile(LocalFileHandle *file_handle) const; + static UniquePtr LoadFromFile(LocalFileHandle *file_handle); void GetCreateTS(SizeT offset, SizeT size, ColumnVector &res) const; diff --git a/src/storage/meta/table_index_meta.cpp b/src/storage/meta/table_index_meta.cpp index da3c91edbf..20ebb52032 100644 --- a/src/storage/meta/table_index_meta.cpp +++ b/src/storage/meta/table_index_meta.cpp @@ -30,7 +30,6 @@ import table_entry; import infinity_exception; import status; import extra_ddl_info; -import local_file_system; import txn; import create_index_info; import base_entry; diff --git a/src/storage/secondary_index/secondary_index_data.cpp b/src/storage/secondary_index/secondary_index_data.cpp index 66ede9db71..a76a78f978 100644 --- a/src/storage/secondary_index/secondary_index_data.cpp +++ b/src/storage/secondary_index/secondary_index_data.cpp @@ -23,7 +23,7 @@ import stl; import default_values; import index_base; -import file_system; +import local_file_handle; import file_system_type; import infinity_exception; import third_party; @@ -126,15 +126,15 @@ class SecondaryIndexDataT final : public SecondaryIndexData { } } - void SaveIndexInner(FileHandler &file_handler) const override { + void SaveIndexInner(LocalFileHandle &file_handle) const override { if (!need_save_) { String error_message = "SaveIndexInner(): error: SecondaryIndexDataT is not allocated."; UnrecoverableError(error_message); } - pgm_index_->SaveIndex(file_handler); + pgm_index_->SaveIndex(file_handle); } - void ReadIndexInner(FileHandler &file_handler) override { pgm_index_->LoadIndex(file_handler); } + void ReadIndexInner(LocalFileHandle &file_handle) override { pgm_index_->LoadIndex(file_handle); } void InsertData(const void *ptr, SharedPtr &chunk_index) override { if (!need_save_) { diff --git a/src/storage/secondary_index/secondary_index_data.cppm b/src/storage/secondary_index/secondary_index_data.cppm index 812ec86a4d..3b70d59a55 100644 --- a/src/storage/secondary_index/secondary_index_data.cppm +++ b/src/storage/secondary_index/secondary_index_data.cppm @@ -18,7 +18,7 @@ export module secondary_index_data; import stl; import default_values; -import file_system; +import local_file_handle; import file_system_type; import infinity_exception; import column_vector; @@ -161,9 +161,9 @@ public: [[nodiscard]] inline u32 GetChunkRowCount() const { return chunk_row_count_; } - virtual void SaveIndexInner(FileHandler &file_handler) const = 0; + virtual void SaveIndexInner(LocalFileHandle &file_handle) const = 0; - virtual void ReadIndexInner(FileHandler &file_handler) = 0; + virtual void ReadIndexInner(LocalFileHandle &file_handle) = 0; virtual void InsertData(const void *ptr, SharedPtr &chunk_index) = 0; diff --git a/src/storage/secondary_index/secondary_index_pgm.cppm b/src/storage/secondary_index/secondary_index_pgm.cppm index 91d73e8bbd..1f004b8300 100644 --- a/src/storage/secondary_index/secondary_index_pgm.cppm +++ b/src/storage/secondary_index/secondary_index_pgm.cppm @@ -19,7 +19,7 @@ export module secondary_index_pgm; import stl; import logger; import third_party; -import file_system; +import local_file_handle; import infinity_exception; namespace infinity { @@ -69,57 +69,57 @@ public: template PGMWithExtraFunction(RandomIt first, RandomIt last) : PGMIndexType(first, last) {} - inline void Load(FileHandler &file_handler) { + inline void Load(LocalFileHandle &file_handle) { { // load n u32 save_n; - file_handler.Read(&save_n, sizeof(save_n)); + file_handle.Read(&save_n, sizeof(save_n)); this->n = save_n; } { // load first_key IndexValueType save_first_key; - file_handler.Read(&save_first_key, sizeof(save_first_key)); + file_handle.Read(&save_first_key, sizeof(save_first_key)); this->first_key = save_first_key; } { // load std::vector segments u32 save_size; - file_handler.Read(&save_size, sizeof(save_size)); + file_handle.Read(&save_size, sizeof(save_size)); this->segments.resize(save_size); - file_handler.Read(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); + file_handle.Read(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); } { // load std::vector levels_offsets u32 save_levels_offsets_size; - file_handler.Read(&save_levels_offsets_size, sizeof(save_levels_offsets_size)); + file_handle.Read(&save_levels_offsets_size, sizeof(save_levels_offsets_size)); this->levels_offsets.resize(save_levels_offsets_size); - file_handler.Read(this->levels_offsets.data(), save_levels_offsets_size * sizeof(typename decltype(this->levels_offsets)::value_type)); + file_handle.Read(this->levels_offsets.data(), save_levels_offsets_size * sizeof(typename decltype(this->levels_offsets)::value_type)); } } - inline void Save(FileHandler &file_handler) const { + inline void Save(LocalFileHandle &file_handle) const { { // save n u32 save_n = this->n; - file_handler.Write(&save_n, sizeof(save_n)); + file_handle.Append(&save_n, sizeof(save_n)); } { // save first_key IndexValueType save_first_key = this->first_key; - file_handler.Write(&save_first_key, sizeof(save_first_key)); + file_handle.Append(&save_first_key, sizeof(save_first_key)); } { // save std::vector segments u32 save_size = this->segments.size(); - file_handler.Write(&save_size, sizeof(save_size)); - file_handler.Write(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); + file_handle.Append(&save_size, sizeof(save_size)); + file_handle.Append(this->segments.data(), save_size * sizeof(typename decltype(this->segments)::value_type)); } { // save std::vector levels_offsets u32 save_levels_offsets_size = this->levels_offsets.size(); - file_handler.Write(&save_levels_offsets_size, sizeof(save_levels_offsets_size)); - file_handler.Write(this->levels_offsets.data(), save_levels_offsets_size * sizeof(typename decltype(this->levels_offsets)::value_type)); + file_handle.Append(&save_levels_offsets_size, sizeof(save_levels_offsets_size)); + file_handle.Append(this->levels_offsets.data(), save_levels_offsets_size * sizeof(typename decltype(this->levels_offsets)::value_type)); } } }; @@ -128,9 +128,9 @@ export class SecondaryPGMIndex { public: virtual ~SecondaryPGMIndex() = default; - virtual void SaveIndex(FileHandler &file_handler) const = 0; + virtual void SaveIndex(LocalFileHandle &file_handle) const = 0; - virtual void LoadIndex(FileHandler &file_handler) = 0; + virtual void LoadIndex(LocalFileHandle &file_handle) = 0; virtual void BuildIndex(SizeT data_cnt, const void *data_ptr) = 0; @@ -147,21 +147,21 @@ public: ~SecondaryPGMIndexTemplate() final = default; - void SaveIndex(FileHandler &file_handler) const final { + void SaveIndex(LocalFileHandle &file_handle) const final { if (!initialized_) { String error_message = "Not initialized yet."; UnrecoverableError(error_message); } - pgm_index_->Save(file_handler); + pgm_index_->Save(file_handle); } - void LoadIndex(FileHandler &file_handler) final { + void LoadIndex(LocalFileHandle &file_handle) final { if (initialized_) { String error_message = "Already initialized."; UnrecoverableError(error_message); } pgm_index_ = MakeUnique>(); - pgm_index_->Load(file_handler); + pgm_index_->Load(file_handle); initialized_ = true; } diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 1814b8dbac..2bd1996b78 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -29,7 +29,6 @@ import wal_manager; import catalog; import txn_manager; import builtin_functions; -import local_file_system; import third_party; import logger; @@ -50,7 +49,7 @@ import memindex_tracer; import cleanup_scanner; import persistence_manager; import extra_ddl_info; -import virtual_storage; +import virtual_store; import virtual_storage_type; namespace infinity { @@ -99,30 +98,24 @@ void Storage::SetStorageMode(StorageMode target_mode) { LOG_INFO(fmt::format("Set storage from admin mode to un-init")); break; } - - // Construct virtual storage system; - if (virtual_storage_system_ != nullptr) { - UnrecoverableError("Virtual storage system was initialized before."); - } - virtual_storage_system_ = MakeUnique(); - - switch (config_ptr_->StorageType()) { + switch(config_ptr_->StorageType()) { case StorageType::kLocal: { - Map configs; - virtual_storage_system_->Init(StorageType::kLocal, configs); + // Not init remote store break; } case StorageType::kMinio: { + if (remote_store_ != nullptr) { + UnrecoverableError("remote storage system was initialized before."); + } Map configs; configs.emplace("url", config_ptr_->ObjectStorageUrl()); - virtual_storage_system_->Init(StorageType::kMinio, configs); + remote_store_->Init(StorageType::kMinio, configs); break; } default: { UnrecoverableError(fmt::format("Unsupported storage type: {}.", ToString(config_ptr_->StorageType()))); } } - // Construct persistence store String persistence_dir = config_ptr_->PersistenceDir(); if (!persistence_dir.empty()) { @@ -257,6 +250,21 @@ void Storage::SetStorageMode(StorageMode target_mode) { memory_index_tracer_.reset(); + switch(config_ptr_->StorageType()) { + case StorageType::kLocal: { + // Not init remote store + break; + } + case StorageType::kMinio: { + remote_store_->UnInit(); + remote_store_.reset(); + break; + } + default: { + UnrecoverableError(fmt::format("Unsupported storage type: {}.", ToString(config_ptr_->StorageType()))); + } + } + wal_mgr_->Stop(); wal_mgr_.reset(); if (target_mode == StorageMode::kAdmin) { @@ -327,6 +335,21 @@ void Storage::SetStorageMode(StorageMode target_mode) { memory_index_tracer_.reset(); + switch(config_ptr_->StorageType()) { + case StorageType::kLocal: { + // Not init remote store + break; + } + case StorageType::kMinio: { + remote_store_->UnInit(); + remote_store_.reset(); + break; + } + default: { + UnrecoverableError(fmt::format("Unsupported storage type: {}.", ToString(config_ptr_->StorageType()))); + } + } + wal_mgr_->Stop(); wal_mgr_.reset(); if (target_mode == StorageMode::kAdmin) { diff --git a/src/storage/storage.cppm b/src/storage/storage.cppm index 453cd0d26c..b42127a875 100644 --- a/src/storage/storage.cppm +++ b/src/storage/storage.cppm @@ -26,7 +26,7 @@ import periodic_trigger_thread; import log_file; import memindex_tracer; import persistence_manager; -import virtual_storage; +import virtual_store; export module storage; @@ -50,7 +50,7 @@ public: [[nodiscard]] inline PersistenceManager *persistence_manager() noexcept { return persistence_manager_.get(); } - [[nodiscard]] inline VirtualStorage *virtual_storage() noexcept { return virtual_storage_system_.get(); } + [[nodiscard]] inline RemoteStore *remote_store() noexcept { return remote_store_.get(); } [[nodiscard]] inline BGTaskProcessor *bg_processor() const noexcept { return bg_processor_.get(); } @@ -76,7 +76,7 @@ private: UniquePtr txn_mgr_{}; UniquePtr wal_mgr_{}; UniquePtr persistence_manager_{}; - UniquePtr virtual_storage_system_{}; + UniquePtr remote_store_{}; UniquePtr bg_processor_{}; UniquePtr compact_processor_{}; UniquePtr periodic_trigger_thread_{}; diff --git a/src/storage/wal/log_file.cpp b/src/storage/wal/log_file.cpp index c6ea948016..44b423b0e7 100644 --- a/src/storage/wal/log_file.cpp +++ b/src/storage/wal/log_file.cpp @@ -19,14 +19,13 @@ module; module log_file; import stl; -import virtual_storage; +import virtual_store; import third_party; import infinity_exception; import logger; import default_values; import infinity_context; import status; -import local_file_system; namespace infinity { @@ -72,8 +71,7 @@ void CatalogFile::RecycleCatalogFile(TxnTimeStamp max_commit_ts, const String &c bool found = false; for (const auto &full_info : full_infos) { if (full_info.max_commit_ts_ < max_commit_ts) { - LocalFileSystem fs; - fs.DeleteFile(full_info.path_); + LocalStore::DeleteFile(full_info.path_); LOG_DEBUG(fmt::format("WalManager::Checkpoint delete catalog file: {}", full_info.path_)); } else if (full_info.max_commit_ts_ == max_commit_ts) { found = true; @@ -85,16 +83,19 @@ void CatalogFile::RecycleCatalogFile(TxnTimeStamp max_commit_ts, const String &c } for (const auto &delta_info : delta_infos) { if (delta_info.max_commit_ts_ <= max_commit_ts) { - LocalFileSystem fs; - fs.DeleteFile(delta_info.path_); + LocalStore::DeleteFile(delta_info.path_); LOG_DEBUG(fmt::format("WalManager::Checkpoint delete catalog file: {}", delta_info.path_)); } } } Pair, Vector> CatalogFile::ParseCheckpointFilenames(const String &catalog_dir) { - LocalFileSystem fs; - const auto &entries = fs.ListDirectory(Path(InfinityContext::instance().config()->DataDir()) / catalog_dir); + auto [entries, status] = LocalStore::ListDirectory(Path(InfinityContext::instance().config()->DataDir()) / catalog_dir); + if(!status.ok()) { + String error_message = fmt::format("Can't list directory: {}/{}", InfinityContext::instance().config()->DataDir(), catalog_dir); + UnrecoverableError(error_message); + } + if (entries.empty()) { return {Vector{}, Vector{}}; } @@ -153,11 +154,11 @@ Pair, Vector> CatalogFile::Par Pair, Vector> WalFile::ParseWalFilenames(const String &wal_dir) { - if (!VirtualStorage::ExistsLocal(wal_dir)) { + if (!LocalStore::Exists(wal_dir)) { return {None, Vector{}}; } - auto [entries, status] = VirtualStorage::ListDirectoryLocal(wal_dir); + auto [entries, status] = LocalStore::ListDirectory(wal_dir); if(!status.ok()) { LOG_CRITICAL(status.message()); UnrecoverableError(status.message()); @@ -222,7 +223,7 @@ void WalFile::RecycleWalFile(TxnTimeStamp max_commit_ts, const String &wal_dir) auto [cur_wal_info, wal_infos] = ParseWalFilenames(wal_dir); for (const auto &wal_info : wal_infos) { if (wal_info.max_commit_ts_ <= max_commit_ts) { - VirtualStorage::DeleteFileLocal(wal_info.path_); + LocalStore::DeleteFile(wal_info.path_); LOG_INFO(fmt::format("WalManager::Checkpoint delete wal file: {}", wal_info.path_)); } } diff --git a/src/storage/wal/wal_entry.cpp b/src/storage/wal/wal_entry.cpp index bb6f75d1db..f269a4932b 100644 --- a/src/storage/wal/wal_entry.cpp +++ b/src/storage/wal/wal_entry.cpp @@ -37,11 +37,11 @@ import segment_entry; import segment_index_entry; import chunk_index_entry; import block_version; -import local_file_system; import index_defines; import create_index_info; import persistence_manager; import infinity_context; +import virtual_store; namespace infinity { @@ -1551,9 +1551,8 @@ void WalListIterator::PurgeBadEntriesAfterLatestCheckpoint() { if (bad_offset != i64(-1)) { String error_message = fmt::format("Found bad wal entry {}@{}", *it, bad_offset); LOG_WARN(error_message); - LocalFileSystem fs; if (bad_offset == 0) { - fs.DeleteFile(*it); + LocalStore::DeleteFile(*it); error_message = fmt::format("Removed wal log {}", *it); LOG_WARN(error_message); ++it; @@ -1562,7 +1561,7 @@ void WalListIterator::PurgeBadEntriesAfterLatestCheckpoint() { } file_num = 0; } else { - fs.Truncate(*it, bad_offset); + LocalStore::Truncate(*it, bad_offset); error_message = fmt::format("Truncated {}@{}", *it, bad_offset); LOG_WARN(error_message); ++it; diff --git a/src/storage/wal/wal_manager.cpp b/src/storage/wal/wal_manager.cpp index 9c7e16e909..9e8f127ed2 100644 --- a/src/storage/wal/wal_manager.cpp +++ b/src/storage/wal/wal_manager.cpp @@ -23,7 +23,7 @@ import logger; import txn_manager; import txn; import storage; -import local_file_system; +import virtual_store; import third_party; import catalog; import table_entry_type; @@ -96,9 +96,8 @@ void WalManager::Start() { bool changed = running_.compare_exchange_strong(expected, true); if (!changed) return; - LocalFileSystem fs; - if (!fs.Exists(wal_dir_)) { - fs.CreateDirectory(wal_dir_); + if (!LocalStore::Exists(wal_dir_)) { + LocalStore::MakeDirectory(wal_dir_); } // TODO: recovery from wal checkpoint ofs_ = std::ofstream(wal_path_, std::ios::app | std::ios::binary); @@ -153,7 +152,6 @@ TxnTimeStamp WalManager::GetCheckpointedTS() { return last_ckp_ts_ == UNCOMMIT_T Vector> WalManager::GetDiffWalEntryString(TxnTimeStamp start_timestamp) const { - LocalFileSystem fs; Vector> log_strings; Vector wal_list{}; @@ -335,8 +333,7 @@ void WalManager::Flush() { // Check if the wal file is too large, swap to a new one. try { - LocalFileSystem fs; - auto file_size = fs.GetFileSizeByPath(wal_path_); + auto file_size = LocalStore::GetFileSize(wal_path_); if (file_size > cfg_wal_size_threshold_) { this->SwapWalFile(max_commit_ts_); } @@ -527,8 +524,7 @@ void WalManager::SwapWalFile(const TxnTimeStamp max_commit_ts) { LOG_INFO(fmt::format("Wal {} swap to new path: {}", wal_path_, new_file_path)); // Rename the current wal file to a new one. - LocalFileSystem fs; - fs.Rename(wal_path_, new_file_path); + LocalStore::Rename(wal_path_, new_file_path); // Create a new wal file with the original name. ofs_ = std::ofstream(wal_path_, std::ios::app | std::ios::binary); @@ -581,8 +577,6 @@ String WalManager::GetWalFilename() const { return wal_path_; } * */ i64 WalManager::ReplayWalFile(StorageMode targe_storage_mode) { - LocalFileSystem fs; - Vector wal_list{}; { auto [temp_wal_info, wal_infos] = WalFile::ParseWalFilenames(wal_dir_); @@ -715,8 +709,6 @@ i64 WalManager::ReplayWalFile(StorageMode targe_storage_mode) { } Optional>> WalManager::GetCatalogFiles() const { - LocalFileSystem fs; - Vector wal_list{}; { auto [temp_wal_info, wal_infos] = WalFile::ParseWalFilenames(wal_dir_); @@ -777,7 +769,6 @@ Optional>> WalManager::Ge } Vector> WalManager::CollectWalEntries() const { - LocalFileSystem fs; Vector> wal_entries; Vector wal_list{}; diff --git a/src/unit_test/main/config.cpp b/src/unit_test/main/config.cpp index 790d98fffb..8da253b546 100644 --- a/src/unit_test/main/config.cpp +++ b/src/unit_test/main/config.cpp @@ -100,7 +100,7 @@ TEST_F(ConfigTest, test2) { // Storage EXPECT_EQ(config.DataDir(), "/var/infinity/data"); EXPECT_EQ(config.WALDir(), "/var/infinity/wal"); - EXPECT_EQ(config.StorageType(), StorageType::kMinio); + EXPECT_EQ(config.StorageType(), StorageType::kLocal); EXPECT_EQ(config.ObjectStorageUrl(), "0.0.0.0:9000"); EXPECT_EQ(config.ObjectStorageBucket(), "infinity"); diff --git a/src/unit_test/storage/io/file_handle/local_file.cpp b/src/unit_test/storage/io/file_handle/local_file.cpp index 0ec3c589b9..90bcf1c608 100644 --- a/src/unit_test/storage/io/file_handle/local_file.cpp +++ b/src/unit_test/storage/io/file_handle/local_file.cpp @@ -16,9 +16,9 @@ import base_test; import stl; -import virtual_storage; +import virtual_store; import virtual_storage_type; -import local_file; +import local_file_handle; import abstract_file_handle; using namespace infinity; @@ -29,14 +29,7 @@ TEST_F(LocalFileTest, TestAppend) { using namespace infinity; String path = String(GetFullTmpDir()) + "/test_file2.abc"; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); - - auto [local_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - - status = local_file_handle->Open(path, FileAccessMode::kWrite); + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kWrite); EXPECT_TRUE(status.ok()); SizeT len = 10; @@ -49,11 +42,9 @@ TEST_F(LocalFileTest, TestAppend) { local_file_handle->Sync(); local_file_handle->Close(); - EXPECT_TRUE(virtual_storage.Exists(path)); - virtual_storage.DeleteFile(path); - EXPECT_FALSE(virtual_storage.Exists(path)); - - virtual_storage.UnInit(); + EXPECT_TRUE(LocalStore::Exists(path)); + LocalStore::DeleteFile(path); + EXPECT_FALSE(LocalStore::Exists(path)); } TEST_F(LocalFileTest, TestDir) { @@ -61,16 +52,9 @@ TEST_F(LocalFileTest, TestDir) { String dir = String(GetFullTmpDir()) + "/unit_test"; String path = dir + "/test_file.test"; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); - - VirtualStorage::MakeDirectoryLocal(dir); - - auto [local_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); + LocalStore::MakeDirectory(dir); - status = local_file_handle->Open(path, FileAccessMode::kWrite); + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kWrite);; EXPECT_TRUE(status.ok()); SizeT len = 10; @@ -83,12 +67,12 @@ TEST_F(LocalFileTest, TestDir) { local_file_handle->Sync(); local_file_handle->Close(); - EXPECT_TRUE(virtual_storage.Exists(path)); - EXPECT_TRUE(VirtualStorage::ExistsLocal(dir)); + EXPECT_TRUE(LocalStore::Exists(path)); + EXPECT_TRUE(LocalStore::Exists(dir)); - VirtualStorage::RemoveDirectoryLocal(dir); - EXPECT_FALSE(virtual_storage.Exists(path)); - EXPECT_FALSE(virtual_storage.Exists(dir)); + LocalStore::RemoveDirectory(dir); + EXPECT_FALSE(LocalStore::Exists(path)); + EXPECT_FALSE(LocalStore::Exists(dir)); } TEST_F(LocalFileTest, TestRead) { @@ -96,56 +80,47 @@ TEST_F(LocalFileTest, TestRead) { String path = String(GetFullTmpDir()) + "/test_file_read.abc"; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); - - auto [local_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - - status = local_file_handle->Open(path, FileAccessMode::kWrite); - EXPECT_TRUE(status.ok()); - SizeT len = 10; - UniquePtr data_array = MakeUnique(len); - for (SizeT i = 0; i < len; ++i) { - data_array[i] = i + 1; + { + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kWrite); + EXPECT_TRUE(status.ok()); + + UniquePtr data_array = MakeUnique(len); + for (SizeT i = 0; i < len; ++i) { + data_array[i] = i + 1; + } + + local_file_handle->Append(data_array.get(), len); + local_file_handle->Sync(); + local_file_handle->Close(); } - local_file_handle->Append(data_array.get(), len); - local_file_handle->Sync(); - local_file_handle->Close(); - - status = local_file_handle->Open(path, FileAccessMode::kRead); - EXPECT_TRUE(status.ok()); + { + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kRead); + EXPECT_TRUE(status.ok()); - UniquePtr read_data = MakeUnique(len); - auto [read_len, read_status] = local_file_handle->Read(read_data.get(), len); - EXPECT_TRUE(read_status.ok()); - local_file_handle->Close(); + UniquePtr read_data = MakeUnique(len); + auto [read_len, read_status] = local_file_handle->Read(read_data.get(), len); + EXPECT_TRUE(read_status.ok()); + local_file_handle->Close(); - EXPECT_EQ(read_len, len); - for(SizeT i = 0; i < len; ++i) { - EXPECT_EQ(read_data[i], i + 1); + EXPECT_EQ(read_len, len); + for(SizeT i = 0; i < len; ++i) { + EXPECT_EQ(read_data[i], i + 1); + } } - virtual_storage.DeleteFile(path); - EXPECT_FALSE(virtual_storage.Exists(path)); + LocalStore::DeleteFile(path); + EXPECT_FALSE(LocalStore::Exists(path)); } TEST_F(LocalFileTest, TestRename) { using namespace infinity; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); String old_path = String(GetFullTmpDir()) + "/test_file_old.abc"; String new_path = String(GetFullTmpDir()) + "/test_file_new.abc"; - auto [local_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - - status = local_file_handle->Open(old_path, FileAccessMode::kWrite); + auto [local_file_handle, status] = LocalStore::Open(old_path, FileAccessMode::kWrite); EXPECT_TRUE(status.ok()); SizeT len = 10; @@ -158,76 +133,68 @@ TEST_F(LocalFileTest, TestRename) { local_file_handle->Sync(); local_file_handle->Close(); - status = VirtualStorage::RenameLocal(old_path, new_path); + status = LocalStore::Rename(old_path, new_path); EXPECT_TRUE(status.ok()); - EXPECT_FALSE(virtual_storage.Exists(old_path)); - EXPECT_TRUE(virtual_storage.Exists(new_path)); + EXPECT_FALSE(LocalStore::Exists(old_path)); + EXPECT_TRUE(LocalStore::Exists(new_path)); - virtual_storage.DeleteFile(new_path); - EXPECT_FALSE(virtual_storage.Exists(new_path)); + LocalStore::DeleteFile(new_path); + EXPECT_FALSE(LocalStore::Exists(new_path)); } TEST_F(LocalFileTest, TestTruncate) { using namespace infinity; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); String path = String(GetFullTmpDir()) + "/test_file_truncate.abc"; - auto [local_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); + { + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kWrite); + EXPECT_TRUE(status.ok()); - status = local_file_handle->Open(path, FileAccessMode::kWrite); - EXPECT_TRUE(status.ok()); + SizeT len = 20; + UniquePtr data_array = MakeUnique(len); + for (SizeT i = 0; i < len; ++i) { + data_array[i] = i + 1; + } - SizeT len = 20; - UniquePtr data_array = MakeUnique(len); - for(SizeT i = 0; i < len; ++ i) { - data_array[i] = i + 1; + local_file_handle->Append(data_array.get(), len); + local_file_handle->Sync(); + local_file_handle->Close(); } - local_file_handle->Append(data_array.get(), len); - local_file_handle->Sync(); - local_file_handle->Close(); + LocalStore::Truncate(path, 10); - VirtualStorage::TruncateLocal(path, 10); + { + auto [local_file_handle, status] = LocalStore::Open(path, FileAccessMode::kRead); + EXPECT_TRUE(status.ok()); - status = local_file_handle->Open(path, FileAccessMode::kRead); - EXPECT_TRUE(status.ok()); + UniquePtr truncated_data = MakeUnique(10); + auto [read_len, read_status] = local_file_handle->Read(truncated_data.get(), 10); + EXPECT_TRUE(read_status.ok()); + EXPECT_EQ(read_len, 10); - UniquePtr truncated_data = MakeUnique(10); - auto [read_len, read_status] = local_file_handle->Read(truncated_data.get(), 10); - EXPECT_TRUE(read_status.ok()); - EXPECT_EQ(read_len, 10); + status = local_file_handle->Close(); + EXPECT_TRUE(status.ok()); - status = local_file_handle->Close(); - EXPECT_TRUE(status.ok()); - - for(SizeT i = 0; i < 10; ++i) { - EXPECT_EQ(truncated_data[i], i + 1); + for(SizeT i = 0; i < 10; ++i) { + EXPECT_EQ(truncated_data[i], i + 1); + } } - virtual_storage.DeleteFile(path); - EXPECT_FALSE(virtual_storage.Exists(path)); + LocalStore::DeleteFile(path); + EXPECT_FALSE(LocalStore::Exists(path)); } TEST_F(LocalFileTest, TestMerge) { using namespace infinity; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); String dst_path = String(GetFullTmpDir()) + "/test_file_append_dst.abc"; String src_path = String(GetFullTmpDir()) + "/test_file_append_src.abc"; // Write source file - auto [src_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - - status = src_file_handle->Open(src_path, FileAccessMode::kWrite); + auto [src_file_handle, status] = LocalStore::Open(src_path, FileAccessMode::kWrite); EXPECT_TRUE(status.ok()); SizeT src_len = 10; @@ -240,11 +207,8 @@ TEST_F(LocalFileTest, TestMerge) { src_file_handle->Sync(); src_file_handle->Close(); - auto [dst_file_handle, dst_status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(dst_status.ok()); - - status = dst_file_handle->Open(dst_path, FileAccessMode::kWrite); - EXPECT_TRUE(status.ok()); + auto [dst_file_handle, status1] = LocalStore::Open(dst_path, FileAccessMode::kWrite); + EXPECT_TRUE(status1.ok()); SizeT dst_len = 10; UniquePtr data_array2 = MakeUnique(dst_len); @@ -256,13 +220,10 @@ TEST_F(LocalFileTest, TestMerge) { dst_file_handle->Sync(); dst_file_handle->Close(); - VirtualStorage::MergeLocal(dst_path, src_path); - - auto [merge_file_handle, merge_status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(merge_status.ok()); + LocalStore::Merge(dst_path, src_path); - status = merge_file_handle->Open(dst_path, FileAccessMode::kRead); - EXPECT_TRUE(status.ok()); + auto [merge_file_handle, merge_status] = LocalStore::Open(dst_path, FileAccessMode::kRead); + EXPECT_TRUE(merge_status.ok()); UniquePtr combined_data = MakeUnique(src_len + dst_len); auto [read_len, merge_read_status] = merge_file_handle->Read(combined_data.get(), src_len + dst_len); @@ -277,30 +238,24 @@ TEST_F(LocalFileTest, TestMerge) { EXPECT_EQ(combined_data[i], i - dst_len + 1); } - virtual_storage.DeleteFile(src_path); - virtual_storage.DeleteFile(dst_path); - EXPECT_FALSE(virtual_storage.Exists(src_path)); - EXPECT_FALSE(virtual_storage.Exists(dst_path)); + LocalStore::DeleteFile(src_path); + LocalStore::DeleteFile(dst_path); + EXPECT_FALSE(LocalStore::Exists(src_path)); + EXPECT_FALSE(LocalStore::Exists(dst_path)); } TEST_F(LocalFileTest, TestCleanDir) { using namespace infinity; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); String dir = String(GetFullTmpDir()) + "/cleanup_test_dir"; String file_path1 = dir + "/file1.txt"; String file_path2 = dir + "/file2.txt"; - VirtualStorage::MakeDirectoryLocal(dir); + LocalStore::MakeDirectory(dir); // Append file1.txt - auto [src_file_handle, status1] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status1.ok()); - - status1 = src_file_handle->Open(file_path1, FileAccessMode::kWrite); - EXPECT_TRUE(status1.ok()); + auto [src_file_handle, status] = LocalStore::Open(file_path1, FileAccessMode::kWrite); + EXPECT_TRUE(status.ok()); SizeT src_len = 10; UniquePtr data_array1 = MakeUnique(src_len); @@ -313,10 +268,7 @@ TEST_F(LocalFileTest, TestCleanDir) { src_file_handle->Close(); // Append more to file1.txt - auto [append_src_file_handle, status2] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status2.ok()); - - status2 = append_src_file_handle->Open(file_path1, FileAccessMode::kWrite); + auto [append_src_file_handle, status2] = LocalStore::Open(file_path1, FileAccessMode::kWrite); EXPECT_TRUE(status2.ok()); append_src_file_handle->Append(data_array1.get(), src_len); @@ -325,10 +277,7 @@ TEST_F(LocalFileTest, TestCleanDir) { // Append more to file2.txt - auto [append_file2_handle, status3] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status3.ok()); - - status3 = append_file2_handle->Open(file_path2, FileAccessMode::kWrite); + auto [append_file2_handle, status3] = LocalStore::Open(file_path2, FileAccessMode::kWrite); EXPECT_TRUE(status3.ok()); SizeT len2 = 20; @@ -341,11 +290,11 @@ TEST_F(LocalFileTest, TestCleanDir) { append_file2_handle->Sync(); append_file2_handle->Close(); - VirtualStorage::CleanupDirectoryLocal(dir); + LocalStore::CleanupDirectory(dir); - EXPECT_FALSE(virtual_storage.Exists(file_path1)); - EXPECT_FALSE(virtual_storage.Exists(file_path2)); - EXPECT_TRUE(virtual_storage.Exists(dir)); - VirtualStorage::RemoveDirectoryLocal(dir); - EXPECT_FALSE(virtual_storage.Exists(dir)); + EXPECT_FALSE(LocalStore::Exists(file_path1)); + EXPECT_FALSE(LocalStore::Exists(file_path2)); + EXPECT_TRUE(LocalStore::Exists(dir)); + LocalStore::RemoveDirectory(dir); + EXPECT_FALSE(LocalStore::Exists(dir)); } diff --git a/src/unit_test/storage/io/file_handle/minio_file.cpp b/src/unit_test/storage/io/file_handle/minio_file.cpp index cb9f41022d..4eb44d402d 100644 --- a/src/unit_test/storage/io/file_handle/minio_file.cpp +++ b/src/unit_test/storage/io/file_handle/minio_file.cpp @@ -19,7 +19,7 @@ import status; import infinity_context; import compilation_config; import virtual_storage_type; -import virtual_storage; +import virtual_store; import abstract_file_handle; import minio_file; diff --git a/src/unit_test/storage/knnindex/emvb_search/test_emvb.cpp b/src/unit_test/storage/knnindex/emvb_search/test_emvb.cpp index 343b30bf8c..76e8dde8ae 100644 --- a/src/unit_test/storage/knnindex/emvb_search/test_emvb.cpp +++ b/src/unit_test/storage/knnindex/emvb_search/test_emvb.cpp @@ -21,6 +21,7 @@ import emvb_search; import emvb_product_quantization; import emvb_shared_vec; import file_system; +import local_file_handle; namespace infinity { extern template class EMVBSharedVec; @@ -46,8 +47,8 @@ class FakePQ final : public EMVBProductQuantizer { GetMultipleIPDistance(u32 embedding_offset, u32 embedding_num, u32 query_id, u32 query_num, const f32 *ip_table, f32 *output_ptr) const override { std::fill_n(output_ptr, embedding_num, 0.0f); } - void Save(FileHandler &file_handler) override {} - void Load(FileHandler &file_handler) override {} + void Save(LocalFileHandle &file_handler) override {} + void Load(LocalFileHandle &file_handler) override {} }; class EMVBTest : public BaseTest {}; diff --git a/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw.cpp b/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw.cpp index 5904b78889..9eccafd52b 100644 --- a/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw.cpp +++ b/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw.cpp @@ -35,6 +35,8 @@ import dist_func_cos; import vec_store_type; import hnsw_common; import infinity_exception; +import virtual_store; +import abstract_file_handle; using namespace infinity; @@ -83,7 +85,6 @@ class HnswAlgTest : public BaseTest { EXPECT_GE(correct_rate, 0.95); }; - LocalFileSystem fs; { auto hnsw_index = Hnsw::Make(chunk_size, max_chunk_n, dim, M, ef_construction); auto iter = DenseVectorIter(data.get(), dim, element_size); @@ -91,24 +92,24 @@ class HnswAlgTest : public BaseTest { test_func(hnsw_index); - u8 file_flags = FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG; - auto [file_handler, status] = fs.OpenFile(save_dir_ + "/test_hnsw.bin", file_flags, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw.bin", FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(status.message()); } - hnsw_index->Save(*file_handler); + hnsw_index->Save(*file_handle); + file_handle->Close(); } { - u8 file_flags = FileFlags::READ_FLAG; - auto [file_handler, status] = fs.OpenFile(save_dir_ + "/test_hnsw.bin", file_flags, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw.bin", FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(status.message()); } - auto hnsw_index = Hnsw::Load(*file_handler); + auto hnsw_index = Hnsw::Load(*file_handle); test_func(hnsw_index); + file_handle->Close(); } } @@ -148,7 +149,6 @@ class HnswAlgTest : public BaseTest { EXPECT_GE(correct_rate, 0.95); }; - LocalFileSystem fs; { auto hnsw_index = Hnsw::Make(chunk_size, max_chunk_n, dim, M, ef_construction); @@ -165,23 +165,23 @@ class HnswAlgTest : public BaseTest { } test_func(compress_hnsw); - u8 file_flags = FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG; - auto [file_handler, status] = fs.OpenFile(save_dir_ + "/test_hnsw.bin", file_flags, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw.bin", FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(status.message()); } - compress_hnsw->Save(*file_handler); + compress_hnsw->Save(*file_handle); + file_handle->Close(); } { - u8 file_flags = FileFlags::READ_FLAG; - auto [file_handler, status] = fs.OpenFile(save_dir_ + "/test_hnsw.bin", file_flags, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw.bin", FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(status.message()); } - auto compress_hnsw = CompressedHnsw::Load(*file_handler); + auto compress_hnsw = CompressedHnsw::Load(*file_handle); test_func(compress_hnsw); + file_handle->Close(); } } diff --git a/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw_sparse.cpp b/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw_sparse.cpp index a47bbefe58..39df34040b 100644 --- a/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw_sparse.cpp +++ b/src/unit_test/storage/knnindex/knn_hnsw/test_hnsw_sparse.cpp @@ -27,6 +27,9 @@ import compilation_config; import infinity_exception; import third_party; import sparse_test_util; +import virtual_store; +import local_file_handle; +import abstract_file_handle; using namespace infinity; @@ -73,29 +76,28 @@ class HnswSparseTest : public BaseTest { continue; } Vector> res = hnsw_index->KnnSearchSorted(query, 1, search_option); -// if (int(res[0].second) != gt_idx[i]) { -// std::cout << (fmt::format("{}, {}", res[0].second, gt_idx[i])) << std::endl; -// std::cout << (fmt::format("{}, {}", -res[0].first, gt_score[i])) << std::endl; -// } + // if (int(res[0].second) != gt_idx[i]) { + // std::cout << (fmt::format("{}, {}", res[0].second, gt_idx[i])) << std::endl; + // std::cout << (fmt::format("{}, {}", -res[0].first, gt_score[i])) << std::endl; + // } // EXPECT_EQ(res[0].second, gt_idx[i]); // EXPECT_NEAR(-res[0].first, gt_score[i], 1e-5); } - auto [file_handler, status] = - fs.OpenFile(save_dir_ + "/test_hnsw_sparse.bin", FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw_sparse.bin", FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(status.message()); } - hnsw_index->Save(*file_handler); - file_handler->Close(); + hnsw_index->Save(*file_handle); + file_handle->Close(); } { - auto [file_handler, status] = fs.OpenFile(save_dir_ + "/test_hnsw_sparse.bin", FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_dir_ + "/test_hnsw_sparse.bin", FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(status.message()); } - auto hnsw_index = Hnsw::Load(*file_handler); + auto hnsw_index = Hnsw::Load(*file_handle); KnnSearchOption search_option{.ef_ = 50}; for (SizeT i = 0; i < element_size; ++i) { SparseVecRef query = dataset.at(i); @@ -103,13 +105,14 @@ class HnswSparseTest : public BaseTest { continue; } Vector> res = hnsw_index->KnnSearchSorted(query, 1, search_option); -// if (int(res[0].second) != gt_idx[i]) { -// std::cout << (fmt::format("{}, {}", res[0].second, gt_idx[i])) << std::endl; -// std::cout << (fmt::format("{}, {}", -res[0].first, gt_score[i])) << std::endl; -// } + // if (int(res[0].second) != gt_idx[i]) { + // std::cout << (fmt::format("{}, {}", res[0].second, gt_idx[i])) << std::endl; + // std::cout << (fmt::format("{}, {}", -res[0].first, gt_score[i])) << std::endl; + // } // EXPECT_EQ(res[0].second, gt_idx[i]); // EXPECT_NEAR(-res[0].first, gt_score[i], 1e-5); } + file_handle->Close(); } } diff --git a/src/unit_test/storage/knnindex/knn_hnsw/test_lvq.cpp b/src/unit_test/storage/knnindex/knn_hnsw/test_lvq.cpp index bbce3e6928..6b1fb18b8f 100644 --- a/src/unit_test/storage/knnindex/knn_hnsw/test_lvq.cpp +++ b/src/unit_test/storage/knnindex/knn_hnsw/test_lvq.cpp @@ -25,6 +25,9 @@ import vec_store_type; import stl; import infinity_exception; import hnsw_common; +import virtual_store; +import local_file_handle; +import abstract_file_handle; using namespace infinity; @@ -171,12 +174,10 @@ TEST_F(HnswLVQTest, test1) { { std::string file_path = file_dir_ + "/lvq_store1.bin"; - LocalFileSystem fs; - fs.CleanupDirectory(file_dir_); + LocalStore::CleanupDirectory(file_dir_); { - uint8_t file_flags = FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG; - auto [file_handler, status] = fs.OpenFile(file_path, file_flags, FileLockType::kWriteLock); + auto [file_handle, status] = LocalStore::Open(file_path, FileAccessMode::kWrite); if(!status.ok()) { UnrecoverableError(status.message()); } @@ -189,17 +190,18 @@ TEST_F(HnswLVQTest, test1) { auto iter2 = DenseVectorIter(data.get() + vec_n_ / 2 * dim_, dim_, vec_n_ - vec_n_ / 2); lvq_store.AddVec(std::move(iter2)); - lvq_store.Save(*file_handler); + lvq_store.Save(*file_handle); + file_handle->Close(); } { - uint8_t file_flags = FileFlags::READ_FLAG; - auto [file_handler, status] = fs.OpenFile(file_path, file_flags, FileLockType::kReadLock); + auto [file_handle, status] = LocalStore::Open(file_path, FileAccessMode::kRead); if(!status.ok()) { UnrecoverableError(status.message()); } - auto lvq_store = DataStore::Load(*file_handler); + auto lvq_store = DataStore::Load(*file_handle); CheckStore(lvq_store, data.get()); + file_handle->Close(); } } } \ No newline at end of file diff --git a/src/unit_test/storage/knnindex/knn_sparse/test_bmp_index.cpp b/src/unit_test/storage/knnindex/knn_sparse/test_bmp_index.cpp index d72afb2c36..0c2454b563 100644 --- a/src/unit_test/storage/knnindex/knn_sparse/test_bmp_index.cpp +++ b/src/unit_test/storage/knnindex/knn_sparse/test_bmp_index.cpp @@ -27,6 +27,8 @@ import local_file_system; import file_system_type; import sparse_test_util; import infinity_exception; +import virtual_store; +import abstract_file_handle; using namespace infinity; @@ -99,20 +101,22 @@ class BMPIndexTest : public BaseTest { index.Optimize(optimize_options); test_query(index); - auto [file_handler, status] = fs.OpenFile(save_path, FileFlags::WRITE_FLAG | FileFlags::CREATE_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_path, FileAccessMode::kWrite); if (!status.ok()) { UnrecoverableError(fmt::format("Failed to open file: {}", save_path)); } - index.Save(*file_handler); + index.Save(*file_handle); + file_handle->Close(); } { - auto [file_handler, status] = fs.OpenFile(save_path, FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(save_path, FileAccessMode::kRead); if (!status.ok()) { UnrecoverableError(fmt::format("Failed to open file: {}", save_path)); } - auto index = BMPAlg::Load(*file_handler); + auto index = BMPAlg::Load(*file_handle); test_query(index); + file_handle->Close(); } } }; diff --git a/src/unit_test/storage/knnindex/knn_sparse/test_bp_reordering.cpp b/src/unit_test/storage/knnindex/knn_sparse/test_bp_reordering.cpp index baeca759b3..84eb09136c 100644 --- a/src/unit_test/storage/knnindex/knn_sparse/test_bp_reordering.cpp +++ b/src/unit_test/storage/knnindex/knn_sparse/test_bp_reordering.cpp @@ -21,10 +21,12 @@ import bp_reordering; import third_party; import infinity_exception; import file_system; -import local_file_system; +import virtual_store; import compilation_config; import file_system_type; import sparse_util; +import abstract_file_handle; +import local_file_handle; using namespace infinity; @@ -163,14 +165,13 @@ TEST_F(BPReorderingTest, test1) { TEST_F(BPReorderingTest, test2) { // GTEST_SKIP() << "Skip this test. This program is not a test but for preprocessing data."; - LocalFileSystem fs; Path dataset_path = Path(test_data_path()) / "benchmark" / "splade" / "base_small.csr"; - auto [file_handler, status] = fs.OpenFile(dataset_path.string(), FileFlags::READ_FLAG, FileLockType::kNoLock); + auto [file_handle, status] = LocalStore::Open(dataset_path.string(), FileAccessMode::kRead); if (!status.ok()) { std::cout << String(status.message()) << std::endl; return; } - auto dataset = SparseMatrix::Load(*file_handler); + auto dataset = SparseMatrix::Load(*file_handle); i32 data_n = dataset.nrow_; // i32 data_n = 1000; Vector> fwd(data_n); diff --git a/src/unit_test/storage/meta/entry/block_version.cpp b/src/unit_test/storage/meta/entry/block_version.cpp index 4f223fca88..49a034b84b 100644 --- a/src/unit_test/storage/meta/entry/block_version.cpp +++ b/src/unit_test/storage/meta/entry/block_version.cpp @@ -24,7 +24,7 @@ import third_party; import infinity_context; import block_version; import file_system; -import virtual_storage; +import virtual_store; import virtual_storage_type; import abstract_file_handle; import file_system_type; @@ -52,25 +52,19 @@ TEST_P(BlockVersionTest, SaveAndLoad) { block_version.Delete(5, 40); String version_path = String(GetFullDataDir()) + "/block_version_test"; - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); - { - auto [version_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - status = version_file_handle->Open(version_path, FileAccessMode::kWrite); + auto [local_file_handle, status] = LocalStore::Open(version_path, FileAccessMode::kWrite); EXPECT_TRUE(status.ok()); - block_version.SpillToFile(version_file_handle.get()); + block_version.SpillToFile(local_file_handle.get()); + local_file_handle->Close(); } { - auto [version_file_handle, status] = virtual_storage.BuildFileHandle(); + auto [local_file_handle, status] = LocalStore::Open(version_path, FileAccessMode::kRead); EXPECT_TRUE(status.ok()); - status = version_file_handle->Open(version_path, FileAccessMode::kRead); - - auto block_version2 = BlockVersion::LoadFromFile(version_file_handle.get()); + auto block_version2 = BlockVersion::LoadFromFile(local_file_handle.get()); ASSERT_EQ(block_version, *block_version2); + local_file_handle->Close(); } } diff --git a/src/unit_test/storage/persistence/persistence_manager.cpp b/src/unit_test/storage/persistence/persistence_manager.cpp index d5e8237069..39691c9029 100644 --- a/src/unit_test/storage/persistence/persistence_manager.cpp +++ b/src/unit_test/storage/persistence/persistence_manager.cpp @@ -2,7 +2,7 @@ import base_test; import stl; import persistence_manager; -import virtual_storage; +import virtual_store; import virtual_storage_type; import abstract_file_handle; import file_system_type; @@ -43,13 +43,7 @@ void PersistenceManagerTest::CheckObjData(const String& local_file_path, const S SizeT obj_file_size = fs::file_size(obj_fp); ASSERT_LE(obj_file_size, ObjSizeLimit); - VirtualStorage virtual_storage; - Map configs; - virtual_storage.Init(StorageType::kLocal, configs); - auto [pm_file_handle, status] = virtual_storage.BuildFileHandle(); - EXPECT_TRUE(status.ok()); - - status = pm_file_handle->Open(obj_path, FileAccessMode::kRead); + auto [pm_file_handle, status] = LocalStore::Open(obj_path, FileAccessMode::kRead); EXPECT_TRUE(status.ok()); status = pm_file_handle->Seek(obj_addr.part_offset_); EXPECT_TRUE(status.ok()); diff --git a/test/data/config/infinity_conf.toml b/test/data/config/infinity_conf.toml index 48d085d0d9..9e3231a949 100644 --- a/test/data/config/infinity_conf.toml +++ b/test/data/config/infinity_conf.toml @@ -22,7 +22,7 @@ log_level = "trace" [storage] persistence_dir = "/var/infinity/persistence" -storage_type = "minio" +storage_type = "local" [storage.object_storage] url = "0.0.0.0:9000"