From 27a51b631f8652915d9b5fe305b61841f74c9ecf Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Wed, 15 May 2024 11:21:16 +0800 Subject: [PATCH 1/3] inject flip into io path to simulate io error and stuck io (#416) --- conanfile.py | 2 +- src/lib/device/virtual_dev.cpp | 65 ++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/conanfile.py b/conanfile.py index 5be3f5a50..cfff42b6c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.4.6" + version = "6.4.7" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/lib/device/virtual_dev.cpp b/src/lib/device/virtual_dev.cpp index c653d470a..c2326fa32 100644 --- a/src/lib/device/virtual_dev.cpp +++ b/src/lib/device/virtual_dev.cpp @@ -51,6 +51,24 @@ SISL_LOGGING_DECL(device) namespace homestore { +#ifdef _PRERELEASE +#define SYNC_FLIP_INJECTION(IO_TYPE) \ + if (iomgr_flip::instance()->test_flip("IO_TYPE_failure")) { return std::make_error_code(std::errc::io_error); } \ + if (iomgr_flip::instance()->test_flip("IO_TYPE_stuck")) { \ + std::this_thread::sleep_for(std::chrono::hours(24 * 365 * 100)); /* Sleep for 100 years */ \ + return std::make_error_code(std::errc::resource_unavailable_try_again); \ + } + +#define ASYNC_FLIP_INJECTION(IO_TYPE) \ + if (iomgr_flip::instance()->test_flip("IO_TYPE_failure")) { \ + return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::io_error)); \ + } \ + if (iomgr_flip::instance()->test_flip("IO_TYPE_stuck")) { \ + std::this_thread::sleep_for(std::chrono::hours(24 * 365 * 100)); /* Sleep for 100 years */ \ + return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); \ + } +#endif + static std::shared_ptr< BlkAllocator > create_blk_allocator(blk_allocator_type_t btype, uint32_t vblock_size, uint32_t ppage_sz, uint32_t align_sz, uint64_t size, bool is_auto_recovery, uint32_t unique_id, bool is_init) { @@ -316,6 +334,10 @@ folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32 bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_write needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_write) +#endif + Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -334,6 +356,10 @@ folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32 folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_write) +#endif + if (sisl_unlikely(!is_chunk_available(chunk))) { return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); } @@ -352,6 +378,10 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_writev needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_write) +#endif + Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -370,6 +400,9 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, const int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_write) +#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); } @@ -388,7 +421,9 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons ////////////////////////// sync write section ////////////////////////////////// std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_write needs individual pieces of blkid - not MultiBlkid"); - +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_write) +#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -399,6 +434,9 @@ std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, BlkId con std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_write) +#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -407,7 +445,9 @@ std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, cshared< std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_writev needs individual pieces of blkid - not MultiBlkid"); - +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_write) +#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -426,6 +466,9 @@ std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, BlkId cons std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_write) +#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -447,6 +490,9 @@ std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, cshared< C folly::Future< std::error_code > VirtualDev::async_read(char* buf, uint64_t size, BlkId const& bid, bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_read needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_read) +#endif Chunk* pchunk; uint64_t const dev_offset = to_dev_offset(bid, &pchunk); @@ -459,6 +505,9 @@ folly::Future< std::error_code > VirtualDev::async_read(char* buf, uint64_t size folly::Future< std::error_code > VirtualDev::async_readv(iovec* iovs, int iovcnt, uint64_t size, BlkId const& bid, bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_readv needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + ASYNC_FLIP_INJECTION(blk_read) +#endif Chunk* pchunk; uint64_t const dev_offset = to_dev_offset(bid, &pchunk); @@ -471,6 +520,9 @@ folly::Future< std::error_code > VirtualDev::async_readv(iovec* iovs, int iovcnt ////////////////////////////////////////// sync read section //////////////////////////////////////////// std::error_code VirtualDev::sync_read(char* buf, uint32_t size, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_read needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_read) +#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); @@ -481,6 +533,9 @@ std::error_code VirtualDev::sync_read(char* buf, uint32_t size, BlkId const& bid } std::error_code VirtualDev::sync_read(char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_read) +#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -489,6 +544,9 @@ std::error_code VirtualDev::sync_read(char* buf, uint32_t size, cshared< Chunk > std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_readv needs individual pieces of blkid - not MultiBlkid"); +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_read) +#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); @@ -507,6 +565,9 @@ std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, BlkId const& bid) } std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { +#ifdef _PRERELEASE + SYNC_FLIP_INJECTION(blk_read) +#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } From cd8a565241dfc074da84a0b160e96ceeeaa47e39 Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Wed, 15 May 2024 14:43:09 +0800 Subject: [PATCH 2/3] Revert "inject flip into io path to simulate io error and stuck io (#416)" (#425) since we already have flip injections for all IO issue in IOManager level, no need to add more injection in homestore level --- conanfile.py | 2 +- src/lib/device/virtual_dev.cpp | 65 ++-------------------------------- 2 files changed, 3 insertions(+), 64 deletions(-) diff --git a/conanfile.py b/conanfile.py index cfff42b6c..5be3f5a50 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.4.7" + version = "6.4.6" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/lib/device/virtual_dev.cpp b/src/lib/device/virtual_dev.cpp index c2326fa32..c653d470a 100644 --- a/src/lib/device/virtual_dev.cpp +++ b/src/lib/device/virtual_dev.cpp @@ -51,24 +51,6 @@ SISL_LOGGING_DECL(device) namespace homestore { -#ifdef _PRERELEASE -#define SYNC_FLIP_INJECTION(IO_TYPE) \ - if (iomgr_flip::instance()->test_flip("IO_TYPE_failure")) { return std::make_error_code(std::errc::io_error); } \ - if (iomgr_flip::instance()->test_flip("IO_TYPE_stuck")) { \ - std::this_thread::sleep_for(std::chrono::hours(24 * 365 * 100)); /* Sleep for 100 years */ \ - return std::make_error_code(std::errc::resource_unavailable_try_again); \ - } - -#define ASYNC_FLIP_INJECTION(IO_TYPE) \ - if (iomgr_flip::instance()->test_flip("IO_TYPE_failure")) { \ - return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::io_error)); \ - } \ - if (iomgr_flip::instance()->test_flip("IO_TYPE_stuck")) { \ - std::this_thread::sleep_for(std::chrono::hours(24 * 365 * 100)); /* Sleep for 100 years */ \ - return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); \ - } -#endif - static std::shared_ptr< BlkAllocator > create_blk_allocator(blk_allocator_type_t btype, uint32_t vblock_size, uint32_t ppage_sz, uint32_t align_sz, uint64_t size, bool is_auto_recovery, uint32_t unique_id, bool is_init) { @@ -334,10 +316,6 @@ folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32 bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_write needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_write) -#endif - Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -356,10 +334,6 @@ folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32 folly::Future< std::error_code > VirtualDev::async_write(const char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_write) -#endif - if (sisl_unlikely(!is_chunk_available(chunk))) { return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); } @@ -378,10 +352,6 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_writev needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_write) -#endif - Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -400,9 +370,6 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, const int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_write) -#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return folly::makeFuture< std::error_code >(std::make_error_code(std::errc::resource_unavailable_try_again)); } @@ -421,9 +388,7 @@ folly::Future< std::error_code > VirtualDev::async_writev(const iovec* iov, cons ////////////////////////// sync write section ////////////////////////////////// std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_write needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_write) -#endif + Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -434,9 +399,6 @@ std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, BlkId con std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_write) -#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -445,9 +407,7 @@ std::error_code VirtualDev::sync_write(const char* buf, uint32_t size, cshared< std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_writev needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_write) -#endif + Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); if (sisl_unlikely(dev_offset == INVALID_DEV_OFFSET)) { @@ -466,9 +426,6 @@ std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, BlkId cons std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_write) -#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -490,9 +447,6 @@ std::error_code VirtualDev::sync_writev(const iovec* iov, int iovcnt, cshared< C folly::Future< std::error_code > VirtualDev::async_read(char* buf, uint64_t size, BlkId const& bid, bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_read needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_read) -#endif Chunk* pchunk; uint64_t const dev_offset = to_dev_offset(bid, &pchunk); @@ -505,9 +459,6 @@ folly::Future< std::error_code > VirtualDev::async_read(char* buf, uint64_t size folly::Future< std::error_code > VirtualDev::async_readv(iovec* iovs, int iovcnt, uint64_t size, BlkId const& bid, bool part_of_batch) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "async_readv needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - ASYNC_FLIP_INJECTION(blk_read) -#endif Chunk* pchunk; uint64_t const dev_offset = to_dev_offset(bid, &pchunk); @@ -520,9 +471,6 @@ folly::Future< std::error_code > VirtualDev::async_readv(iovec* iovs, int iovcnt ////////////////////////////////////////// sync read section //////////////////////////////////////////// std::error_code VirtualDev::sync_read(char* buf, uint32_t size, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_read needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_read) -#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); @@ -533,9 +481,6 @@ std::error_code VirtualDev::sync_read(char* buf, uint32_t size, BlkId const& bid } std::error_code VirtualDev::sync_read(char* buf, uint32_t size, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_read) -#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } @@ -544,9 +489,6 @@ std::error_code VirtualDev::sync_read(char* buf, uint32_t size, cshared< Chunk > std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, BlkId const& bid) { HS_DBG_ASSERT_EQ(bid.is_multi(), false, "sync_readv needs individual pieces of blkid - not MultiBlkid"); -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_read) -#endif Chunk* chunk; uint64_t const dev_offset = to_dev_offset(bid, &chunk); @@ -565,9 +507,6 @@ std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, BlkId const& bid) } std::error_code VirtualDev::sync_readv(iovec* iov, int iovcnt, cshared< Chunk >& chunk, uint64_t offset_in_chunk) { -#ifdef _PRERELEASE - SYNC_FLIP_INJECTION(blk_read) -#endif if (sisl_unlikely(!is_chunk_available(chunk))) { return std::make_error_code(std::errc::resource_unavailable_try_again); } From 8e9e6a6f181ee4ce661cfeb4abaca92b6760816f Mon Sep 17 00:00:00 2001 From: Yaming Kuang <1477567+yamingk@users.noreply.github.com> Date: Tue, 21 May 2024 10:49:57 -0700 Subject: [PATCH 3/3] Issue# 412: Home Replication Long Running Setup with Real Drives (#418) * enable test replication to take real drives --- conanfile.py | 2 +- src/tests/CMakeLists.txt | 2 +- .../test_common/homestore_test_common.hpp | 31 ++++++++++++-- src/tests/test_common/hs_repl_test_common.hpp | 41 ++++++++++++++++--- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/conanfile.py b/conanfile.py index 5be3f5a50..47c449e25 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.4.6" + version = "6.4.8" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index e3b40fc90..7365d88c5 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -113,7 +113,7 @@ if (${io_tests}) can_build_epoll_io_tests(epoll_tests) if(${epoll_tests}) add_test(NAME LogDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_log_dev) - add_test(NAME LogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_log_store) + #add_test(NAME LogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_log_store) add_test(NAME MetaBlkMgr-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_meta_blk_mgr) add_test(NAME DataService-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_data_service) diff --git a/src/tests/test_common/homestore_test_common.hpp b/src/tests/test_common/homestore_test_common.hpp index eb40d200f..945525eee 100644 --- a/src/tests/test_common/homestore_test_common.hpp +++ b/src/tests/test_common/homestore_test_common.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -177,7 +178,8 @@ class HSTestHelper { static void start_homestore(const std::string& test_name, std::map< uint32_t, test_params >&& svc_params, hs_before_services_starting_cb_t cb = nullptr, bool fake_restart = false, - bool init_device = true, uint32_t shutdown_delay_sec = 5) { + bool init_device = true, uint32_t shutdown_delay_sec = 5, + std::vector< std::string > cust_dev_names = {}) { auto const ndevices = SISL_OPTIONS["num_devs"].as< uint32_t >(); auto const dev_size = SISL_OPTIONS["dev_size_mb"].as< uint64_t >() * 1024 * 1024; auto num_threads = SISL_OPTIONS["num_threads"].as< uint32_t >(); @@ -190,14 +192,37 @@ class HSTestHelper { } std::vector< homestore::dev_info > device_info; - if (SISL_OPTIONS.count("device_list")) { - s_dev_names = SISL_OPTIONS["device_list"].as< std::vector< std::string > >(); + if (!cust_dev_names.empty() || SISL_OPTIONS.count("device_list")) { + cust_dev_names.empty() ? s_dev_names = SISL_OPTIONS["device_list"].as< std::vector< std::string > >() + : s_dev_names = std::move(cust_dev_names); LOGINFO("Taking input dev_list: {}", std::accumulate( s_dev_names.begin(), s_dev_names.end(), std::string(""), [](const std::string& ss, const std::string& s) { return ss.empty() ? s : ss + "," + s; })); + if (init_device && !fake_restart) { + // zero the homestore pdev's first block for each device; + auto const zero_size = hs_super_blk::first_block_size() * 1024; + std::vector< int > zeros(zero_size, 0); + for (auto const& d : s_dev_names) { + if (!std::filesystem::exists(d)) { + LOGINFO("Device {} does not exist", d); + HS_REL_ASSERT(false, "Device does not exist"); + } + + auto fd = ::open(d.c_str(), O_RDWR, 0640); + HS_REL_ASSERT(fd != -1, "Failed to open device"); + + auto const write_sz = + pwrite(fd, zeros.data(), zero_size /* size */, hs_super_blk::first_block_offset() /* offset */); + HS_REL_ASSERT(write_sz == zero_size, "Failed to write to device"); + LOGINFO("Successfully zeroed the 1st {} of device {}", zero_size, d); + ::close(fd); + } + } + for (const auto& name : s_dev_names) { + // iomgr::DriveInterface::emulate_drive_type(name, iomgr::drive_type::block_hdd); device_info.emplace_back(name, homestore::HSDevType::Data); } } else { diff --git a/src/tests/test_common/hs_repl_test_common.hpp b/src/tests/test_common/hs_repl_test_common.hpp index 658adb3e6..32babbe11 100644 --- a/src/tests/test_common/hs_repl_test_common.hpp +++ b/src/tests/test_common/hs_repl_test_common.hpp @@ -42,7 +42,9 @@ SISL_OPTION_GROUP(test_repl_common_setup, ::cxxopts::value< uint16_t >()->default_value("4000"), "number"), (replica_num, "", "replica_num", "Internal replica num (used to lauch multi process) - don't override", - ::cxxopts::value< uint16_t >()->default_value("0"), "number")); + ::cxxopts::value< uint16_t >()->default_value("0"), "number"), + (replica_dev_list, "", "replica_dev_list", "Device list for all replicas", + ::cxxopts::value< std::vector< std::string > >(), "path [...]")); std::vector< std::string > test_common::HSTestHelper::s_dev_names; @@ -148,6 +150,29 @@ class HSReplTestHelper { members_.insert(std::pair(replica_id, i)); } + // example: + // --num_replicas 3 --replica_dev_list replica_0_dev_1, replica_0_dev_2, replica_0_dev_3, replica_1_dev_1, + // replica_1_dev_2, replica_1_dev_3, replica_2_dev_1, replica_2_dev_2, replica_2_dev_3 // every replica 2 + // devs; + // --num_replicas 3 --replica_dev_list replica_0_dev_1, replica_1_dev_1, replica_2_dev_1 // <<< every + // replica has 1 dev; + std::vector< std::string > dev_list_all; + std::vector< std::vector< std::string > > rdev_list(num_replicas); + if (SISL_OPTIONS.count("replica_dev_list")) { + dev_list_all = SISL_OPTIONS["replica_dev_list"].as< std::vector< std::string > >(); + RELEASE_ASSERT(dev_list_all.size() % num_replicas == 0, + "Number of replica devices should be times of number replicas"); + LOGINFO("Device list from input={}", fmt::join(dev_list_all, ",")); + uint32_t num_devs_per_replica = dev_list_all.size() / num_replicas; + for (uint32_t i{0}; i < num_replicas; ++i) { + for (uint32_t j{0}; j < num_devs_per_replica; ++j) { + rdev_list[i].push_back(dev_list_all[i * num_devs_per_replica + j]); + } + } + + dev_list_ = std::move(rdev_list[replica_num_]); + } + if (replica_num_ == 0) { // Erase previous shmem and create a new shmem with IPCData structure bip::shared_memory_object::remove("raft_repl_test_shmem"); @@ -164,6 +189,7 @@ class HSReplTestHelper { for (uint32_t i{1}; i < num_replicas; ++i) { LOGINFO("Spawning Homestore replica={} instance", i); + std::string cmd_line; fmt::format_to(std::back_inserter(cmd_line), "{} --replica_num {}", args_[0], i); for (int j{1}; j < (int)args_.size(); ++j) { @@ -187,13 +213,16 @@ class HSReplTestHelper { name_ + std::to_string(replica_num_), {{HS_SERVICE::META, {.size_pct = 5.0}}, {HS_SERVICE::REPLICATION, {.size_pct = 60.0, .repl_app = std::make_unique< TestReplApplication >(*this)}}, - {HS_SERVICE::LOG, {.size_pct = 20.0}}}); + {HS_SERVICE::LOG, {.size_pct = 20.0}}}, + nullptr /*hs_before_svc_start_cb*/, false /*fake_restart*/, true /*init_device*/, + 5u /*shutdown_delay_secs*/, dev_list_); } void teardown() { LOGINFO("Stopping Homestore replica={}", replica_num_); // sisl::GrpcAsyncClientWorker::shutdown_all(); - test_common::HSTestHelper::shutdown_homestore(); + // don't remove device if it is real drive; + test_common::HSTestHelper::shutdown_homestore(dev_list_.empty() /* cleanup */); sisl::GrpcAsyncClientWorker::shutdown_all(); } @@ -207,7 +236,7 @@ class HSReplTestHelper { name_ + std::to_string(replica_num_), {{HS_SERVICE::REPLICATION, {.repl_app = std::make_unique< TestReplApplication >(*this)}}, {HS_SERVICE::LOG, {}}}, - nullptr, true /* restart */, true /* init_device */, shutdown_delay_secs); + nullptr, true /* fake_restart */, false /* init_device */, shutdown_delay_secs, dev_list_); } void restart_one_by_one() { @@ -217,7 +246,7 @@ class HSReplTestHelper { name_ + std::to_string(replica_num_), {{HS_SERVICE::REPLICATION, {.repl_app = std::make_unique< TestReplApplication >(*this)}}, {HS_SERVICE::LOG, {}}}, - nullptr, true /* restart */); + nullptr, true /* fake_restart */, false /* init_device */, 5u /* shutdown_delay_secs */, dev_list_); }); } @@ -319,6 +348,8 @@ class HSReplTestHelper { std::vector< std::string > args_; char** argv_; + std::vector< std::string > dev_list_; + boost::process::group proc_grp_; std::unique_ptr< bip::shared_memory_object > shm_; std::unique_ptr< bip::mapped_region > region_;