From ed5b061e0a596ce23b912bf6244f0c3c3592d550 Mon Sep 17 00:00:00 2001 From: Sanal Date: Thu, 11 Apr 2024 18:04:45 -0700 Subject: [PATCH] Add multiple chunk test for journal vdev. (#358) --- conanfile.py | 2 +- src/lib/device/journal_vdev.cpp | 24 +++-- src/lib/device/journal_vdev.hpp | 10 +- src/lib/logstore/log_dev.hpp | 2 + src/tests/test_journal_vdev.cpp | 185 +++++++++++++++++++++++++++----- src/tests/test_log_dev.cpp | 28 +++-- src/tests/test_log_store.cpp | 7 +- 7 files changed, 216 insertions(+), 42 deletions(-) diff --git a/conanfile.py b/conanfile.py index 7d38763c1..9186f0b5a 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.2.1" + version = "6.2.2" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/lib/device/journal_vdev.cpp b/src/lib/device/journal_vdev.cpp index d6063ae54..424a4e57c 100644 --- a/src/lib/device/journal_vdev.cpp +++ b/src/lib/device/journal_vdev.cpp @@ -472,11 +472,19 @@ off_t JournalVirtualDev::Descriptor::dev_offset(off_t nbytes) const { } void JournalVirtualDev::Descriptor::update_data_start_offset(off_t offset) { - m_data_start_offset = offset; - auto data_start_offset_aligned = sisl::round_down(m_data_start_offset, m_vdev.info().chunk_size); - m_end_offset = data_start_offset_aligned + m_journal_chunks.size() * m_vdev.info().chunk_size; - LOGINFOMOD(journalvdev, "Updated data start offset off 0x{} {}", to_hex(offset), to_string()); - RELEASE_ASSERT_EQ(m_end_offset - data_start_offset_aligned, m_total_size, "offset size mismatch {}", to_string()); + if (!m_journal_chunks.empty()) { + m_data_start_offset = offset; + auto data_start_offset_aligned = sisl::round_down(m_data_start_offset, m_vdev.info().chunk_size); + m_end_offset = data_start_offset_aligned + m_journal_chunks.size() * m_vdev.info().chunk_size; + LOGINFOMOD(journalvdev, "Updated data start offset off 0x{} {}", to_hex(offset), to_string()); + RELEASE_ASSERT_EQ(m_end_offset - data_start_offset_aligned, m_total_size, "offset size mismatch {}", + to_string()); + } else { + // If there are no chunks, we round up to the next chunk size. + m_data_start_offset = sisl::round_up(offset, m_vdev.info().chunk_size); + m_end_offset = m_data_start_offset; + LOGINFOMOD(journalvdev, "No chunks, updated data start offset off 0x{} {}", to_hex(offset), to_string()); + } } off_t JournalVirtualDev::Descriptor::tail_offset(bool reserve_space_include) const { @@ -536,6 +544,9 @@ void JournalVirtualDev::Descriptor::truncate(off_t truncate_offset) { for (auto it = m_journal_chunks.begin(); it != m_journal_chunks.end();) { auto chunk = *it; start += chunk->size(); + + // Also if its the last chunk and there is no data after truncate, we release chunk. + auto write_sz_in_total = m_write_sz_in_total.load(std::memory_order_relaxed); if (start >= truncate_offset) { break; } m_total_size -= chunk->size(); @@ -551,6 +562,7 @@ void JournalVirtualDev::Descriptor::truncate(off_t truncate_offset) { // to know the end offset of the log dev during recovery. // Format and add back to pool. m_vdev.m_chunk_pool->enqueue(chunk); + LOGINFOMOD(journalvdev, "After truncate released chunk {}", chunk->to_string()); } // Update our start offset, to keep track of actual size @@ -561,7 +573,7 @@ void JournalVirtualDev::Descriptor::truncate(off_t truncate_offset) { m_write_sz_in_total.fetch_sub(size_to_truncate, std::memory_order_relaxed); m_truncate_done = true; - HS_PERIODIC_LOG(DEBUG, journalvdev, "After truncate desc {}", to_string()); + HS_PERIODIC_LOG(INFO, journalvdev, "After truncate desc {}", to_string()); } #if 0 diff --git a/src/lib/device/journal_vdev.hpp b/src/lib/device/journal_vdev.hpp index 18bc9608d..b0a304553 100644 --- a/src/lib/device/journal_vdev.hpp +++ b/src/lib/device/journal_vdev.hpp @@ -63,7 +63,7 @@ class JournalVirtualDev : public VirtualDev { off_t m_data_start_offset{0}; // Start offset of where actual data begin for this vdev std::atomic< uint64_t > m_write_sz_in_total{0}; // Size will be decreased by truncate and increased by append; - bool m_truncate_done{true}; + bool m_truncate_done{false}; uint64_t m_reserved_sz{0}; // write size within chunk, used to check chunk boundary; std::vector< shared< Chunk > > m_journal_chunks; // Chunks part of this journal in order. uint64_t m_total_size{0}; // Total size of all chunks. @@ -227,6 +227,14 @@ class JournalVirtualDev : public VirtualDev { off_t end_offset() const { return m_end_offset; } + uint64_t write_sz_in_total() const { return m_write_sz_in_total.load(); } + + uint32_t num_chunks_used() const { return m_journal_chunks.size(); } + + bool truncate_done() const { return m_truncate_done; } + + uint64_t reserved_size() const { return m_reserved_sz; } + /** * @brief : persist start logical offset to vdev's super block * Supposed to be called when truncate happens; diff --git a/src/lib/logstore/log_dev.hpp b/src/lib/logstore/log_dev.hpp index f356102a0..530388d0e 100644 --- a/src/lib/logstore/log_dev.hpp +++ b/src/lib/logstore/log_dev.hpp @@ -796,6 +796,7 @@ class LogDev : public std::enable_shared_from_this< LogDev > { logdev_key do_device_truncate(bool dry_run = false); void handle_unopened_log_stores(bool format); logdev_id_t get_id() { return m_logdev_id; } + shared< JournalVirtualDev::Descriptor > get_journal_descriptor() const { return m_vdev_jd; } private: LogGroup* make_log_group(uint32_t estimated_records) { @@ -872,6 +873,7 @@ class LogDev : public std::enable_shared_from_this< LogDev > { std::atomic< bool > m_flush_status = false; // Timer handle iomgr::timer_handle_t m_flush_timer_hdl{iomgr::null_timer_handle}; + }; // LogDev } // namespace homestore diff --git a/src/tests/test_journal_vdev.cpp b/src/tests/test_journal_vdev.cpp index 0b8ddc34f..84b47daf4 100644 --- a/src/tests/test_journal_vdev.cpp +++ b/src/tests/test_journal_vdev.cpp @@ -76,7 +76,8 @@ class VDevJournalIOTest : public ::testing::Test { {HS_SERVICE::META, {.size_pct = 15.0}}, {HS_SERVICE::LOG, {.size_pct = 50.0, - .chunk_size = 32 * 1024 * 1024, + .chunk_size = 8 * 1024 * 1024, + .min_chunk_size = 8 * 1024 * 1024, .vdev_size_type = vdev_size_type_t::VDEV_SIZE_DYNAMIC}}, }, nullptr /* starting_cb */, false /* restart */); @@ -90,7 +91,8 @@ class VDevJournalIOTest : public ::testing::Test { {HS_SERVICE::META, {.size_pct = 15.0}}, {HS_SERVICE::LOG, {.size_pct = 50.0, - .chunk_size = 32 * 1024 * 1024, + .chunk_size = 8 * 1024 * 1024, + .min_chunk_size = 8 * 1024 * 1024, .vdev_size_type = vdev_size_type_t::VDEV_SIZE_DYNAMIC}}, }, nullptr /* starting_cb */, true /* restart */); @@ -103,14 +105,33 @@ class JournalDescriptorTest { uint64_t crc; }; + struct VerifyDescriptor { + uint64_t ds{0}; + uint64_t end{0}; + uint64_t writesz{0}; + uint64_t tail{0}; + uint64_t rsvdsz{0}; + int32_t chunks{0}; + bool trunc{false}; + uint64_t total{0}; + uint64_t seek{0}; + }; + public: - JournalDescriptorTest(logdev_id_t id) : m_logdev_id(id) { reinit(); } + JournalDescriptorTest(logdev_id_t id) : m_logdev_id(id) { restore(); } std::shared_ptr< JournalVirtualDev::Descriptor > vdev_jd() { return m_vdev_jd; } - void reinit() { + void save() { + last_start_offset = m_vdev_jd->data_start_offset(); + last_tail_offset = m_vdev_jd->tail_offset(); + } + + void restore() { auto vdev = hs()->logstore_service().get_vdev(); m_vdev_jd = vdev->open(m_logdev_id); + m_vdev_jd->update_data_start_offset(last_start_offset); + m_vdev_jd->update_tail_offset(last_tail_offset); } uint64_t get_elapsed_time(Clock::time_point start) { @@ -173,8 +194,7 @@ class JournalDescriptorTest { } void truncate(off_t off_to_truncate) { - LOGDEBUG("truncating to offset: 0x{}, start: 0x{}, tail: 0x{}", to_hex(off_to_truncate), - to_hex(m_vdev_jd->data_start_offset()), to_hex(m_vdev_jd->tail_offset())); + LOGDEBUG("truncating to offset: 0x{}, desc: {}", to_hex(off_to_truncate), m_vdev_jd->to_string()); validate_truncate_offset(off_to_truncate); @@ -182,8 +202,10 @@ class JournalDescriptorTest { m_vdev_jd->truncate(off_to_truncate); auto tail_after = m_vdev_jd->tail_offset(); - HS_DBG_ASSERT_EQ(tail_before, tail_after); - HS_DBG_ASSERT_EQ(off_to_truncate, m_vdev_jd->data_start_offset()); + if (m_vdev_jd->num_chunks_used()) { + HS_DBG_ASSERT_EQ(tail_before, tail_after); + HS_DBG_ASSERT_EQ(off_to_truncate, m_vdev_jd->data_start_offset()); + } if (off_to_truncate > m_start_off) { // remove the offsets before truncate offset, since they are not valid for read anymore; @@ -231,9 +253,13 @@ class JournalDescriptorTest { auto used_space = m_vdev_jd->used_size(); auto start_off = m_vdev_jd->data_start_offset(); - HS_DBG_ASSERT_GT(m_vdev_jd->size(), 0); - HS_DBG_ASSERT_LT(used_space, m_vdev_jd->size()); - HS_DBG_ASSERT_EQ(start_off, m_start_off); + if (m_vdev_jd->num_chunks_used() != 0) { + HS_DBG_ASSERT_GT(m_vdev_jd->size(), 0); + HS_DBG_ASSERT_LE(used_space, m_vdev_jd->size()); + HS_DBG_ASSERT_EQ(start_off, m_start_off); + } else { + HS_DBG_ASSERT_EQ(m_vdev_jd->size(), 0); + } } bool time_to_truncate() { @@ -270,7 +296,7 @@ class JournalDescriptorTest { HS_DBG_ASSERT_EQ(m_start_off, start_offset); if (start_offset < tail_offset) { HS_DBG_ASSERT_GE(off, start_offset, "Wrong offset: {}, start_off: {}", off, start_offset); - HS_DBG_ASSERT_LT(off, tail_offset, "Wrong offset: {}, tail_offset: {}", off, tail_offset); + HS_DBG_ASSERT_LE(off, tail_offset, "Wrong offset: {}, tail_offset: {}", off, tail_offset); } else { HS_DBG_ASSERT(off < tail_offset || off >= start_offset, "Wrong offset: {}, start: {}, tail: {}", off, start_offset, tail_offset); @@ -311,8 +337,9 @@ class JournalDescriptorTest { m_read_cnt++; } - void random_write() { - auto sz_to_wrt = rand_size(); + void random_write() { alloc_write(rand_size()); } + void fixed_write(int size) { alloc_write(size); } + void alloc_write(int sz_to_wrt) { auto off_to_wrt = m_vdev_jd->alloc_next_append_blk(sz_to_wrt); auto it = m_off_to_info_map.find(off_to_wrt); @@ -361,6 +388,17 @@ class JournalDescriptorTest { return sisl::round_up(dist(generator), dma_alignment); } + void verify_journal_descriptor(shared< JournalVirtualDev::Descriptor > logdev_jd, VerifyDescriptor d) { + LOGINFO("{}", logdev_jd->to_string()); + ASSERT_EQ(logdev_jd->data_start_offset(), d.ds); + ASSERT_EQ(logdev_jd->end_offset(), d.end); + ASSERT_EQ(logdev_jd->write_sz_in_total(), d.writesz); + ASSERT_EQ(logdev_jd->reserved_size(), d.rsvdsz); + ASSERT_EQ(logdev_jd->num_chunks_used(), d.chunks); + ASSERT_EQ(logdev_jd->truncate_done(), d.trunc); + ASSERT_EQ(logdev_jd->size(), d.total); + } + private: logdev_id_t m_logdev_id = 0; off_t m_start_off = 0; @@ -371,6 +409,8 @@ class JournalDescriptorTest { std::map< off_t, write_info > m_off_to_info_map; Clock::time_point m_start_time; std::shared_ptr< JournalVirtualDev::Descriptor > m_vdev_jd; + uint64_t last_tail_offset = 0; + uint64_t last_start_offset = 0; friend class VDevJournalIOTest; }; @@ -426,22 +466,16 @@ TEST_F(VDevJournalIOTest, Recovery) { LOGINFO("Restart homestore"); // Record the offsets of the journal descriptors. - std::vector< uint64_t > last_tail_offset, last_start_offset; - for (uint32_t i = 0; i < tests.size(); i++) { - auto vdev_jd = tests[i].vdev_jd(); - last_tail_offset.push_back(vdev_jd->tail_offset()); - last_start_offset.push_back(vdev_jd->data_start_offset()); + for (auto& t : tests) { + t.save(); } // Restart homestore. restart_homestore(); - // Set the offsets after restart. - for (uint32_t i = 0; i < tests.size(); i++) { - tests[i].reinit(); - auto vdev_jd = tests[i].vdev_jd(); - vdev_jd->update_data_start_offset(last_start_offset[i]); - vdev_jd->update_tail_offset(last_tail_offset[i]); + // Restore the offsets after restart. + for (auto& t : tests) { + t.restore(); } // Validate all logs. @@ -470,6 +504,107 @@ TEST_F(VDevJournalIOTest, Recovery) { } } +TEST_F(VDevJournalIOTest, MultipleChunkTest) { + // Chunk size is 8MB and each data log entry will be of size 3MB to create gaps. + uint64_t MB = 1024 * 1024; + uint64_t chunk_size = 8 * MB; + uint64_t data_size = 3 * MB; + JournalDescriptorTest test(1); + auto log_dev_jd = test.vdev_jd(); + + auto restart_restore = [&]() { + test.save(); + restart_homestore(); + test.restore(); + log_dev_jd = test.vdev_jd(); + }; + + // clang-format off + // Initially no chunks used and offsets are zero. + uint64_t writesz = 0; + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = 0, .writesz = writesz, .rsvdsz = 0, .chunks = 0, + .trunc = false, .total = 0, .seek = 0x0}); + + // Insert two entries. Create one chunk 1. + LOGINFO("Inserting two entries"); + for (int i = 0; i < 2; i++) { + test.fixed_write(data_size); + } + + // Verify write size has two data entries and one chunk. + writesz = 2 * data_size; + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = chunk_size, .writesz = writesz, .rsvdsz = 0, + .chunks = 1, .trunc = false, .total = chunk_size, .seek = 0x0}); + + // Add three more entries. Now chunk 2 and 3 has to be created. + LOGINFO("Inserting three entries"); + for (int i = 0; i < 3; i++) { + test.fixed_write(data_size); + } + + // Total three chunks of 8MB, write size will be two whole chunk and last chunk 3 contains + // one data log entries. There will be gap at the end of chunk 1 and 2. + writesz = 2 * chunk_size + data_size; + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = 3 * chunk_size, .writesz = writesz, .rsvdsz = 0, + .chunks = 3, .trunc = false, .total = 3 * chunk_size, .seek = 0x0}); + test.read_all(); + + // Restart homestore and restore the offsets. + LOGINFO("Restart homestore"); + restart_restore(); + + // Verify the same as above after restart. + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = 3 * chunk_size, .writesz = writesz, .rsvdsz = 0, + .chunks = 3, .trunc = false, .total = 3 * chunk_size, .seek = 0x0}); + test.read_all(); + + // Add one data entry. No additional chunks because there is enough space in chunk 3 but write size increased. + LOGINFO("Inserting one entry"); + test.fixed_write(data_size); + writesz = 2 * chunk_size + 2 * data_size; + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = 3 * chunk_size, .writesz = writesz, .rsvdsz = 0, + .chunks = 3, .trunc = false, .total = 3 * chunk_size, .seek = 0x0}); + + // Add one data entry. No more space in chunk 3. Additional chunk 4 created. + LOGINFO("Inserting one entry"); + test.fixed_write(data_size); + writesz = 3 * chunk_size + data_size; + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x0, .end = 4 * chunk_size, .writesz = writesz, .rsvdsz = 0, + .chunks = 4, .trunc = false, .total = 4 * chunk_size, .seek = 0x0}); + + // Truncate two data entries. No change in chunk count, only write size and data start changed. + LOGINFO("Truncating two entries"); + uint64_t trunc_sz = 2 * data_size; + uint64_t trunc_offset = 2 * data_size; + test.truncate(trunc_offset); + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x600000, .end = 4 * chunk_size, .writesz = writesz - trunc_sz, + .rsvdsz = 0, .chunks = 4, .trunc = true, .total = 4 * chunk_size, .seek = 0x0}); + + // Truncate one more entry. Release one chunk back and reduce chunk count. Increase the data start. + LOGINFO("Truncating one entry"); + trunc_offset = chunk_size + data_size; + trunc_sz = chunk_size + data_size; + test.truncate(trunc_offset); + test.verify_journal_descriptor(log_dev_jd, {.ds = 0xb00000, .end = 4 * chunk_size, .writesz = writesz - trunc_sz, + .rsvdsz = 0, .chunks = 3, .trunc = true, .total = 3 * chunk_size, .seek = 0x0}); + + // Restart homestore and restore the offsets. + LOGINFO("Restart homestore"); + restart_restore(); + test.verify_journal_descriptor(log_dev_jd, {.ds = 0xb00000, .end = 4 * chunk_size, .writesz = writesz - trunc_sz, + .rsvdsz = 0, .chunks = 3, .trunc = false, .total = 3 * chunk_size, .seek = 0x0}); + test.read_all(); + + // Truncate all entries. Num chunks 1, write sz should be 0. + LOGINFO("Truncating all entries"); + trunc_offset = log_dev_jd->tail_offset(); + test.truncate(trunc_offset); + test.verify_journal_descriptor(log_dev_jd, {.ds = 0x1b00000, .end = 0x2000000, .writesz = 0, .rsvdsz = 0, + .chunks = 1, .trunc = true, .total = 8388608, .seek = 0x0}); + + // clang-format on +} + SISL_OPTION_GROUP(test_journal_vdev, (truncate_watermark_percentage, "", "truncate_watermark_percentage", "percentage of space usage to trigger truncate", ::cxxopts::value< uint32_t >()->default_value("80"), diff --git a/src/tests/test_log_dev.cpp b/src/tests/test_log_dev.cpp index 2a1bba56b..d56af0303 100644 --- a/src/tests/test_log_dev.cpp +++ b/src/tests/test_log_dev.cpp @@ -77,6 +77,15 @@ class LogDevTest : public ::testing::Test { void start_homestore(bool restart = false, hs_before_services_starting_cb_t starting_cb = nullptr) { auto const ndevices = SISL_OPTIONS["num_devs"].as< uint32_t >(); auto const dev_size = SISL_OPTIONS["dev_size_mb"].as< uint64_t >() * 1024 * 1024; + if (starting_cb == nullptr) { + starting_cb = [this]() { + HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { + // Disable flush timer in UT. + s.logstore.flush_timer_frequency_us = 0; + }); + HS_SETTINGS_FACTORY().save(); + }; + } test_common::HSTestHelper::start_homestore("test_log_dev", { {HS_SERVICE::META, {.size_pct = 15.0}}, @@ -91,7 +100,7 @@ class LogDevTest : public ::testing::Test { virtual void TearDown() override { test_common::HSTestHelper::shutdown_homestore(); } - test_log_data* prepare_data(const logstore_seq_num_t lsn, bool& io_memory) { + test_log_data* prepare_data(const logstore_seq_num_t lsn, bool& io_memory, uint32_t fixed_size = 0) { static thread_local std::random_device rd{}; static thread_local std::default_random_engine re{rd()}; uint32_t sz{0}; @@ -100,14 +109,18 @@ class LogDevTest : public ::testing::Test { // Generate buffer of random size and fill with specific data std::uniform_int_distribution< uint8_t > gen_percentage{0, 99}; std::uniform_int_distribution< uint32_t > gen_data_size{0, max_data_size - 1}; + if (fixed_size == 0) { + sz = gen_data_size(re); + } else { + sz = fixed_size; + } if (gen_percentage(re) < static_cast< uint8_t >(10)) { // 10% of data is dma'ble aligned boundary - const auto alloc_sz = sisl::round_up(gen_data_size(re) + sizeof(test_log_data), s_max_flush_multiple); + const auto alloc_sz = sisl::round_up(sz + sizeof(test_log_data), s_max_flush_multiple); raw_buf = iomanager.iobuf_alloc(dma_address_boundary, alloc_sz); sz = alloc_sz - sizeof(test_log_data); io_memory = true; } else { - sz = gen_data_size(re); raw_buf = static_cast< uint8_t* >(std::malloc(sizeof(test_log_data) + sz)); io_memory = false; } @@ -132,9 +145,9 @@ class LogDevTest : public ::testing::Test { << " size=" << d->size; } - void insert_sync(std::shared_ptr< HomeLogStore > log_store, logstore_seq_num_t lsn) { + void insert_sync(std::shared_ptr< HomeLogStore > log_store, logstore_seq_num_t lsn, uint32_t fixed_size = 0) { bool io_memory{false}; - auto* d = prepare_data(lsn, io_memory); + auto* d = prepare_data(lsn, io_memory, fixed_size); const bool succ = log_store->write_sync(lsn, {uintptr_cast(d), d->total_size(), false}); EXPECT_TRUE(succ); LOGINFO("Written sync data for LSN -> {}:{}", log_store->get_store_id(), lsn); @@ -145,10 +158,11 @@ class LogDevTest : public ::testing::Test { } } - void kickstart_inserts(std::shared_ptr< HomeLogStore > log_store, logstore_seq_num_t& cur_lsn, int64_t batch) { + void kickstart_inserts(std::shared_ptr< HomeLogStore > log_store, logstore_seq_num_t& cur_lsn, int64_t batch, + uint32_t fixed_size = 0) { auto last = cur_lsn + batch; for (; cur_lsn < last; cur_lsn++) { - insert_sync(log_store, cur_lsn); + insert_sync(log_store, cur_lsn, fixed_size); } } diff --git a/src/tests/test_log_store.cpp b/src/tests/test_log_store.cpp index 586268eaf..7d197e5e8 100644 --- a/src/tests/test_log_store.cpp +++ b/src/tests/test_log_store.cpp @@ -320,8 +320,9 @@ class SampleLogStoreClient { void recovery_validate() { LOGINFO("Truncated upto {}", get_truncated_upto()); - LOGINFO("Totally recovered {} non-truncated lsns and {} truncated lsns for store {}", m_n_recovered_lsns, - m_n_recovered_truncated_lsns, m_log_store->get_store_id()); + LOGINFO("Totally recovered {} non-truncated lsns and {} truncated lsns for store {} log_dev {}", + m_n_recovered_lsns, m_n_recovered_truncated_lsns, m_log_store->get_store_id(), + m_log_store->get_logdev()->get_id()); if (m_n_recovered_lsns != (m_cur_lsn.load() - m_truncated_upto_lsn.load() - 1)) { EXPECT_EQ(m_n_recovered_lsns, m_cur_lsn.load() - m_truncated_upto_lsn.load() - 1) << "Recovered " << m_n_recovered_lsns << " valid lsns for store " << m_log_store->get_store_id() @@ -1296,6 +1297,8 @@ int main(int argc, char* argv[]) { SISL_OPTIONS_LOAD(parsed_argc, argv, logging, test_log_store, iomgr, test_common_setup); sisl::logging::SetLogger("test_log_store"); spdlog::set_pattern("[%D %T%z] [%^%l%$] [%t] %v"); + sisl::logging::SetModuleLogLevel("logstore", spdlog::level::level_enum::trace); + sisl::logging::SetModuleLogLevel("journalvdev", spdlog::level::level_enum::debug); SampleDB::instance().start_homestore(); const int ret = RUN_ALL_TESTS();