From fcb47c0fbdf303b71cd5d1d38ec2fc69b9dc6886 Mon Sep 17 00:00:00 2001 From: Yaming Kuang <1477567+yamingk@users.noreply.github.com> Date: Wed, 25 Oct 2023 04:02:04 -0700 Subject: [PATCH] get defrag nblks api (#209) --- conanfile.py | 2 +- src/include/homestore/homestore.hpp | 2 ++ src/lib/blkalloc/append_blk_allocator.cpp | 10 +++++--- src/lib/blkalloc/append_blk_allocator.h | 25 +++++++++++++++++++ src/lib/homestore.cpp | 19 ++++++++++++++ src/tests/test_append_blkalloc.cpp | 6 +++-- .../test_common/homestore_test_common.hpp | 2 ++ 7 files changed, 59 insertions(+), 7 deletions(-) diff --git a/conanfile.py b/conanfile.py index b35bbb3e8..7d4a73a3a 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "4.5.7" + version = "4.5.8" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/include/homestore/homestore.hpp b/src/include/homestore/homestore.hpp index cd4b46f8e..c27594a87 100644 --- a/src/include/homestore/homestore.hpp +++ b/src/include/homestore/homestore.hpp @@ -125,6 +125,8 @@ class HomeStore { HS_SERVICE m_services; // Services homestore is starting with hs_before_services_starting_cb_t m_before_services_starting_cb{nullptr}; + bool m_init_done{false}; + public: HomeStore() = default; virtual ~HomeStore() = default; diff --git a/src/lib/blkalloc/append_blk_allocator.cpp b/src/lib/blkalloc/append_blk_allocator.cpp index 7601caaa9..6505eca49 100644 --- a/src/lib/blkalloc/append_blk_allocator.cpp +++ b/src/lib/blkalloc/append_blk_allocator.cpp @@ -164,10 +164,6 @@ void AppendBlkAllocator::free(const BlkId& bid) { set_dirty_offset(cur_cp->id() % MAX_CP_COUNT); } -blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); } - -blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; } - bool AppendBlkAllocator::is_blk_alloced(const BlkId& in_bid, bool) const { // blk_num starts from 0; return in_bid.blk_num() < get_used_blks(); @@ -179,6 +175,12 @@ std::string AppendBlkAllocator::to_string() const { return fmt::format("{}, last_append_offset: {}", get_name(), m_last_append_offset); } +blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); } + +blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; } + blk_num_t AppendBlkAllocator::get_freeable_nblks() const { return m_freeable_nblks; } +blk_num_t AppendBlkAllocator::get_defrag_nblks() const { return get_freeable_nblks() - available_blks(); } + } // namespace homestore diff --git a/src/lib/blkalloc/append_blk_allocator.h b/src/lib/blkalloc/append_blk_allocator.h index 15c573113..1818ab8e8 100644 --- a/src/lib/blkalloc/append_blk_allocator.h +++ b/src/lib/blkalloc/append_blk_allocator.h @@ -79,11 +79,36 @@ class AppendBlkAllocator : public BlkAllocator { BlkAllocStatus alloc(blk_count_t nblks, blk_alloc_hints const& hints, BlkId& out_blkid) override; void free(BlkId const& b) override; + /** + * @brief : the number of available blocks that can be allocated by the AppendBlkAllocator. + * @return : the number of available blocks. + */ blk_num_t available_blks() const override; + + /** + * @brief : the number of used blocks by the AppendBlkAllocator. + * @return : the number of used blocks. + */ blk_num_t get_used_blks() const override; + + /** + * @brief : the number of freeable blocks by the AppendBlkAllocator. + * @return : the number of freeable blocks. + */ blk_num_t get_freeable_nblks() const; + /** + * @brief : the number of blocks that have been allocated by the AppendBlkAllocator. + * @return : the number of allocated blocks. + */ + blk_num_t get_defrag_nblks() const; + + /** + * @brief : check if the input blk id is allocated or not. + * @return : true if blkid is allocated, false if not; + */ bool is_blk_alloced(const BlkId& in_bid, bool use_lock = false) const override; + std::string to_string() const override; /// @brief : needs to be called with cp_guard(); diff --git a/src/lib/homestore.cpp b/src/lib/homestore.cpp index 502547e19..e75bb2fbf 100644 --- a/src/lib/homestore.cpp +++ b/src/lib/homestore.cpp @@ -145,6 +145,16 @@ bool HomeStore::start(const hs_input_params& input, hs_before_services_starting_ } void HomeStore::format_and_start(std::map< uint32_t, hs_format_params >&& format_opts) { + auto total_pct_sum = 0.0f; + for (const auto& [svc_type, fparams] : format_opts) { + total_pct_sum += fparams.size_pct; + } + + if (total_pct_sum > 100.0f) { + LOGERROR("Total percentage of all services is greater than 100.0f, total_pct_sum={}", total_pct_sum); + throw std::invalid_argument("total percentage of all services is greater than 100.0f"); + } + m_dev_mgr->format_devices(); hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast})); @@ -209,9 +219,18 @@ void HomeStore::do_start() { // In case of custom recovery, let consumer starts the recovery and it is consumer module's responsibilities // to start log store if (has_log_service() && inp_params.auto_recovery) { m_log_service->start(is_first_time_boot() /* format */); } + + init_done(); } +void HomeStore::init_done() { m_init_done = true; } + void HomeStore::shutdown() { + if (!m_init_done) { + LOGWARN("Homestore shutdown is called before init is completed"); + return; + } + LOGINFO("Homestore shutdown is started"); if (has_index_service()) { diff --git a/src/tests/test_append_blkalloc.cpp b/src/tests/test_append_blkalloc.cpp index 43c7a50aa..c1073691f 100644 --- a/src/tests/test_append_blkalloc.cpp +++ b/src/tests/test_append_blkalloc.cpp @@ -69,7 +69,8 @@ class AppendBlkAllocatorTest : public testing::Test { test_common::HSTestHelper::start_homestore( "test_append_blkalloc", {{HS_SERVICE::META, {.size_pct = 5.0}}, - {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}}); + {HS_SERVICE::DATA, + {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}}); } virtual void TearDown() override { test_common::HSTestHelper::shutdown_homestore(); } @@ -78,7 +79,8 @@ class AppendBlkAllocatorTest : public testing::Test { test_common::HSTestHelper::start_homestore( "test_append_blkalloc", {{HS_SERVICE::META, {.size_pct = 5.0}}, - {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}}, + {HS_SERVICE::DATA, + {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}}, nullptr /* before_svc_start_cb */, true /* restart */); } diff --git a/src/tests/test_common/homestore_test_common.hpp b/src/tests/test_common/homestore_test_common.hpp index cb93db64b..bdac774eb 100644 --- a/src/tests/test_common/homestore_test_common.hpp +++ b/src/tests/test_common/homestore_test_common.hpp @@ -98,6 +98,7 @@ class HSTestHelper { shared< ChunkSelector > custom_chunk_selector{nullptr}; IndexServiceCallbacks* index_svc_cbs{nullptr}; repl_impl_type repl_impl{repl_impl_type::solo}; + chunk_num_t num_chunks{1}; }; #if 0 @@ -182,6 +183,7 @@ class HSTestHelper { {HS_SERVICE::LOG_LOCAL, {.size_pct = svc_params[HS_SERVICE::LOG_LOCAL].size_pct}}, {HS_SERVICE::DATA, {.size_pct = svc_params[HS_SERVICE::DATA].size_pct, + .num_chunks = svc_params[HS_SERVICE::DATA].num_chunks, .alloc_type = svc_params[HS_SERVICE::DATA].blkalloc_type, .chunk_sel_type = svc_params[HS_SERVICE::DATA].custom_chunk_selector ? chunk_selector_type_t::CUSTOM