get defrag nblks api (#209)

conanfile.py

@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "4.5.7"
+    version = "4.5.8"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/include/homestore/homestore.hpp

@@ -125,6 +125,8 @@ class HomeStore {
     HS_SERVICE m_services; // Services homestore is starting with
     hs_before_services_starting_cb_t m_before_services_starting_cb{nullptr};
 
+    bool m_init_done{false};
+
 public:
     HomeStore() = default;
     virtual ~HomeStore() = default;

src/lib/blkalloc/append_blk_allocator.cpp

@@ -164,10 +164,6 @@ void AppendBlkAllocator::free(const BlkId& bid) {
     set_dirty_offset(cur_cp->id() % MAX_CP_COUNT);
 }
 
-blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); }
-
-blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; }
-
 bool AppendBlkAllocator::is_blk_alloced(const BlkId& in_bid, bool) const {
     // blk_num starts from 0;
     return in_bid.blk_num() < get_used_blks();
@@ -179,6 +175,12 @@ std::string AppendBlkAllocator::to_string() const {
     return fmt::format("{}, last_append_offset: {}", get_name(), m_last_append_offset);
 }
 
+blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); }
+
+blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; }
+
 blk_num_t AppendBlkAllocator::get_freeable_nblks() const { return m_freeable_nblks; }
 
+blk_num_t AppendBlkAllocator::get_defrag_nblks() const { return get_freeable_nblks() - available_blks(); }
+
 } // namespace homestore

src/lib/blkalloc/append_blk_allocator.h

@@ -79,11 +79,36 @@ class AppendBlkAllocator : public BlkAllocator {
     BlkAllocStatus alloc(blk_count_t nblks, blk_alloc_hints const& hints, BlkId& out_blkid) override;
     void free(BlkId const& b) override;
 
+    /**
+     * @brief :  the number of available blocks that can be allocated by the AppendBlkAllocator.
+     * @return : the number of available blocks.
+     */
     blk_num_t available_blks() const override;
+
+    /**
+     * @brief : the number of used blocks by the AppendBlkAllocator.
+     * @return : the number of used blocks.
+     */
     blk_num_t get_used_blks() const override;
+
+    /**
+     * @brief : the number of freeable blocks by the AppendBlkAllocator.
+     * @return : the number of freeable blocks.
+     */
     blk_num_t get_freeable_nblks() const;
 
+    /**
+     * @brief : the number of blocks that have been allocated by the AppendBlkAllocator.
+     * @return : the number of allocated blocks.
+     */
+    blk_num_t get_defrag_nblks() const;
+
+    /**
+     * @brief : check if the input blk id is allocated or not.
+     * @return : true if blkid is allocated, false if not;
+     */
     bool is_blk_alloced(const BlkId& in_bid, bool use_lock = false) const override;
+
     std::string to_string() const override;
 
     /// @brief : needs to be called with cp_guard();

src/lib/homestore.cpp

@@ -145,6 +145,16 @@ bool HomeStore::start(const hs_input_params& input, hs_before_services_starting_
 }
 
 void HomeStore::format_and_start(std::map< uint32_t, hs_format_params >&& format_opts) {
+    auto total_pct_sum = 0.0f;
+    for (const auto& [svc_type, fparams] : format_opts) {
+        total_pct_sum += fparams.size_pct;
+    }
+
+    if (total_pct_sum > 100.0f) {
+        LOGERROR("Total percentage of all services is greater than 100.0f, total_pct_sum={}", total_pct_sum);
+        throw std::invalid_argument("total percentage of all services is greater than 100.0f");
+    }
+
     m_dev_mgr->format_devices();
     hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
 
@@ -209,9 +219,18 @@ void HomeStore::do_start() {
     // In case of custom recovery, let consumer starts the recovery and it is consumer module's responsibilities
     // to start log store
     if (has_log_service() && inp_params.auto_recovery) { m_log_service->start(is_first_time_boot() /* format */); }
+
+    init_done();
 }
 
+void HomeStore::init_done() { m_init_done = true; }
+
 void HomeStore::shutdown() {
+    if (!m_init_done) { 
+        LOGWARN("Homestore shutdown is called before init is completed");
+        return; 
+    }
+
     LOGINFO("Homestore shutdown is started");
 
     if (has_index_service()) {

src/tests/test_append_blkalloc.cpp

@@ -69,7 +69,8 @@ class AppendBlkAllocatorTest : public testing::Test {
         test_common::HSTestHelper::start_homestore(
             "test_append_blkalloc",
             {{HS_SERVICE::META, {.size_pct = 5.0}},
-             {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}});
+             {HS_SERVICE::DATA,
+              {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}});
     }
 
     virtual void TearDown() override { test_common::HSTestHelper::shutdown_homestore(); }
@@ -78,7 +79,8 @@ class AppendBlkAllocatorTest : public testing::Test {
         test_common::HSTestHelper::start_homestore(
             "test_append_blkalloc",
             {{HS_SERVICE::META, {.size_pct = 5.0}},
-             {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}},
+             {HS_SERVICE::DATA,
+              {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}},
             nullptr /* before_svc_start_cb */, true /* restart */);
     }
 

src/tests/test_common/homestore_test_common.hpp

@@ -98,6 +98,7 @@ class HSTestHelper {
         shared< ChunkSelector > custom_chunk_selector{nullptr};
         IndexServiceCallbacks* index_svc_cbs{nullptr};
         repl_impl_type repl_impl{repl_impl_type::solo};
+        chunk_num_t num_chunks{1};
     };
 
 #if 0
@@ -182,6 +183,7 @@ class HSTestHelper {
                  {HS_SERVICE::LOG_LOCAL, {.size_pct = svc_params[HS_SERVICE::LOG_LOCAL].size_pct}},
                  {HS_SERVICE::DATA,
                   {.size_pct = svc_params[HS_SERVICE::DATA].size_pct,
+                   .num_chunks = svc_params[HS_SERVICE::DATA].num_chunks,
                    .alloc_type = svc_params[HS_SERVICE::DATA].blkalloc_type,
                    .chunk_sel_type = svc_params[HS_SERVICE::DATA].custom_chunk_selector
                        ? chunk_selector_type_t::CUSTOM