Merge branch 'master' into yk_ds_test

eBay · Nov 1, 2023 · a203eef · a203eef
2 parents a2cbcfe + ddff069
commit a203eef
Show file tree

Hide file tree

Showing 13 changed files with 97 additions and 7 deletions.
diff --git a/conanfile.py b/conanfile.py
@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "4.5.8"
+    version = "4.5.10"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

diff --git a/src/include/homestore/homestore.hpp b/src/include/homestore/homestore.hpp
@@ -125,6 +125,8 @@ class HomeStore {
     HS_SERVICE m_services; // Services homestore is starting with
     hs_before_services_starting_cb_t m_before_services_starting_cb{nullptr};
 
+    bool m_init_done{false};
+
 public:
     HomeStore() = default;
     virtual ~HomeStore() = default;

diff --git a/src/include/homestore/vchunk.h b/src/include/homestore/vchunk.h
@@ -30,6 +30,8 @@ class VChunk {
     void set_user_private(const sisl::blob& data);
     const uint8_t* get_user_private() const;
     blk_num_t available_blks() const;
+    blk_num_t get_freeable_nblks() const;
+    blk_num_t get_defrag_nblks() const;
     uint32_t get_pdev_id() const;
     uint16_t get_chunk_id() const;
     cshared< Chunk > get_internal_chunk() const;

diff --git a/src/lib/blkalloc/append_blk_allocator.cpp b/src/lib/blkalloc/append_blk_allocator.cpp
@@ -164,10 +164,6 @@ void AppendBlkAllocator::free(const BlkId& bid) {
     set_dirty_offset(cur_cp->id() % MAX_CP_COUNT);
 }
 
-blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); }
-
-blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; }
-
 bool AppendBlkAllocator::is_blk_alloced(const BlkId& in_bid, bool) const {
     // blk_num starts from 0;
     return in_bid.blk_num() < get_used_blks();
@@ -179,6 +175,12 @@ std::string AppendBlkAllocator::to_string() const {
     return fmt::format("{}, last_append_offset: {}", get_name(), m_last_append_offset);
 }
 
+blk_num_t AppendBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); }
+
+blk_num_t AppendBlkAllocator::get_used_blks() const { return m_last_append_offset; }
+
 blk_num_t AppendBlkAllocator::get_freeable_nblks() const { return m_freeable_nblks; }
 
+blk_num_t AppendBlkAllocator::get_defrag_nblks() const { return get_freeable_nblks() - available_blks(); }
+
 } // namespace homestore
diff --git a/src/lib/blkalloc/append_blk_allocator.h b/src/lib/blkalloc/append_blk_allocator.h
@@ -79,11 +79,36 @@ class AppendBlkAllocator : public BlkAllocator {
     BlkAllocStatus alloc(blk_count_t nblks, blk_alloc_hints const& hints, BlkId& out_blkid) override;
     void free(BlkId const& b) override;
 
+    /**
+     * @brief :  the number of available blocks that can be allocated by the AppendBlkAllocator.
+     * @return : the number of available blocks.
+     */
     blk_num_t available_blks() const override;
+
+    /**
+     * @brief : the number of used blocks by the AppendBlkAllocator.
+     * @return : the number of used blocks.
+     */
     blk_num_t get_used_blks() const override;
+
+    /**
+     * @brief : the number of freeable blocks by the AppendBlkAllocator.
+     * @return : the number of freeable blocks.
+     */
     blk_num_t get_freeable_nblks() const;
 
+    /**
+     * @brief : the number of blocks that have been allocated by the AppendBlkAllocator.
+     * @return : the number of allocated blocks.
+     */
+    blk_num_t get_defrag_nblks() const;
+
+    /**
+     * @brief : check if the input blk id is allocated or not.
+     * @return : true if blkid is allocated, false if not;
+     */
     bool is_blk_alloced(const BlkId& in_bid, bool use_lock = false) const override;
+
     std::string to_string() const override;
 
     /// @brief : needs to be called with cp_guard();

diff --git a/src/lib/blkalloc/blk_allocator.h b/src/lib/blkalloc/blk_allocator.h
@@ -186,6 +186,8 @@ class BlkAllocator {
     virtual BlkAllocStatus alloc(blk_count_t nblks, blk_alloc_hints const& hints, BlkId& out_blkid) = 0;
     virtual void free(BlkId const& id) = 0;
     virtual blk_num_t available_blks() const = 0;
+    virtual blk_num_t get_freeable_nblks() const = 0;
+    virtual blk_num_t get_defrag_nblks() const = 0;
     virtual blk_num_t get_used_blks() const = 0;
     virtual bool is_blk_alloced(BlkId const& b, bool use_lock = false) const = 0;
     virtual std::string to_string() const = 0;
@@ -319,6 +321,8 @@ class FixedBlkAllocator : public BlkAllocator {
     void inited() override;
 
     blk_num_t available_blks() const override;
+    blk_num_t get_freeable_nblks() const override;
+    blk_num_t get_defrag_nblks() const override;
     blk_num_t get_used_blks() const override;
     bool is_blk_alloced(BlkId const& in_bid, bool use_lock = false) const override;
     std::string to_string() const override;

diff --git a/src/lib/blkalloc/fixed_blk_allocator.cpp b/src/lib/blkalloc/fixed_blk_allocator.cpp
@@ -85,6 +85,19 @@ void FixedBlkAllocator::free(BlkId const& b) {
 }
 
 blk_num_t FixedBlkAllocator::available_blks() const { return m_blk_q.sizeGuess(); }
+
+blk_num_t FixedBlkAllocator::get_freeable_nblks() const {
+    // TODO: implement this
+    HS_DBG_ASSERT_EQ(false, true, "FixedBlkAllocator get_freeable_nblks Not implemented");
+    return 0;
+}
+
+blk_num_t FixedBlkAllocator::get_defrag_nblks() const {
+    // TODO: implement this
+    HS_DBG_ASSERT_EQ(false, true, "FixedBlkAllocator get_defrag_nblks Not implemented");
+    return 0;
+}
+
 blk_num_t FixedBlkAllocator::get_used_blks() const { return get_total_blks() - available_blks(); }
 
 std::string FixedBlkAllocator::to_string() const {

diff --git a/src/lib/blkalloc/varsize_blk_allocator.cpp b/src/lib/blkalloc/varsize_blk_allocator.cpp
@@ -731,6 +731,19 @@ bool VarsizeBlkAllocator::is_blk_alloced(BlkId const& bid, bool use_lock) const
 }
 
 blk_num_t VarsizeBlkAllocator::available_blks() const { return get_total_blks() - get_used_blks(); }
+
+blk_num_t VarsizeBlkAllocator::get_freeable_nblks() const {
+    // TODO: implement this
+    BLKALLOC_REL_ASSERT(false, "VarsizeBlkAllocator get_freeable_nblks Not implemented")
+    return 0;
+}
+
+blk_num_t VarsizeBlkAllocator::get_defrag_nblks() const {
+    // TODO: implement this
+    BLKALLOC_REL_ASSERT(false, "VarsizeBlkAllocator get_defrag_nblks Not implemented")
+    return 0;
+}
+
 blk_num_t VarsizeBlkAllocator::get_used_blks() const { return get_alloced_blk_count(); }
 
 #ifdef _PRERELEASE

diff --git a/src/lib/blkalloc/varsize_blk_allocator.h b/src/lib/blkalloc/varsize_blk_allocator.h
@@ -216,6 +216,8 @@ class VarsizeBlkAllocator : public BlkAllocator {
     void inited() override;
 
     blk_num_t available_blks() const override;
+    blk_num_t get_freeable_nblks() const override;
+    blk_num_t get_defrag_nblks() const override;
     blk_num_t get_used_blks() const override;
     bool is_blk_alloced(BlkId const& in_bid, bool use_lock = false) const override;
     std::string to_string() const override;

diff --git a/src/lib/device/vchunk.cpp b/src/lib/device/vchunk.cpp
@@ -25,6 +25,10 @@ const uint8_t* VChunk::get_user_private() const { return m_internal_chunk->user_
 
 blk_num_t VChunk::available_blks() const { return m_internal_chunk->blk_allocator()->available_blks(); }
 
+blk_num_t VChunk::get_freeable_nblks() const { return m_internal_chunk->blk_allocator()->get_freeable_nblks(); }
+
+blk_num_t VChunk::get_defrag_nblks() const { return m_internal_chunk->blk_allocator()->get_defrag_nblks(); }
+
 uint32_t VChunk::get_pdev_id() const { return m_internal_chunk->physical_dev()->pdev_id(); }
 
 uint16_t VChunk::get_chunk_id() const { return m_internal_chunk->chunk_id(); }

diff --git a/src/lib/homestore.cpp b/src/lib/homestore.cpp
@@ -145,6 +145,16 @@ bool HomeStore::start(const hs_input_params& input, hs_before_services_starting_
 }
 
 void HomeStore::format_and_start(std::map< uint32_t, hs_format_params >&& format_opts) {
+    auto total_pct_sum = 0.0f;
+    for (const auto& [svc_type, fparams] : format_opts) {
+        total_pct_sum += fparams.size_pct;
+    }
+
+    if (total_pct_sum > 100.0f) {
+        LOGERROR("Total percentage of all services is greater than 100.0f, total_pct_sum={}", total_pct_sum);
+        throw std::invalid_argument("total percentage of all services is greater than 100.0f");
+    }
+
     m_dev_mgr->format_devices();
     hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
 
@@ -209,9 +219,18 @@ void HomeStore::do_start() {
     // In case of custom recovery, let consumer starts the recovery and it is consumer module's responsibilities
     // to start log store
     if (has_log_service() && inp_params.auto_recovery) { m_log_service->start(is_first_time_boot() /* format */); }
+
+    init_done();
 }
 
+void HomeStore::init_done() { m_init_done = true; }
+
 void HomeStore::shutdown() {
+    if (!m_init_done) { 
+        LOGWARN("Homestore shutdown is called before init is completed");
+        return; 
+    }
+
     LOGINFO("Homestore shutdown is started");
 
     if (has_index_service()) {

diff --git a/src/tests/test_append_blkalloc.cpp b/src/tests/test_append_blkalloc.cpp
@@ -69,7 +69,8 @@ class AppendBlkAllocatorTest : public testing::Test {
         test_common::HSTestHelper::start_homestore(
             "test_append_blkalloc",
             {{HS_SERVICE::META, {.size_pct = 5.0}},
-             {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}});
+             {HS_SERVICE::DATA,
+              {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}});
     }
 
     virtual void TearDown() override { test_common::HSTestHelper::shutdown_homestore(); }
@@ -78,7 +79,8 @@ class AppendBlkAllocatorTest : public testing::Test {
         test_common::HSTestHelper::start_homestore(
             "test_append_blkalloc",
             {{HS_SERVICE::META, {.size_pct = 5.0}},
-             {HS_SERVICE::DATA, {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append}}},
+             {HS_SERVICE::DATA,
+              {.size_pct = 80.0, .blkalloc_type = homestore::blk_allocator_type_t::append, .num_chunks = 65000}}},
             nullptr /* before_svc_start_cb */, true /* restart */);
     }
 

diff --git a/src/tests/test_common/homestore_test_common.hpp b/src/tests/test_common/homestore_test_common.hpp
@@ -98,6 +98,7 @@ class HSTestHelper {
         shared< ChunkSelector > custom_chunk_selector{nullptr};
         IndexServiceCallbacks* index_svc_cbs{nullptr};
         repl_impl_type repl_impl{repl_impl_type::solo};
+        chunk_num_t num_chunks{1};
     };
 
 #if 0
@@ -182,6 +183,7 @@ class HSTestHelper {
                  {HS_SERVICE::LOG_LOCAL, {.size_pct = svc_params[HS_SERVICE::LOG_LOCAL].size_pct}},
                  {HS_SERVICE::DATA,
                   {.size_pct = svc_params[HS_SERVICE::DATA].size_pct,
+                   .num_chunks = svc_params[HS_SERVICE::DATA].num_chunks,
                    .alloc_type = svc_params[HS_SERVICE::DATA].blkalloc_type,
                    .chunk_sel_type = svc_params[HS_SERVICE::DATA].custom_chunk_selector
                        ? chunk_selector_type_t::CUSTOM