1. Implement shard identical layout using blk_alloc_hints and add v_c…

…hunk_id field to shard_info_superblk. 2. Enhance defensive checks in ChunkSelector for input validation and exception handling. 3. Adapt unit tests and introduce PGExceedSpaceTest.
eBay · Nov 20, 2024 · 2aeb7eb · 2aeb7eb
1 parent 20d84b4
commit 2aeb7eb
Show file tree

Hide file tree

Showing 13 changed files with 690 additions and 522 deletions.
diff --git a/conanfile.py b/conanfile.py
@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.1.8"
+    version = "2.1.9"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"

diff --git a/src/include/homeobject/pg_manager.hpp b/src/include/homeobject/pg_manager.hpp
@@ -42,6 +42,7 @@ struct PGInfo {
     mutable MemberSet members;
     peer_id_t replica_set_uuid;
     uint64_t size;
+    uint64_t chunk_size;
 
     auto operator<=>(PGInfo const& rhs) const { return id <=> rhs.id; }
     auto operator==(PGInfo const& rhs) const { return id == rhs.id; }

diff --git a/src/include/homeobject/shard_manager.hpp b/src/include/homeobject/shard_manager.hpp
@@ -10,7 +10,7 @@
 namespace homeobject {
 
 ENUM(ShardError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNSUPPORTED_OP, UNKNOWN_PG, UNKNOWN_SHARD,
-     PG_NOT_READY, CRC_MISMATCH);
+     PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT);
 
 struct ShardInfo {
     enum class State : uint8_t {

diff --git a/src/lib/homestore_backend/heap_chunk_selector.cpp b/src/lib/homestore_backend/heap_chunk_selector.cpp
diff --git a/src/lib/homestore_backend/heap_chunk_selector.h b/src/lib/homestore_backend/heap_chunk_selector.h
@@ -6,6 +6,7 @@
 #include <homestore/vchunk.h>
 #include <homestore/homestore_decl.hpp>
 #include <homestore/blk.h>
+#include <sisl/utility/enum.hpp>
 
 #include <queue>
 #include <vector>
@@ -15,6 +16,8 @@
 
 namespace homeobject {
 
+ENUM(ChunkState, uint8_t, AVAILABLE = 0, INUSE);
+
 using csharedChunk = homestore::cshared< homestore::Chunk >;
 
 class HeapChunkSelector : public homestore::ChunkSelector {
@@ -23,45 +26,58 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     ~HeapChunkSelector() = default;
 
     using VChunk = homestore::VChunk;
-    class VChunkComparator {
+    using chunk_num_t = homestore::chunk_num_t;
+
+    class ExtendedVChunk : public VChunk {
     public:
-        bool operator()(VChunk& lhs, VChunk& rhs) { return lhs.available_blks() < rhs.available_blks(); }
+        ExtendedVChunk(csharedChunk const& chunk) :
+                VChunk(chunk), m_state(ChunkState::AVAILABLE), m_pg_id(), m_v_chunk_id() {}
+        ~ExtendedVChunk() = default;
+        ChunkState m_state;
+        std::optional< pg_id_t > m_pg_id;
+        std::optional< chunk_num_t > m_v_chunk_id;
+        bool available() const { return m_state == ChunkState::AVAILABLE; }
     };
 
-    class VChunkDefragComparator {
+    class ExtendedVChunkComparator {
     public:
-        bool operator()(VChunk& lhs, VChunk& rhs) { return lhs.get_defrag_nblks() < rhs.get_defrag_nblks(); }
+        bool operator()(std::shared_ptr< ExtendedVChunk >& lhs, std::shared_ptr< ExtendedVChunk >& rhs) {
+            return lhs->available_blks() < rhs->available_blks();
+        }
     };
-
-    using VChunkHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkComparator >;
-    using VChunkDefragHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkDefragComparator >;
-    using ChunkIdMap = std::unordered_map < homestore::chunk_num_t, homestore::chunk_num_t >; // used for real chunk id -> virtual chunk id map
-    using chunk_num_t = homestore::chunk_num_t;
+    using ExtendedVChunkHeap =
+        std::priority_queue< std::shared_ptr< ExtendedVChunk >, std::vector< std::shared_ptr< ExtendedVChunk > >,
+                             ExtendedVChunkComparator >;
 
     struct ChunkHeap {
         std::mutex mtx;
-        VChunkHeap m_heap;
+        ExtendedVChunkHeap m_heap;
         std::atomic_size_t available_blk_count;
         uint64_t m_total_blks{0}; // initlized during boot, and will not change during runtime;
         uint32_t size() const { return m_heap.size(); }
     };
 
+    struct PGChunkCollection {
+        std::mutex mtx;
+        std::vector< std::shared_ptr< ExtendedVChunk > > m_pg_chunks;
+        std::atomic_size_t available_num_chunks;
+        std::atomic_size_t available_blk_count;
+        uint64_t m_total_blks{0}; // initlized during boot, and will not change during runtime;
+    };
+
     void add_chunk(csharedChunk&) override;
 
     void foreach_chunks(std::function< void(csharedChunk&) >&& cb) override;
 
     csharedChunk select_chunk([[maybe_unused]] homestore::blk_count_t nblks, const homestore::blk_alloc_hints& hints);
 
-    // this function will be used by GC flow or recovery flow to mark one specific chunk to be busy, caller should be
-    // responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
-    csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t);
-
-    // this function will be used by GC flow to select a chunk for GC
-    csharedChunk most_defrag_chunk();
+    // this function will be used by create shard or recovery flow to mark one specific chunk to be busy, caller should
+    // be responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
+    csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
 
-    // this function is used to return a chunk back to ChunkSelector when sealing a shard, and will only be used by
-    // Homeobject.
-    void release_chunk(const pg_id_t pg_id, const chunk_num_t);
+    // This function returns a chunk back to ChunkSelector.
+    // It is used in two scenarios: 1. seal shard  2. create shard rollback
+    bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
 
     /**
      * select chunks for pg, chunks need to be in same pdev.
@@ -70,34 +86,35 @@ class HeapChunkSelector : public homestore::ChunkSelector {
      * @param pg_size The fix pg size.
      * @return An optional uint32_t value representing num_chunk, or std::nullopt if no space left.
      */
-    std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, u_int64_t pg_size);
+    std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, uint64_t pg_size);
 
-    std::shared_ptr< const std::vector <chunk_num_t> > get_pg_chunks(pg_id_t pg_id) const;
+    // this function is used for pg info superblk persist v_chunk_id <-> p_chunk_id
+    std::shared_ptr< const std::vector< chunk_num_t > > get_pg_chunks(pg_id_t pg_id) const;
+
+    /**
+     * pop pg top chunk
+     *
+     * @param pg_id The ID of the pg.
+     * @return An optional chunk_num_t value representing v_chunk_id, or std::nullopt if no space left.
+     */
+    std::optional< chunk_num_t > get_most_available_blk_chunk(pg_id_t pg_id) const;
 
     // this should be called on each pg meta blk found
-    void set_pg_chunks(pg_id_t pg_id, std::vector<chunk_num_t>&& chunk_ids);
+    bool recover_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);
 
     // this should be called after all pg meta blk recovered
     void recover_per_dev_chunk_heap();
 
     // this should be called after ShardManager is initialized and get all the open shards
-    void recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excludingChunks);
-
-    /**
-     * Retrieves the block allocation hints for a given chunk.
-     *
-     * @param chunk_id The ID of the chunk.
-     * @return The block allocation hints for the specified chunk.
-     */
-    homestore::blk_alloc_hints chunk_to_hints(chunk_num_t chunk_id) const;
+    bool recover_pg_chunks_states(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids);
 
     /**
      * Returns the number of available blocks of the given device id.
      *
      * @param dev_id (optional) The device ID. if nullopt, it returns the maximum available blocks among all devices.
      * @return The number of available blocks.
      */
-    uint64_t avail_blks(std::optional< uint32_t > dev_id) const;
+    uint64_t avail_blks(pg_id_t pg_id) const;
 
     /**
      * Returns the total number of blocks of the given device;
@@ -116,12 +133,12 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     uint32_t most_avail_num_chunks() const;
 
     /**
-     * Returns the number of available chunks for a given device ID.
+     * Returns the number of available chunks for a given pg id.
      *
-     * @param dev_id The device ID.
+     * @param pg_id The pg id.
      * @return The number of available chunks.
      */
-    uint32_t avail_num_chunks(uint32_t dev_id) const;
+    uint32_t avail_num_chunks(pg_id_t pg_id) const;
 
     /**
      * @brief Returns the total number of chunks.
@@ -135,24 +152,15 @@ class HeapChunkSelector : public homestore::ChunkSelector {
     uint32_t get_chunk_size() const;
 
 private:
-    std::unordered_map< uint32_t, std::shared_ptr< ChunkHeap > > m_per_dev_heap;
-    std::unordered_map< pg_id_t, std::shared_ptr< ChunkHeap > > m_per_pg_heap;
+    void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);
 
-    // These mappings ensure "identical layout" by providing bidirectional indexing between virtual and real chunk IDs.
-    // m_v2r_chunk_map: Maps each pg_id to a vector of real chunk IDs (r_chunk_id). The index in the vector corresponds to the virtual chunk ID (v_chunk_id).
-    std::unordered_map< pg_id_t, std::shared_ptr< std::vector <chunk_num_t> > > m_v2r_chunk_map;
-    // m_r2v_chunk_map: Maps each pg_id to a map that inversely maps real chunk IDs (r_chunk_id) to virtual chunk IDs (v_chunk_id).
-    std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_r2v_chunk_map;
+private:
+    std::unordered_map< uint32_t, std::shared_ptr< ChunkHeap > > m_per_dev_heap;
 
+    std::unordered_map< pg_id_t, std::shared_ptr< PGChunkCollection > > m_per_pg_chunks;
     // hold all the chunks , selected or not
-    std::unordered_map< chunk_num_t, csharedChunk > m_chunks;
+    std::unordered_map< chunk_num_t, homestore::cshared< ExtendedVChunk > > m_chunks;
 
     mutable std::shared_mutex m_chunk_selector_mtx;
-    void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);
-
-    VChunkDefragHeap m_defrag_heap;
-    std::mutex m_defrag_mtx;
-
-    void remove_chunk_from_defrag_heap(const chunk_num_t);
 };
 } // namespace homeobject
diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp
@@ -351,10 +351,10 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive<
     }
 
     auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get());
-    BLOGD(msg_header->shard_id, "n/a", "Picked chunk_id={}", hs_shard->sb_->chunk_id);
+    BLOGD(msg_header->shard_id, "n/a", "Picked p_chunk_id={}", hs_shard->sb_->p_chunk_id);
 
     homestore::blk_alloc_hints hints;
-    hints.chunk_id_hint = hs_shard->sb_->chunk_id;
+    hints.chunk_id_hint = hs_shard->sb_->p_chunk_id;
     return hints;
 }
 

diff --git a/src/lib/homestore_backend/hs_homeobject.hpp b/src/lib/homestore_backend/hs_homeobject.hpp
@@ -90,9 +90,9 @@ class HSHomeObject : public HomeObjectImpl {
         // Data layout inside 'data':
         // First, an array of 'pg_members' structures:
         // | pg_members[0] | pg_members[1] | ... | pg_members[num_members-1] |
-        // Immediately followed by an array of 'chunk_num_t' values (representing r_chunk_ids):
+        // Immediately followed by an array of 'chunk_num_t' values (representing physical chunkID):
         // | chunk_num_t[0] | chunk_num_t[1] | ... | chunk_num_t[num_chunks-1] |
-        // Here, 'chunk_num_t[i]' represents the r_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and
+        // Here, 'chunk_num_t[i]' represents the p_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and
         // increases sequentially.
 
         uint32_t size() const {
@@ -148,7 +148,8 @@ class HSHomeObject : public HomeObjectImpl {
 
     struct shard_info_superblk : public DataHeader {
         ShardInfo info;
-        homestore::chunk_num_t chunk_id;
+        homestore::chunk_num_t p_chunk_id;
+        homestore::chunk_num_t v_chunk_id;
     };
 #pragma pack()
 
@@ -217,7 +218,6 @@ class HSHomeObject : public HomeObjectImpl {
     public:
         homestore::superblk< pg_info_superblk > pg_sb_;
         shared< homestore::ReplDev > repl_dev_;
-        std::optional< homestore::chunk_num_t > any_allocated_chunk_id_{};
         std::shared_ptr< BlobIndexTable > index_table_;
         PGMetrics metrics_;
 
@@ -240,24 +240,16 @@ class HSHomeObject : public HomeObjectImpl {
          * Returns the number of open shards on this PG.
          */
         uint32_t open_shards() const;
-
-        /**
-         * Retrieves the device hint associated with this PG(if any shard is created).
-         *
-         * @param selector The HeapChunkSelector object.
-         * @return An optional uint32_t value representing the device hint, or std::nullopt if no hint is available.
-         */
-        std::optional< uint32_t > dev_hint(cshared< HeapChunkSelector >) const;
     };
 
     struct HS_Shard : public Shard {
         homestore::superblk< shard_info_superblk > sb_;
-        HS_Shard(ShardInfo info, homestore::chunk_num_t chunk_id);
+        HS_Shard(ShardInfo info, homestore::chunk_num_t p_chunk_id);
         HS_Shard(homestore::superblk< shard_info_superblk >&& sb);
         ~HS_Shard() override = default;
 
         void update_info(const ShardInfo& info);
-        auto chunk_id() const { return sb_->chunk_id; }
+        auto p_chunk_id() const { return sb_->p_chunk_id; }
     };
 
 #pragma pack(1)
@@ -437,9 +429,17 @@ class HSHomeObject : public HomeObjectImpl {
      * @brief Retrieves the chunk number associated with the given shard ID.
      *
      * @param id The ID of the shard to retrieve the chunk number for.
-     * @return An optional chunk number if the shard ID is valid, otherwise an empty optional.
+     * @return An optional chunk number values shard p_chunk_id if the shard ID is valid, otherwise an empty optional.
+     */
+    std::optional< homestore::chunk_num_t > get_shard_p_chunk_id(shard_id_t id) const;
+
+    /**
+     * @brief Retrieves the chunk number associated with the given shard ID.
+     *
+     * @param id The ID of the shard to retrieve the chunk number for.
+     * @return An optional chunk number values shard v_chunk_id if the shard ID is valid, otherwise an empty optional.
      */
-    std::optional< homestore::chunk_num_t > get_shard_chunk(shard_id_t id) const;
+    std::optional< homestore::chunk_num_t > get_shard_v_chunk_id(shard_id_t id) const;
 
     /**
      * @brief recover PG and shard from the superblock.
@@ -448,13 +448,27 @@ class HSHomeObject : public HomeObjectImpl {
     void on_replica_restart();
 
     /**
-     * @brief Returns any chunk number for the given pg ID.
+     * @brief Extracts the physical chunk ID for create shard from the message.
+     *
+     * @param header The message header that includes the shard_info_superblk, which contains the data necessary for
+     * extracting and mapping the chunk ID.
+     * @return An optional virtual chunk id if the extraction and mapping process is successful, otherwise an empty
+     * optional.
+     */
+    std::optional< homestore::chunk_num_t > resolve_v_chunk_id_from_msg(sisl::blob const& header);
+
+    /**
+     * @brief Releases a chunk based on the information provided in a CREATE_SHARD message.
+     *
+     * This function is invoked during log rollback or when the proposer encounters an error.
+     * Its primary purpose is to ensure that the state of pg_chunks is reverted to the correct state.
      *
-     * @param pg The pg ID to get the chunk number for.
-     * @return A tuple of <if pg exist, if shard exist, chunk number if both exist>.
+     * @param header The message header that includes the shard_info_superblk, which contains the data necessary for
+     * extracting and mapping the chunk ID.
+     * @return Returns true if the chunk was successfully released, false otherwise.
      */
-    std::tuple< bool, bool, homestore::chunk_num_t > get_any_chunk_id(pg_id_t pg);
 
+    bool release_chunk_based_on_create_shard_message(sisl::blob const& header);
     bool pg_exists(pg_id_t pg_id) const;
 
     cshared< HeapChunkSelector > chunk_selector() const { return chunk_selector_; }