1. Implement shard identical layout using blk_alloc_hints and add v_c…

…hunk_id field to shard_info_superblk.

2. Enhance defensive checks in ChunkSelector for input validation and exception handling.

3. Adapt unit tests and introduce PGExceedSpaceTest.

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.1.8"
+    version = "2.1.9"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"

src/include/homeobject/pg_manager.hpp

@@ -42,6 +42,7 @@ struct PGInfo {
     mutable MemberSet members;
     peer_id_t replica_set_uuid;
     uint64_t size;
+    uint64_t chunk_size;
 
     auto operator<=>(PGInfo const& rhs) const { return id <=> rhs.id; }
     auto operator==(PGInfo const& rhs) const { return id == rhs.id; }

src/include/homeobject/shard_manager.hpp

@@ -10,7 +10,7 @@
 namespace homeobject {
 
 ENUM(ShardError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNSUPPORTED_OP, UNKNOWN_PG, UNKNOWN_SHARD,
-     PG_NOT_READY, CRC_MISMATCH);
+     PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT);
 
 struct ShardInfo {
     enum class State : uint8_t {

src/lib/homestore_backend/heap_chunk_selector.h

@@ -35,7 +35,9 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     using VChunkHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkComparator >;
     using VChunkDefragHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkDefragComparator >;
-    using ChunkIdMap = std::unordered_map < homestore::chunk_num_t, homestore::chunk_num_t >; // used for real chunk id -> virtual chunk id map
+    using ChunkIdMap =
+        std::unordered_map< homestore::chunk_num_t,
+                            homestore::chunk_num_t >; // used for physical chunk id -> virtual chunk id map
     using chunk_num_t = homestore::chunk_num_t;
 
     struct ChunkHeap {
@@ -52,16 +54,16 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     csharedChunk select_chunk([[maybe_unused]] homestore::blk_count_t nblks, const homestore::blk_alloc_hints& hints);
 
-    // this function will be used by GC flow or recovery flow to mark one specific chunk to be busy, caller should be
-    // responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
-    csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t);
+    // this function will be used by create shard or recovery flow to mark one specific chunk to be busy, caller should
+    // be responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
+    csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
 
     // this function will be used by GC flow to select a chunk for GC
     csharedChunk most_defrag_chunk();
 
-    // this function is used to return a chunk back to ChunkSelector when sealing a shard, and will only be used by
-    // Homeobject.
-    void release_chunk(const pg_id_t pg_id, const chunk_num_t);
+    // This function returns a chunk back to ChunkSelector.
+    // It is used in two scenarios: 1. seal shard  2. create shard rollback
+    bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);
 
     /**
      * select chunks for pg, chunks need to be in same pdev.
@@ -70,26 +72,35 @@ class HeapChunkSelector : public homestore::ChunkSelector {
      * @param pg_size The fix pg size.
      * @return An optional uint32_t value representing num_chunk, or std::nullopt if no space left.
      */
-    std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, u_int64_t pg_size);
+    std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, uint64_t pg_size);
 
-    std::shared_ptr< const std::vector <chunk_num_t> > get_pg_chunks(pg_id_t pg_id) const;
+    // this function is used for pg info superblk persist v_chunk_id <-> p_chunk_id
+    std::shared_ptr< const std::vector< chunk_num_t > > get_pg_chunks(pg_id_t pg_id) const;
+
+    /**
+     * pop pg top chunk
+     *
+     * @param pg_id The ID of the pg.
+     * @return An optional chunk_num_t value representing v_chunk_id, or std::nullopt if no space left.
+     */
+    std::optional< chunk_num_t > pop_top_chunk(pg_id_t pg_id) const;
 
     // this should be called on each pg meta blk found
-    void set_pg_chunks(pg_id_t pg_id, std::vector<chunk_num_t>&& chunk_ids);
+    bool set_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);
 
     // this should be called after all pg meta blk recovered
     void recover_per_dev_chunk_heap();
 
     // this should be called after ShardManager is initialized and get all the open shards
-    void recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excludingChunks);
+    bool recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids);
 
     /**
      * Retrieves the block allocation hints for a given chunk.
      *
-     * @param chunk_id The ID of the chunk.
+     * @param p_chunk_id The physical chunk id.
      * @return The block allocation hints for the specified chunk.
      */
-    homestore::blk_alloc_hints chunk_to_hints(chunk_num_t chunk_id) const;
+    homestore::blk_alloc_hints chunk_to_hints(chunk_num_t p_chunk_id) const;
 
     /**
      * Returns the number of available blocks of the given device id.
@@ -134,25 +145,33 @@ class HeapChunkSelector : public homestore::ChunkSelector {
 
     uint32_t get_chunk_size() const;
 
+private:
+    std::optional< chunk_num_t > get_virtual_chunk_id(pg_id_t pg_id, chunk_num_t p_chunk_id) const;
+
+    std::optional< chunk_num_t > get_physical_chunk_id(pg_id_t pg_id, chunk_num_t v_chunk_id) const;
+
+    void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);
+
+    void remove_chunk_from_defrag_heap(const chunk_num_t);
+
 private:
     std::unordered_map< uint32_t, std::shared_ptr< ChunkHeap > > m_per_dev_heap;
     std::unordered_map< pg_id_t, std::shared_ptr< ChunkHeap > > m_per_pg_heap;
 
-    // These mappings ensure "identical layout" by providing bidirectional indexing between virtual and real chunk IDs.
-    // m_v2r_chunk_map: Maps each pg_id to a vector of real chunk IDs (r_chunk_id). The index in the vector corresponds to the virtual chunk ID (v_chunk_id).
-    std::unordered_map< pg_id_t, std::shared_ptr< std::vector <chunk_num_t> > > m_v2r_chunk_map;
-    // m_r2v_chunk_map: Maps each pg_id to a map that inversely maps real chunk IDs (r_chunk_id) to virtual chunk IDs (v_chunk_id).
-    std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_r2v_chunk_map;
+    // These mappings ensure "identical layout" by providing bidirectional indexing between virtual and physical chunk
+    // IDs. m_v2p_chunk_map: Maps each pg_id to a vector of physical chunk IDs (p_chunk_id). The index in the vector
+    // corresponds to the virtual chunk ID (v_chunk_id).
+    std::unordered_map< pg_id_t, std::shared_ptr< std::vector< chunk_num_t > > > m_v2p_chunk_map;
+    // m_p2v_chunk_map: Maps each pg_id to a map that inversely maps physical chunk IDs (p_chunk_id) to virtual chunk
+    // IDs (v_chunk_id).
+    std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_p2v_chunk_map;
 
     // hold all the chunks , selected or not
     std::unordered_map< chunk_num_t, csharedChunk > m_chunks;
 
     mutable std::shared_mutex m_chunk_selector_mtx;
-    void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);
 
     VChunkDefragHeap m_defrag_heap;
     std::mutex m_defrag_mtx;
-
-    void remove_chunk_from_defrag_heap(const chunk_num_t);
 };
 } // namespace homeobject

src/lib/homestore_backend/hs_blob_manager.cpp

@@ -351,10 +351,10 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive<
     }
 
     auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get());
-    BLOGD(msg_header->shard_id, "n/a", "Picked chunk_id={}", hs_shard->sb_->chunk_id);
+    BLOGD(msg_header->shard_id, "n/a", "Picked p_chunk_id={}", hs_shard->sb_->p_chunk_id);
 
     homestore::blk_alloc_hints hints;
-    hints.chunk_id_hint = hs_shard->sb_->chunk_id;
+    hints.chunk_id_hint = hs_shard->sb_->p_chunk_id;
     return hints;
 }
 

src/lib/homestore_backend/hs_homeobject.hpp

@@ -90,9 +90,9 @@ class HSHomeObject : public HomeObjectImpl {
         // Data layout inside 'data':
         // First, an array of 'pg_members' structures:
         // | pg_members[0] | pg_members[1] | ... | pg_members[num_members-1] |
-        // Immediately followed by an array of 'chunk_num_t' values (representing r_chunk_ids):
+        // Immediately followed by an array of 'chunk_num_t' values (representing physical chunkID):
         // | chunk_num_t[0] | chunk_num_t[1] | ... | chunk_num_t[num_chunks-1] |
-        // Here, 'chunk_num_t[i]' represents the r_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and
+        // Here, 'chunk_num_t[i]' represents the p_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and
         // increases sequentially.
 
         uint32_t size() const {
@@ -148,7 +148,8 @@ class HSHomeObject : public HomeObjectImpl {
 
     struct shard_info_superblk : public DataHeader {
         ShardInfo info;
-        homestore::chunk_num_t chunk_id;
+        homestore::chunk_num_t p_chunk_id;
+        homestore::chunk_num_t v_chunk_id;
     };
 #pragma pack()
 
@@ -217,7 +218,6 @@ class HSHomeObject : public HomeObjectImpl {
     public:
         homestore::superblk< pg_info_superblk > pg_sb_;
         shared< homestore::ReplDev > repl_dev_;
-        std::optional< homestore::chunk_num_t > any_allocated_chunk_id_{};
         std::shared_ptr< BlobIndexTable > index_table_;
         PGMetrics metrics_;
 
@@ -252,12 +252,12 @@ class HSHomeObject : public HomeObjectImpl {
 
     struct HS_Shard : public Shard {
         homestore::superblk< shard_info_superblk > sb_;
-        HS_Shard(ShardInfo info, homestore::chunk_num_t chunk_id);
+        HS_Shard(ShardInfo info, homestore::chunk_num_t p_chunk_id);
         HS_Shard(homestore::superblk< shard_info_superblk >&& sb);
         ~HS_Shard() override = default;
 
         void update_info(const ShardInfo& info);
-        auto chunk_id() const { return sb_->chunk_id; }
+        auto p_chunk_id() const { return sb_->p_chunk_id; }
     };
 
 #pragma pack(1)
@@ -437,9 +437,17 @@ class HSHomeObject : public HomeObjectImpl {
      * @brief Retrieves the chunk number associated with the given shard ID.
      *
      * @param id The ID of the shard to retrieve the chunk number for.
-     * @return An optional chunk number if the shard ID is valid, otherwise an empty optional.
+     * @return An optional chunk number values shard p_chunk_id if the shard ID is valid, otherwise an empty optional.
      */
-    std::optional< homestore::chunk_num_t > get_shard_chunk(shard_id_t id) const;
+    std::optional< homestore::chunk_num_t > get_shard_p_chunk_id(shard_id_t id) const;
+
+    /**
+     * @brief Retrieves the chunk number associated with the given shard ID.
+     *
+     * @param id The ID of the shard to retrieve the chunk number for.
+     * @return An optional chunk number values shard v_chunk_id if the shard ID is valid, otherwise an empty optional.
+     */
+    std::optional< homestore::chunk_num_t > get_shard_v_chunk_id(shard_id_t id) const;
 
     /**
      * @brief recover PG and shard from the superblock.
@@ -448,12 +456,14 @@ class HSHomeObject : public HomeObjectImpl {
     void on_replica_restart();
 
     /**
-     * @brief Returns any chunk number for the given pg ID.
+     * @brief Extracts the physical chunk ID for create shard from the message.
      *
-     * @param pg The pg ID to get the chunk number for.
-     * @return A tuple of <if pg exist, if shard exist, chunk number if both exist>.
+     * @param header The message header that includes the shard_info_superblk, which contains the data necessary for
+     * extracting and mapping the chunk ID.
+     * @return An optional virtual chunk id if the extraction and mapping process is successful, otherwise an empty
+     * optional.
      */
-    std::tuple< bool, bool, homestore::chunk_num_t > get_any_chunk_id(pg_id_t pg);
+    std::optional< homestore::chunk_num_t > resolve_v_chunk_id_from_msg(sisl::blob const& header);
 
     bool pg_exists(pg_id_t pg_id) const;
 

src/lib/homestore_backend/hs_pg_manager.cpp

@@ -69,6 +69,7 @@ PGManager::NullAsyncResult HSHomeObject::_create_pg(PGInfo&& pg_info, std::set<
         return folly::makeUnexpected(PGError::NO_SPACE_LEFT);
     }
 
+    pg_info.chunk_size = chunk_size;
     pg_info.replica_set_uuid = boost::uuids::random_generator()();
     return hs_repl_service()
         .create_repl_dev(pg_info.replica_set_uuid, peers)
@@ -136,15 +137,27 @@ void HSHomeObject::on_create_pg_message_commit(int64_t lsn, sisl::blob const& he
         return;
     }
 
+    auto local_chunk_size = chunk_selector()->get_chunk_size();
+    if (pg_info.chunk_size != local_chunk_size) {
+        LOGE("Chunk sizes are inconsistent, leader_chunk_size={}, local_chunk_size={}", pg_info.chunk_size,
+             local_chunk_size);
+        if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::UNKNOWN)); }
+        return;
+    }
+
     // select chunks for pg
     auto const num_chunk = chunk_selector()->select_chunks_for_pg(pg_id, pg_info.size);
     if (!num_chunk.has_value()) {
-        LOGW("select chunks for pg failed, pg_id {}", pg_id);
+        LOGW("Failed to select chunks for pg {}", pg_id);
         if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); }
         return;
     }
     auto chunk_ids = chunk_selector()->get_pg_chunks(pg_id);
-
+    if (chunk_ids == nullptr) {
+        LOGW("Failed to get pg chunks, pg_id {}", pg_id);
+        if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); }
+        return;
+    }
     // create index table and pg
     // TODO create index table during create shard.
     auto index_table = create_index_table();
@@ -245,6 +258,7 @@ std::string HSHomeObject::serialize_pg_info(const PGInfo& pginfo) {
     nlohmann::json j;
     j["pg_info"]["pg_id_t"] = pginfo.id;
     j["pg_info"]["pg_size"] = pginfo.size;
+    j["pg_info"]["chunk_size"] = pginfo.chunk_size;
     j["pg_info"]["repl_uuid"] = boost::uuids::to_string(pginfo.replica_set_uuid);
 
     nlohmann::json members_j{};
@@ -263,7 +277,8 @@ PGInfo HSHomeObject::deserialize_pg_info(const unsigned char* json_str, size_t s
     auto pg_json = nlohmann::json::parse(json_str, json_str + size);
 
     PGInfo pg_info(pg_json["pg_info"]["pg_id_t"].get< pg_id_t >());
-    pg_info.size = pg_json["pg_info"]["pg_size"].get< u_int64_t >();
+    pg_info.size = pg_json["pg_info"]["pg_size"].get< uint64_t >();
+    pg_info.chunk_size = pg_json["pg_info"]["chunk_size"].get< uint64_t >();
     pg_info.replica_set_uuid = boost::uuids::string_generator()(pg_json["pg_info"]["repl_uuid"].get< std::string >());
 
     for (auto const& m : pg_json["pg_info"]["members"]) {
@@ -287,8 +302,9 @@ void HSHomeObject::on_pg_meta_blk_found(sisl::byte_view const& buf, void* meta_c
         return;
     }
     auto pg_id = pg_sb->id;
-    std::vector< chunk_num_t > chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks);
-    chunk_selector_->set_pg_chunks(pg_id, std::move(chunk_ids));
+    std::vector< chunk_num_t > p_chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks);
+    bool set_pg_chunks_res = chunk_selector_->set_pg_chunks(pg_id, std::move(p_chunk_ids));
+    RELEASE_ASSERT(set_pg_chunks_res, "Failed to set pg={} chunks", pg_id);
     auto uuid_str = boost::uuids::to_string(pg_sb->index_table_uuid);
     auto hs_pg = std::make_unique< HS_PG >(std::move(pg_sb), std::move(v.value()));
     // During PG recovery check if index is already recoverd else
@@ -368,7 +384,7 @@ uint32_t HSHomeObject::HS_PG::open_shards() const {
 std::optional< uint32_t > HSHomeObject::HS_PG::dev_hint(cshared< HeapChunkSelector > chunk_sel) const {
     if (shards_.empty()) { return std::nullopt; }
     auto const hs_shard = d_cast< HS_Shard* >(shards_.front().get());
-    auto const hint = chunk_sel->chunk_to_hints(hs_shard->chunk_id());
+    auto const hint = chunk_sel->chunk_to_hints(hs_shard->p_chunk_id());
     return hint.pdev_id_hint;
 }