diff --git a/conanfile.py b/conanfile.py index f2702f3..8a04410 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "2.1.8" + version = "2.1.9" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" diff --git a/src/include/homeobject/pg_manager.hpp b/src/include/homeobject/pg_manager.hpp index 8c74f3d..4819052 100644 --- a/src/include/homeobject/pg_manager.hpp +++ b/src/include/homeobject/pg_manager.hpp @@ -42,6 +42,7 @@ struct PGInfo { mutable MemberSet members; peer_id_t replica_set_uuid; uint64_t size; + uint64_t chunk_size; auto operator<=>(PGInfo const& rhs) const { return id <=> rhs.id; } auto operator==(PGInfo const& rhs) const { return id == rhs.id; } diff --git a/src/include/homeobject/shard_manager.hpp b/src/include/homeobject/shard_manager.hpp index 812ebe9..19ee7e2 100644 --- a/src/include/homeobject/shard_manager.hpp +++ b/src/include/homeobject/shard_manager.hpp @@ -10,7 +10,7 @@ namespace homeobject { ENUM(ShardError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNSUPPORTED_OP, UNKNOWN_PG, UNKNOWN_SHARD, - PG_NOT_READY, CRC_MISMATCH); + PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT); struct ShardInfo { enum class State : uint8_t { diff --git a/src/lib/homestore_backend/heap_chunk_selector.cpp b/src/lib/homestore_backend/heap_chunk_selector.cpp index 6e2a149..36aa6ac 100644 --- a/src/lib/homestore_backend/heap_chunk_selector.cpp +++ b/src/lib/homestore_backend/heap_chunk_selector.cpp @@ -20,11 +20,7 @@ namespace homeobject { void HeapChunkSelector::add_chunk(csharedChunk& chunk) { m_chunks.emplace(VChunk(chunk).get_chunk_id(), chunk); } void HeapChunkSelector::add_chunk_internal(const chunk_num_t chunkID, bool add_to_heap) { - if (m_chunks.find(chunkID) == m_chunks.end()) { - // sanity check - LOGWARNMOD(homeobject, "No chunk found for ChunkID {}", chunkID); - return; - } + // private function chunkID must belong to m_chunks const auto& chunk = m_chunks[chunkID]; VChunk vchunk(chunk); @@ -59,56 +55,43 @@ csharedChunk HeapChunkSelector::select_chunk(homestore::blk_count_t count, const return nullptr; } - std::shared_lock lock_guard(m_chunk_selector_mtx); - // FIXME @Hooper: Temporary bypass using pdev_id_hint to represent pg_id_hint, "identical layout" will change it - pg_id_t pg_id = 0; - auto& pg_id_hint = hint.pdev_id_hint; - if (!pg_id_hint.has_value()) { - LOGWARNMOD(homeobject, "should not allocated a chunk without exiting pg_id in hint!"); + if (!hint.pdev_id_hint.has_value()) { + LOGWARNMOD(homeobject, "should not allocated a chunk without exiting pdev_id in hint!"); return nullptr; } else { - pg_id = pg_id_hint.value(); + // use pdev_id_hint (of type uint32_t) to store the values of pg_id and v_chunk_id. + // Both chunk_num_t and pg_id_t are of type uint16_t. + static_assert(std::is_same< pg_id_t, uint16_t >::value, "pg_id_t is not uint16_t"); + static_assert(std::is_same< homestore::chunk_num_t, uint16_t >::value, "chunk_num_t is not uint16_t"); + uint32_t pdev_id_hint = hint.pdev_id_hint.value(); + pg_id_t pg_id = (uint16_t)(pdev_id_hint >> 16 & 0xFFFF); + homestore::chunk_num_t v_chunk_id = (uint16_t)(pdev_id_hint & 0xFFFF); + return select_specific_chunk(pg_id, v_chunk_id); } +} - auto it = m_per_pg_heap.find(pg_id); - if (it == m_per_pg_heap.end()) { - LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); +csharedChunk HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id) { + std::shared_lock lock_guard(m_chunk_selector_mtx); + auto p_chunkID = get_physical_chunk_id(pg_id, v_chunk_id); + if (!p_chunkID.has_value()) { + LOGWARNMOD(homeobject, "No chunk found for v_chunk_id {} in pg {}", v_chunk_id, pg_id); return nullptr; } - auto vchunk = VChunk(nullptr); - auto& heap = it->second->m_heap; - if (auto lock_guard = std::lock_guard< std::mutex >(it->second->mtx); !heap.empty()) { - vchunk = heap.top(); - heap.pop(); - } - - if (vchunk.get_internal_chunk()) { - auto& avalableBlkCounter = it->second->available_blk_count; - avalableBlkCounter.fetch_sub(vchunk.available_blks()); - remove_chunk_from_defrag_heap(vchunk.get_chunk_id()); - } else { - LOGWARNMOD(homeobject, "no available chunks left for pg {}", pg_id); - } - - return vchunk.get_internal_chunk(); -} - -csharedChunk HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const chunk_num_t chunkID) { - if (m_chunks.find(chunkID) == m_chunks.end()) { + auto p_chunk_id = p_chunkID.value(); + if (m_chunks.find(p_chunk_id) == m_chunks.end()) { // sanity check - LOGWARNMOD(homeobject, "No chunk found for ChunkID {}", chunkID); + LOGWARNMOD(homeobject, "No chunk found for ChunkID {}", p_chunk_id); return nullptr; } - std::shared_lock lock_guard(m_chunk_selector_mtx); auto pg_it = m_per_pg_heap.find(pg_id); if (pg_it == m_per_pg_heap.end()) { LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); return nullptr; } - VChunk vchunk(nullptr); + bool exist = false; auto& heap = pg_it->second->m_heap; if (auto lock_guard = std::lock_guard< std::mutex >(pg_it->second->mtx); !heap.empty()) { std::vector< VChunk > chunks; @@ -116,8 +99,9 @@ csharedChunk HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const while (!heap.empty()) { auto c = heap.top(); heap.pop(); - if (c.get_chunk_id() == chunkID) { - vchunk = c; + if (c.get_chunk_id() == p_chunk_id) { + exist = true; + // Stop and don't reinsert when the target is found. break; } chunks.push_back(std::move(c)); @@ -127,13 +111,15 @@ csharedChunk HeapChunkSelector::select_specific_chunk(const pg_id_t pg_id, const } } - if (vchunk.get_internal_chunk()) { + auto chunk = m_chunks[p_chunk_id]; + if (exist) { + VChunk vchunk(chunk); auto& avalableBlkCounter = pg_it->second->available_blk_count; - avalableBlkCounter.fetch_sub(vchunk.available_blks()); + avalableBlkCounter -= vchunk.available_blks(); remove_chunk_from_defrag_heap(vchunk.get_chunk_id()); } - return vchunk.get_internal_chunk(); + return chunk; } // Temporarily commented out, the subsequent GC implementation needs to be adapted to fix pg size @@ -175,30 +161,56 @@ void HeapChunkSelector::foreach_chunks(std::function< void(csharedChunk&) >&& cb [cb = std::move(cb)](auto& p) { cb(p.second); }); } -void HeapChunkSelector::release_chunk(const pg_id_t pg_id, const chunk_num_t chunkID) { +bool HeapChunkSelector::release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id) { std::shared_lock lock_guard(m_chunk_selector_mtx); - if (m_chunks.find(chunkID) == m_chunks.end()) { + auto p_chunkID = get_physical_chunk_id(pg_id, v_chunk_id); + if (!p_chunkID.has_value()) { + LOGWARNMOD(homeobject, "No chunk found for v_chunk_id {} in pg {}", v_chunk_id, pg_id); + return false; + } + + auto p_chunk_id = p_chunkID.value(); + if (m_chunks.find(p_chunk_id) == m_chunks.end()) { // sanity check - LOGWARNMOD(homeobject, "No chunk found for ChunkID {}", chunkID); - return; + LOGWARNMOD(homeobject, "No chunk found for p_chunk_id {}", p_chunk_id); + return false; } auto pg_it = m_per_pg_heap.find(pg_id); if (pg_it == m_per_pg_heap.end()) { - LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); - return; + LOGWARNMOD(homeobject, "No pg chunk heap found for pg_id {}", pg_id); + return false; } - const auto& chunk = m_chunks[chunkID]; - VChunk vchunk(chunk); - { - std::lock_guard< std::mutex > l(pg_it->second->mtx); - auto& pg_heap = pg_it->second->m_heap; - pg_heap.emplace(chunk); + bool exist = false; + auto& heap = pg_it->second->m_heap; + if (auto lock_guard = std::lock_guard< std::mutex >(pg_it->second->mtx); !heap.empty()) { + std::vector< VChunk > chunks; + chunks.reserve(heap.size()); + while (!heap.empty()) { + auto c = heap.top(); + heap.pop(); + if (c.get_chunk_id() == p_chunk_id) { + exist = true; + chunks.push_back(std::move(c)); + break; + } + chunks.push_back(std::move(c)); + } + for (auto& c : chunks) { + heap.emplace(c); + } + if (!exist) { + const auto& chunk = m_chunks[p_chunk_id]; + heap.emplace(chunk); + + VChunk vchunk(chunk); + auto& avalableBlkCounter = pg_it->second->available_blk_count; + avalableBlkCounter += vchunk.available_blks(); + } } - auto& avalableBlkCounter = pg_it->second->available_blk_count; - avalableBlkCounter += vchunk.available_blks(); + return true; } uint32_t HeapChunkSelector::get_chunk_size() const { @@ -207,7 +219,7 @@ uint32_t HeapChunkSelector::get_chunk_size() const { return vchunk.size(); } -std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id, u_int64_t pg_size) { +std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id, uint64_t pg_size) { std::unique_lock lock_guard(m_chunk_selector_mtx); if (m_per_pg_heap.find(pg_id) != m_per_pg_heap.end()) { LOGWARNMOD(homeobject, "PG had already created, pg_id {}", pg_id); @@ -217,13 +229,13 @@ std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id, const auto chunk_size = get_chunk_size(); const uint32_t num_chunk = sisl::round_down(pg_size, chunk_size) / chunk_size; - //Select a pdev with the most available num chunk - auto &&most_avail_dev_it = + // Select a pdev with the most available num chunk + auto&& most_avail_dev_it = std::max_element(m_per_dev_heap.begin(), m_per_dev_heap.end(), - [](const std::pair< const uint32_t, std::shared_ptr< ChunkHeap > >& lhs, + [](const std::pair< const uint32_t, std::shared_ptr< ChunkHeap > >& lhs, const std::pair< const uint32_t, std::shared_ptr< ChunkHeap > >& rhs) { - return lhs.second->size() < rhs.second->size(); - }); + return lhs.second->size() < rhs.second->size(); + }); auto& pdev_heap = most_avail_dev_it->second; if (num_chunk > pdev_heap->size()) { LOGWARNMOD(homeobject, "Pdev has no enough space to create pg {} with num_chunk {}", pg_id, num_chunk); @@ -231,15 +243,15 @@ std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id, } auto vchunk = VChunk(nullptr); auto it = m_per_pg_heap.emplace(pg_id, std::make_shared< ChunkHeap >()).first; - auto v2r_vector = m_v2r_chunk_map.emplace(pg_id, std::make_shared< std::vector < chunk_num_t > >()).first->second; - auto r2v_map = m_r2v_chunk_map.emplace(pg_id, std::make_shared< ChunkIdMap >()).first->second; + auto v2p_vector = m_v2p_chunk_map.emplace(pg_id, std::make_shared< std::vector< chunk_num_t > >()).first->second; + auto p2v_map = m_p2v_chunk_map.emplace(pg_id, std::make_shared< ChunkIdMap >()).first->second; auto& pg_heap = it->second; std::scoped_lock lock(pdev_heap->mtx, pg_heap->mtx); - v2r_vector->reserve(num_chunk); + v2p_vector->reserve(num_chunk); for (chunk_num_t i = 0; i < num_chunk; ++i) { vchunk = pdev_heap->m_heap.top(); - //sanity check + // sanity check RELEASE_ASSERT(vchunk.get_total_blks() == vchunk.available_blks(), "vchunk should be empty"); pdev_heap->m_heap.pop(); pdev_heap->available_blk_count -= vchunk.available_blks(); @@ -249,76 +261,99 @@ std::optional< uint32_t > HeapChunkSelector::select_chunks_for_pg(pg_id_t pg_id, pg_heap->available_blk_count += vchunk.available_blks(); // v_chunk_id start from 0. chunk_num_t v_chunk_id = i; - chunk_num_t r_chunk_id = vchunk.get_chunk_id(); - v2r_vector->emplace_back(r_chunk_id); - r2v_map->emplace(r_chunk_id, v_chunk_id); + chunk_num_t p_chunk_id = vchunk.get_chunk_id(); + v2p_vector->emplace_back(p_chunk_id); + p2v_map->emplace(p_chunk_id, v_chunk_id); } return num_chunk; } -void HeapChunkSelector::set_pg_chunks(pg_id_t pg_id, std::vector&& chunk_ids) { +bool HeapChunkSelector::set_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids) { std::unique_lock lock_guard(m_chunk_selector_mtx); - if (m_v2r_chunk_map.find(pg_id) != m_v2r_chunk_map.end()) { + // check pg exist + if (m_v2p_chunk_map.find(pg_id) != m_v2p_chunk_map.end()) { LOGWARNMOD(homeobject, "PG {} had been recovered", pg_id); - return; + return false; + } + + // check chunks valid, must belong to m_chunks and have same pdev_id + std::optional< uint32_t > last_pdev_id; + for (auto p_chunk_id : p_chunk_ids) { + auto it = m_chunks.find(p_chunk_id); + if (it == m_chunks.end()) { + LOGWARNMOD(homeobject, "No chunk found for ChunkID {}", p_chunk_id); + return false; + } + VChunk v_chunk(it->second); + if (last_pdev_id.has_value() && last_pdev_id.value() != v_chunk.get_pdev_id()) { + LOGWARNMOD(homeobject, "The pdev value is different, last_pdev_id={}, pdev_id={}", last_pdev_id.value(), + v_chunk.get_pdev_id()); + return false; + } else { + last_pdev_id = v_chunk.get_pdev_id(); + } } - auto v2r_vector = m_v2r_chunk_map.emplace(pg_id, std::make_shared< std::vector < chunk_num_t > >(std::move(chunk_ids))).first->second; - auto r2v_map = m_r2v_chunk_map.emplace(pg_id, std::make_shared< ChunkIdMap >()).first->second; + auto v2p_vector = + m_v2p_chunk_map.emplace(pg_id, std::make_shared< std::vector< chunk_num_t > >(std::move(p_chunk_ids))) + .first->second; + auto p2v_map = m_p2v_chunk_map.emplace(pg_id, std::make_shared< ChunkIdMap >()).first->second; - for (chunk_num_t i = 0; i < v2r_vector->size(); ++i) { + for (chunk_num_t i = 0; i < v2p_vector->size(); ++i) { // v_chunk_id start from 0. chunk_num_t v_chunk_id = i; - chunk_num_t r_chunk_id = (*v2r_vector)[i]; - r2v_map->emplace(r_chunk_id, v_chunk_id); + chunk_num_t p_chunk_id = (*v2p_vector)[i]; + p2v_map->emplace(p_chunk_id, v_chunk_id); } + return true; } void HeapChunkSelector::recover_per_dev_chunk_heap() { std::unique_lock lock_guard(m_chunk_selector_mtx); - for (const auto& [chunk_id, _] : m_chunks) { + for (const auto& [p_chunk_id, _] : m_chunks) { bool add_to_heap = true; - for (const auto& [_, chunk_map] : m_r2v_chunk_map) { - if (chunk_map->find(chunk_id) != chunk_map->end()) { + for (const auto& [_, chunk_map] : m_p2v_chunk_map) { + if (chunk_map->find(p_chunk_id) != chunk_map->end()) { add_to_heap = false; break; } } - add_chunk_internal(chunk_id, add_to_heap); - + add_chunk_internal(p_chunk_id, add_to_heap); } } -void HeapChunkSelector::recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excludingChunks) -{ +bool HeapChunkSelector::recover_pg_chunk_heap(pg_id_t pg_id, + const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids) { std::unique_lock lock_guard(m_chunk_selector_mtx); if (m_per_pg_heap.find(pg_id) != m_per_pg_heap.end()) { LOGWARNMOD(homeobject, "Pg_heap {} had been recovered", pg_id); - return; + return false; } - auto it = m_v2r_chunk_map.find(pg_id); - if (it == m_v2r_chunk_map.end()) { - LOGWARNMOD(homeobject, "Pg_chunk_map {} had never been recovered", pg_id); - return; + auto it = m_v2p_chunk_map.find(pg_id); + if (it == m_v2p_chunk_map.end()) { + LOGWARNMOD(homeobject, "Pg_chunk_map {} should be recovered beforehand", pg_id); + return false; } const auto& chunk_ids = it->second; auto& pg_heap = m_per_pg_heap.emplace(pg_id, std::make_shared< ChunkHeap >()).first->second; - for (const auto& chunk_id : *chunk_ids) { - if (excludingChunks.find(chunk_id) == excludingChunks.end()) { - const auto& chunk = m_chunks[chunk_id]; + for (size_t v_chunk_id = 0; v_chunk_id < chunk_ids->size(); ++v_chunk_id) { + if (excluding_v_chunk_ids.find(v_chunk_id) == excluding_v_chunk_ids.end()) { + chunk_num_t p_chunk_id = (*chunk_ids)[v_chunk_id]; + const auto& chunk = m_chunks[p_chunk_id]; auto vchunk = VChunk(chunk); pg_heap->m_heap.emplace(vchunk); pg_heap->m_total_blks += vchunk.get_total_blks(); pg_heap->available_blk_count += vchunk.available_blks(); } } + return true; } -std::shared_ptr< const std::vector > HeapChunkSelector::get_pg_chunks(pg_id_t pg_id) const { +std::shared_ptr< const std::vector< homestore::chunk_num_t > > HeapChunkSelector::get_pg_chunks(pg_id_t pg_id) const { std::shared_lock lock_guard(m_chunk_selector_mtx); - auto it = m_v2r_chunk_map.find(pg_id); - if (it != m_v2r_chunk_map.end()) { + auto it = m_v2p_chunk_map.find(pg_id); + if (it != m_v2p_chunk_map.end()) { return it->second; } else { LOGWARNMOD(homeobject, "PG {} had never been created", pg_id); @@ -326,10 +361,68 @@ std::shared_ptr< const std::vector > HeapChunkSelector: } } -homestore::blk_alloc_hints HeapChunkSelector::chunk_to_hints(chunk_num_t chunk_id) const { - auto iter = m_chunks.find(chunk_id); +std::optional< homestore::chunk_num_t > HeapChunkSelector::pop_top_chunk(pg_id_t pg_id) const { + std::shared_lock lock_guard(m_chunk_selector_mtx); + auto pg_it = m_per_pg_heap.find(pg_id); + if (pg_it == m_per_pg_heap.end()) { + LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); + return std::nullopt; + } + VChunk vchunk(nullptr); + auto& heap = pg_it->second->m_heap; + if (auto lock_guard = std::lock_guard< std::mutex >(pg_it->second->mtx); !heap.empty()) { + vchunk = heap.top(); + heap.pop(); + } + + if (vchunk.get_internal_chunk()) { + auto& avalableBlkCounter = pg_it->second->available_blk_count; + avalableBlkCounter -= vchunk.available_blks(); + auto p_chunk_id = vchunk.get_chunk_id(); + auto v_chunkID = get_virtual_chunk_id(pg_id, p_chunk_id); + RELEASE_ASSERT(v_chunkID.has_value(), "No virutal chunk id for physical chunk id {} in pg {}", p_chunk_id, + pg_id); + return v_chunkID.value(); + } else { + return std::nullopt; + } +} + +std::optional< homestore::chunk_num_t > HeapChunkSelector::get_virtual_chunk_id(pg_id_t pg_id, + chunk_num_t p_chunk_id) const { + auto pg_it = m_p2v_chunk_map.find(pg_id); + if (pg_it == m_p2v_chunk_map.end()) { + LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); + return std::nullopt; + } + auto& p2v_map = pg_it->second; + auto it = p2v_map->find(p_chunk_id); + if (it == p2v_map->end()) { + return std::nullopt; + } else { + return it->second; + } +} + +std::optional< homestore::chunk_num_t > HeapChunkSelector::get_physical_chunk_id(pg_id_t pg_id, + chunk_num_t v_chunk_id) const { + auto pg_it = m_v2p_chunk_map.find(pg_id); + if (pg_it == m_v2p_chunk_map.end()) { + LOGWARNMOD(homeobject, "No pg found for pg_id {}", pg_id); + return std::nullopt; + } + auto& v2p_vec = pg_it->second; + if (v_chunk_id >= v2p_vec->size()) { + return std::nullopt; + } else { + return (*v2p_vec)[v_chunk_id]; + } +} + +homestore::blk_alloc_hints HeapChunkSelector::chunk_to_hints(chunk_num_t p_chunk_id) const { + auto iter = m_chunks.find(p_chunk_id); if (iter == m_chunks.end()) { - LOGWARNMOD(homeobject, "No chunk found for chunk_id {}, will return default blk alloc hints", chunk_id); + LOGWARNMOD(homeobject, "No chunk found for p_chunk_id {}, will return default blk alloc hints", p_chunk_id); return homestore::blk_alloc_hints(); } homestore::blk_alloc_hints hints; diff --git a/src/lib/homestore_backend/heap_chunk_selector.h b/src/lib/homestore_backend/heap_chunk_selector.h index 259ecfb..1c5d7a8 100644 --- a/src/lib/homestore_backend/heap_chunk_selector.h +++ b/src/lib/homestore_backend/heap_chunk_selector.h @@ -35,7 +35,9 @@ class HeapChunkSelector : public homestore::ChunkSelector { using VChunkHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkComparator >; using VChunkDefragHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkDefragComparator >; - using ChunkIdMap = std::unordered_map < homestore::chunk_num_t, homestore::chunk_num_t >; // used for real chunk id -> virtual chunk id map + using ChunkIdMap = + std::unordered_map< homestore::chunk_num_t, + homestore::chunk_num_t >; // used for physical chunk id -> virtual chunk id map using chunk_num_t = homestore::chunk_num_t; struct ChunkHeap { @@ -52,16 +54,16 @@ class HeapChunkSelector : public homestore::ChunkSelector { csharedChunk select_chunk([[maybe_unused]] homestore::blk_count_t nblks, const homestore::blk_alloc_hints& hints); - // this function will be used by GC flow or recovery flow to mark one specific chunk to be busy, caller should be - // responsible to use release_chunk() interface to release it when no longer to use the chunk anymore. - csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t); + // this function will be used by create shard or recovery flow to mark one specific chunk to be busy, caller should + // be responsible to use release_chunk() interface to release it when no longer to use the chunk anymore. + csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id); // this function will be used by GC flow to select a chunk for GC csharedChunk most_defrag_chunk(); - // this function is used to return a chunk back to ChunkSelector when sealing a shard, and will only be used by - // Homeobject. - void release_chunk(const pg_id_t pg_id, const chunk_num_t); + // This function returns a chunk back to ChunkSelector. + // It is used in two scenarios: 1. seal shard 2. create shard rollback + bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id); /** * select chunks for pg, chunks need to be in same pdev. @@ -70,26 +72,35 @@ class HeapChunkSelector : public homestore::ChunkSelector { * @param pg_size The fix pg size. * @return An optional uint32_t value representing num_chunk, or std::nullopt if no space left. */ - std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, u_int64_t pg_size); + std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, uint64_t pg_size); - std::shared_ptr< const std::vector > get_pg_chunks(pg_id_t pg_id) const; + // this function is used for pg info superblk persist v_chunk_id <-> p_chunk_id + std::shared_ptr< const std::vector< chunk_num_t > > get_pg_chunks(pg_id_t pg_id) const; + + /** + * pop pg top chunk + * + * @param pg_id The ID of the pg. + * @return An optional chunk_num_t value representing v_chunk_id, or std::nullopt if no space left. + */ + std::optional< chunk_num_t > pop_top_chunk(pg_id_t pg_id) const; // this should be called on each pg meta blk found - void set_pg_chunks(pg_id_t pg_id, std::vector&& chunk_ids); + bool set_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids); // this should be called after all pg meta blk recovered void recover_per_dev_chunk_heap(); // this should be called after ShardManager is initialized and get all the open shards - void recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excludingChunks); + bool recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids); /** * Retrieves the block allocation hints for a given chunk. * - * @param chunk_id The ID of the chunk. + * @param p_chunk_id The physical chunk id. * @return The block allocation hints for the specified chunk. */ - homestore::blk_alloc_hints chunk_to_hints(chunk_num_t chunk_id) const; + homestore::blk_alloc_hints chunk_to_hints(chunk_num_t p_chunk_id) const; /** * Returns the number of available blocks of the given device id. @@ -134,25 +145,33 @@ class HeapChunkSelector : public homestore::ChunkSelector { uint32_t get_chunk_size() const; +private: + std::optional< chunk_num_t > get_virtual_chunk_id(pg_id_t pg_id, chunk_num_t p_chunk_id) const; + + std::optional< chunk_num_t > get_physical_chunk_id(pg_id_t pg_id, chunk_num_t v_chunk_id) const; + + void add_chunk_internal(const chunk_num_t, bool add_to_heap = true); + + void remove_chunk_from_defrag_heap(const chunk_num_t); + private: std::unordered_map< uint32_t, std::shared_ptr< ChunkHeap > > m_per_dev_heap; std::unordered_map< pg_id_t, std::shared_ptr< ChunkHeap > > m_per_pg_heap; - // These mappings ensure "identical layout" by providing bidirectional indexing between virtual and real chunk IDs. - // m_v2r_chunk_map: Maps each pg_id to a vector of real chunk IDs (r_chunk_id). The index in the vector corresponds to the virtual chunk ID (v_chunk_id). - std::unordered_map< pg_id_t, std::shared_ptr< std::vector > > m_v2r_chunk_map; - // m_r2v_chunk_map: Maps each pg_id to a map that inversely maps real chunk IDs (r_chunk_id) to virtual chunk IDs (v_chunk_id). - std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_r2v_chunk_map; + // These mappings ensure "identical layout" by providing bidirectional indexing between virtual and physical chunk + // IDs. m_v2p_chunk_map: Maps each pg_id to a vector of physical chunk IDs (p_chunk_id). The index in the vector + // corresponds to the virtual chunk ID (v_chunk_id). + std::unordered_map< pg_id_t, std::shared_ptr< std::vector< chunk_num_t > > > m_v2p_chunk_map; + // m_p2v_chunk_map: Maps each pg_id to a map that inversely maps physical chunk IDs (p_chunk_id) to virtual chunk + // IDs (v_chunk_id). + std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_p2v_chunk_map; // hold all the chunks , selected or not std::unordered_map< chunk_num_t, csharedChunk > m_chunks; mutable std::shared_mutex m_chunk_selector_mtx; - void add_chunk_internal(const chunk_num_t, bool add_to_heap = true); VChunkDefragHeap m_defrag_heap; std::mutex m_defrag_mtx; - - void remove_chunk_from_defrag_heap(const chunk_num_t); }; } // namespace homeobject diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp index c0aa81e..2283eb8 100644 --- a/src/lib/homestore_backend/hs_blob_manager.cpp +++ b/src/lib/homestore_backend/hs_blob_manager.cpp @@ -351,10 +351,10 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive< } auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get()); - BLOGD(msg_header->shard_id, "n/a", "Picked chunk_id={}", hs_shard->sb_->chunk_id); + BLOGD(msg_header->shard_id, "n/a", "Picked p_chunk_id={}", hs_shard->sb_->p_chunk_id); homestore::blk_alloc_hints hints; - hints.chunk_id_hint = hs_shard->sb_->chunk_id; + hints.chunk_id_hint = hs_shard->sb_->p_chunk_id; return hints; } diff --git a/src/lib/homestore_backend/hs_homeobject.hpp b/src/lib/homestore_backend/hs_homeobject.hpp index 3a723cf..190ae3d 100644 --- a/src/lib/homestore_backend/hs_homeobject.hpp +++ b/src/lib/homestore_backend/hs_homeobject.hpp @@ -90,9 +90,9 @@ class HSHomeObject : public HomeObjectImpl { // Data layout inside 'data': // First, an array of 'pg_members' structures: // | pg_members[0] | pg_members[1] | ... | pg_members[num_members-1] | - // Immediately followed by an array of 'chunk_num_t' values (representing r_chunk_ids): + // Immediately followed by an array of 'chunk_num_t' values (representing physical chunkID): // | chunk_num_t[0] | chunk_num_t[1] | ... | chunk_num_t[num_chunks-1] | - // Here, 'chunk_num_t[i]' represents the r_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and + // Here, 'chunk_num_t[i]' represents the p_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and // increases sequentially. uint32_t size() const { @@ -148,7 +148,8 @@ class HSHomeObject : public HomeObjectImpl { struct shard_info_superblk : public DataHeader { ShardInfo info; - homestore::chunk_num_t chunk_id; + homestore::chunk_num_t p_chunk_id; + homestore::chunk_num_t v_chunk_id; }; #pragma pack() @@ -217,7 +218,6 @@ class HSHomeObject : public HomeObjectImpl { public: homestore::superblk< pg_info_superblk > pg_sb_; shared< homestore::ReplDev > repl_dev_; - std::optional< homestore::chunk_num_t > any_allocated_chunk_id_{}; std::shared_ptr< BlobIndexTable > index_table_; PGMetrics metrics_; @@ -252,12 +252,12 @@ class HSHomeObject : public HomeObjectImpl { struct HS_Shard : public Shard { homestore::superblk< shard_info_superblk > sb_; - HS_Shard(ShardInfo info, homestore::chunk_num_t chunk_id); + HS_Shard(ShardInfo info, homestore::chunk_num_t p_chunk_id); HS_Shard(homestore::superblk< shard_info_superblk >&& sb); ~HS_Shard() override = default; void update_info(const ShardInfo& info); - auto chunk_id() const { return sb_->chunk_id; } + auto p_chunk_id() const { return sb_->p_chunk_id; } }; #pragma pack(1) @@ -437,9 +437,17 @@ class HSHomeObject : public HomeObjectImpl { * @brief Retrieves the chunk number associated with the given shard ID. * * @param id The ID of the shard to retrieve the chunk number for. - * @return An optional chunk number if the shard ID is valid, otherwise an empty optional. + * @return An optional chunk number values shard p_chunk_id if the shard ID is valid, otherwise an empty optional. */ - std::optional< homestore::chunk_num_t > get_shard_chunk(shard_id_t id) const; + std::optional< homestore::chunk_num_t > get_shard_p_chunk_id(shard_id_t id) const; + + /** + * @brief Retrieves the chunk number associated with the given shard ID. + * + * @param id The ID of the shard to retrieve the chunk number for. + * @return An optional chunk number values shard v_chunk_id if the shard ID is valid, otherwise an empty optional. + */ + std::optional< homestore::chunk_num_t > get_shard_v_chunk_id(shard_id_t id) const; /** * @brief recover PG and shard from the superblock. @@ -448,12 +456,14 @@ class HSHomeObject : public HomeObjectImpl { void on_replica_restart(); /** - * @brief Returns any chunk number for the given pg ID. + * @brief Extracts the physical chunk ID for create shard from the message. * - * @param pg The pg ID to get the chunk number for. - * @return A tuple of . + * @param header The message header that includes the shard_info_superblk, which contains the data necessary for + * extracting and mapping the chunk ID. + * @return An optional virtual chunk id if the extraction and mapping process is successful, otherwise an empty + * optional. */ - std::tuple< bool, bool, homestore::chunk_num_t > get_any_chunk_id(pg_id_t pg); + std::optional< homestore::chunk_num_t > resolve_v_chunk_id_from_msg(sisl::blob const& header); bool pg_exists(pg_id_t pg_id) const; diff --git a/src/lib/homestore_backend/hs_pg_manager.cpp b/src/lib/homestore_backend/hs_pg_manager.cpp index 7e0fe73..1321b45 100644 --- a/src/lib/homestore_backend/hs_pg_manager.cpp +++ b/src/lib/homestore_backend/hs_pg_manager.cpp @@ -69,6 +69,7 @@ PGManager::NullAsyncResult HSHomeObject::_create_pg(PGInfo&& pg_info, std::set< return folly::makeUnexpected(PGError::NO_SPACE_LEFT); } + pg_info.chunk_size = chunk_size; pg_info.replica_set_uuid = boost::uuids::random_generator()(); return hs_repl_service() .create_repl_dev(pg_info.replica_set_uuid, peers) @@ -136,15 +137,27 @@ void HSHomeObject::on_create_pg_message_commit(int64_t lsn, sisl::blob const& he return; } + auto local_chunk_size = chunk_selector()->get_chunk_size(); + if (pg_info.chunk_size != local_chunk_size) { + LOGE("Chunk sizes are inconsistent, leader_chunk_size={}, local_chunk_size={}", pg_info.chunk_size, + local_chunk_size); + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::UNKNOWN)); } + return; + } + // select chunks for pg auto const num_chunk = chunk_selector()->select_chunks_for_pg(pg_id, pg_info.size); if (!num_chunk.has_value()) { - LOGW("select chunks for pg failed, pg_id {}", pg_id); + LOGW("Failed to select chunks for pg {}", pg_id); if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); } return; } auto chunk_ids = chunk_selector()->get_pg_chunks(pg_id); - + if (chunk_ids == nullptr) { + LOGW("Failed to get pg chunks, pg_id {}", pg_id); + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); } + return; + } // create index table and pg // TODO create index table during create shard. auto index_table = create_index_table(); @@ -245,6 +258,7 @@ std::string HSHomeObject::serialize_pg_info(const PGInfo& pginfo) { nlohmann::json j; j["pg_info"]["pg_id_t"] = pginfo.id; j["pg_info"]["pg_size"] = pginfo.size; + j["pg_info"]["chunk_size"] = pginfo.chunk_size; j["pg_info"]["repl_uuid"] = boost::uuids::to_string(pginfo.replica_set_uuid); nlohmann::json members_j{}; @@ -263,7 +277,8 @@ PGInfo HSHomeObject::deserialize_pg_info(const unsigned char* json_str, size_t s auto pg_json = nlohmann::json::parse(json_str, json_str + size); PGInfo pg_info(pg_json["pg_info"]["pg_id_t"].get< pg_id_t >()); - pg_info.size = pg_json["pg_info"]["pg_size"].get< u_int64_t >(); + pg_info.size = pg_json["pg_info"]["pg_size"].get< uint64_t >(); + pg_info.chunk_size = pg_json["pg_info"]["chunk_size"].get< uint64_t >(); pg_info.replica_set_uuid = boost::uuids::string_generator()(pg_json["pg_info"]["repl_uuid"].get< std::string >()); for (auto const& m : pg_json["pg_info"]["members"]) { @@ -287,8 +302,9 @@ void HSHomeObject::on_pg_meta_blk_found(sisl::byte_view const& buf, void* meta_c return; } auto pg_id = pg_sb->id; - std::vector< chunk_num_t > chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks); - chunk_selector_->set_pg_chunks(pg_id, std::move(chunk_ids)); + std::vector< chunk_num_t > p_chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks); + bool set_pg_chunks_res = chunk_selector_->set_pg_chunks(pg_id, std::move(p_chunk_ids)); + RELEASE_ASSERT(set_pg_chunks_res, "Failed to set pg={} chunks", pg_id); auto uuid_str = boost::uuids::to_string(pg_sb->index_table_uuid); auto hs_pg = std::make_unique< HS_PG >(std::move(pg_sb), std::move(v.value())); // During PG recovery check if index is already recoverd else @@ -368,7 +384,7 @@ uint32_t HSHomeObject::HS_PG::open_shards() const { std::optional< uint32_t > HSHomeObject::HS_PG::dev_hint(cshared< HeapChunkSelector > chunk_sel) const { if (shards_.empty()) { return std::nullopt; } auto const hs_shard = d_cast< HS_Shard* >(shards_.front().get()); - auto const hint = chunk_sel->chunk_to_hints(hs_shard->chunk_id()); + auto const hint = chunk_sel->chunk_to_hints(hs_shard->p_chunk_id()); return hint.pdev_id_hint; } diff --git a/src/lib/homestore_backend/hs_shard_manager.cpp b/src/lib/homestore_backend/hs_shard_manager.cpp index 938c64e..479f6ab 100644 --- a/src/lib/homestore_backend/hs_shard_manager.cpp +++ b/src/lib/homestore_backend/hs_shard_manager.cpp @@ -111,6 +111,13 @@ ShardManager::AsyncResult< ShardInfo > HSHomeObject::_create_shard(pg_id_t pg_ow auto new_shard_id = generate_new_shard_id(pg_owner); auto create_time = get_current_timestamp(); + // select chunk for shard. + const auto v_chunkID = chunk_selector()->pop_top_chunk(pg_owner); + if (!v_chunkID.has_value()) { + LOGW("no availble chunk left to create shard for pg [{}]", pg_owner); + return folly::makeUnexpected(ShardError::NO_SPACE_LEFT); + } + // Prepare the shard info block sisl::io_blob_safe sb_blob(sisl::round_up(sizeof(shard_info_superblk), repl_dev->get_blk_size()), io_align); shard_info_superblk* sb = new (sb_blob.bytes()) shard_info_superblk(); @@ -124,7 +131,8 @@ ShardManager::AsyncResult< ShardInfo > HSHomeObject::_create_shard(pg_id_t pg_ow .available_capacity_bytes = size_bytes, .total_capacity_bytes = size_bytes, .deleted_capacity_bytes = 0}; - sb->chunk_id = 0; + sb->p_chunk_id = 0; + sb->v_chunk_id = v_chunkID.value(); auto req = repl_result_ctx< ShardManager::Result< ShardInfo > >::make( sizeof(shard_info_superblk) /* header_extn_size */, 0u /* key_size */); @@ -172,7 +180,9 @@ ShardManager::AsyncResult< ShardInfo > HSHomeObject::_seal_shard(ShardInfo const shard_info_superblk* sb = new (sb_blob.bytes()) shard_info_superblk(); sb->type = DataHeader::data_type_t::SHARD_INFO; sb->info = tmp_info; - sb->chunk_id = 0; + // p_chunk_id and v_chunk_id will never be used in seal shard workflow. + sb->p_chunk_id = 0; + sb->v_chunk_id = 0; auto req = repl_result_ctx< ShardManager::Result< ShardInfo > >::make( sizeof(shard_info_superblk) /* header_extn_size */, 0u /* key_size */); @@ -252,6 +262,12 @@ void HSHomeObject::on_shard_message_rollback(int64_t lsn, sisl::blob const& head } switch (msg_header->msg_type) { + case ReplicationMessageType::CREATE_SHARD_MSG: { + auto sb = r_cast< shard_info_superblk const* >(header.cbytes() + sizeof(ReplicationMessageHeader)); + bool res = chunk_selector_->release_chunk(sb->info.placement_group, sb->v_chunk_id); + RELEASE_ASSERT(res, "Failed to release chunk {} to pg {}", sb->v_chunk_id, sb->info.placement_group); + break; + } case ReplicationMessageType::SEAL_SHARD_MSG: { auto sb = r_cast< shard_info_superblk const* >(header.cbytes() + sizeof(ReplicationMessageHeader)); auto const shard_info = sb->info; @@ -269,6 +285,7 @@ void HSHomeObject::on_shard_message_rollback(int64_t lsn, sisl::blob const& head shard_info.id); } } + break; } default: { break; @@ -363,9 +380,10 @@ void HSHomeObject::on_shard_message_commit(int64_t lsn, sisl::blob const& h, hom if (state == ShardInfo::State::SEALED) { auto pg_id = shard_info.placement_group; - auto chunk_id = get_shard_chunk(shard_info.id); - RELEASE_ASSERT(chunk_id.has_value(), "Chunk id not found"); - chunk_selector()->release_chunk(pg_id, chunk_id.value()); + auto v_chunkID = get_shard_v_chunk_id(shard_info.id); + RELEASE_ASSERT(v_chunkID.has_value(), "v_chunk id not found"); + bool res = chunk_selector()->release_chunk(pg_id, v_chunkID.value()); + RELEASE_ASSERT(res, "Failed to release chunk {}, pg_id {}", v_chunkID.value(), pg_id); update_shard_in_map(shard_info); } else LOGW("try to commit SEAL_SHARD_MSG but shard state is not sealed, shard_id: {}", shard_info.id); @@ -392,10 +410,11 @@ void HSHomeObject::on_shard_meta_blk_recover_completed(bool success) { excluding_chunks.reserve(pair.second->shards_.size()); for (auto& shard : pair.second->shards_) { if (shard->info.state == ShardInfo::State::OPEN) { - excluding_chunks.emplace(d_cast< HS_Shard* >(shard.get())->sb_->chunk_id); + excluding_chunks.emplace(d_cast< HS_Shard* >(shard.get())->sb_->v_chunk_id); } } - chunk_selector_->recover_pg_chunk_heap(pair.first, excluding_chunks); + bool res = chunk_selector_->recover_pg_chunk_heap(pair.first, excluding_chunks); + RELEASE_ASSERT(res, "Failed to recover pg chunk heap, pg={}", pair.first); } } @@ -424,37 +443,55 @@ void HSHomeObject::update_shard_in_map(const ShardInfo& shard_info) { hs_shard->update_info(shard_info); } -std::optional< homestore::chunk_num_t > HSHomeObject::get_shard_chunk(shard_id_t id) const { +std::optional< homestore::chunk_num_t > HSHomeObject::get_shard_p_chunk_id(shard_id_t id) const { std::scoped_lock lock_guard(_shard_lock); auto shard_iter = _shard_map.find(id); if (shard_iter == _shard_map.end()) { return std::nullopt; } auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get()); - return std::make_optional< homestore::chunk_num_t >(hs_shard->sb_->chunk_id); + return std::make_optional< homestore::chunk_num_t >(hs_shard->sb_->p_chunk_id); } -std::tuple< bool, bool, homestore::chunk_num_t > HSHomeObject::get_any_chunk_id(pg_id_t pg_id) { - std::scoped_lock lock_guard(_pg_lock); - auto pg_iter = _pg_map.find(pg_id); - if (pg_iter == _pg_map.end()) { return {false /* pg_found */, false /* shards_found */, 0 /* chunk_id */}; } +std::optional< homestore::chunk_num_t > HSHomeObject::get_shard_v_chunk_id(shard_id_t id) const { + std::scoped_lock lock_guard(_shard_lock); + auto shard_iter = _shard_map.find(id); + if (shard_iter == _shard_map.end()) { return std::nullopt; } + auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get()); + return std::make_optional< homestore::chunk_num_t >(hs_shard->sb_->v_chunk_id); +} - HS_PG* pg = static_cast< HS_PG* >(pg_iter->second.get()); - if (pg->any_allocated_chunk_id_.has_value()) { // it is already cached and use it; - return {true /* pg_found */, true /* shards_found */, *pg->any_allocated_chunk_id_}; +std::optional< homestore::chunk_num_t > HSHomeObject::resolve_v_chunk_id_from_msg(sisl::blob const& header) { + const ReplicationMessageHeader* msg_header = r_cast< const ReplicationMessageHeader* >(header.cbytes()); + if (msg_header->corrupted()) { + LOGW("replication message header is corrupted with crc error"); + return std::nullopt; } - auto& shards = pg->shards_; - if (shards.empty()) { return {true /* pg_found */, false /* shards_found */, 0 /* chunk_id */}; } - - auto hs_shard = d_cast< HS_Shard* >(shards.front().get()); - pg->any_allocated_chunk_id_ = hs_shard->sb_->chunk_id; // cache it; - return {true /* pg_found */, true /* shards_found */, *pg->any_allocated_chunk_id_}; + switch (msg_header->msg_type) { + case ReplicationMessageType::CREATE_SHARD_MSG: { + const pg_id_t pg_id = msg_header->pg_id; + std::scoped_lock lock_guard(_pg_lock); + auto pg_iter = _pg_map.find(pg_id); + if (pg_iter == _pg_map.end()) { + LOGW("Requesting a chunk for an unknown pg={}", pg_id); + return std::nullopt; + } + auto sb = r_cast< shard_info_superblk const* >(header.cbytes() + sizeof(ReplicationMessageHeader)); + return sb->v_chunk_id; + break; + } + default: { + LOGW("Unexpected message type encountered: {}. This function should only be called with 'CREATE_SHARD_MSG'.", + msg_header->msg_type); + return std::nullopt; + } + } } -HSHomeObject::HS_Shard::HS_Shard(ShardInfo shard_info, homestore::chunk_num_t chunk_id) : +HSHomeObject::HS_Shard::HS_Shard(ShardInfo shard_info, homestore::chunk_num_t p_chunk_id) : Shard(std::move(shard_info)), sb_(_shard_meta_name) { sb_.create(sizeof(shard_info_superblk)); sb_->info = info; - sb_->chunk_id = chunk_id; + sb_->p_chunk_id = p_chunk_id; sb_.write(); } diff --git a/src/lib/homestore_backend/replication_state_machine.cpp b/src/lib/homestore_backend/replication_state_machine.cpp index 3ff25ad..d379fa9 100644 --- a/src/lib/homestore_backend/replication_state_machine.cpp +++ b/src/lib/homestore_backend/replication_state_machine.cpp @@ -78,6 +78,10 @@ void ReplicationStateMachine::on_rollback(int64_t lsn, sisl::blob const& header, return; } switch (msg_header->msg_type) { + case ReplicationMessageType::CREATE_SHARD_MSG: { + home_object_->on_shard_message_rollback(lsn, header, key, ctx); + break; + } case ReplicationMessageType::SEAL_SHARD_MSG: { home_object_->on_shard_message_rollback(lsn, header, key, ctx); break; @@ -133,25 +137,31 @@ ReplicationStateMachine::get_blk_alloc_hints(sisl::blob const& header, uint32_t const ReplicationMessageHeader* msg_header = r_cast< const ReplicationMessageHeader* >(header.cbytes()); switch (msg_header->msg_type) { case ReplicationMessageType::CREATE_SHARD_MSG: { - auto& pg_id = msg_header->pg_id; + pg_id_t pg_id = msg_header->pg_id; // check whether the pg exists if (!home_object_->pg_exists(pg_id)) { LOGI("can not find pg {} when getting blk_alloc_hint", pg_id); // TODO:: add error code to indicate the pg not found in homestore side return folly::makeUnexpected(homestore::ReplServiceError::NO_SPACE_LEFT); } - // Since chunks are selected when a pg is created, the chunkselector selects one of the chunks owned by the pg + + auto v_chunkID = home_object_->resolve_v_chunk_id_from_msg(header); + RELEASE_ASSERT(v_chunkID.has_value(), "unkown chunk for create shard"); homestore::blk_alloc_hints hints; - hints.pdev_id_hint = pg_id; // FIXME @Hooper: Temporary bypass using pdev_id_hint to represent - // pg_id_hint, "identical layout" will change it + // use pdev_id_hint (of type uint32_t) to store the values of pg_id and v_chunk_id. + // Both chunk_num_t and pg_id_t are of type uint16_t. + static_assert(std::is_same< pg_id_t, uint16_t >::value, "pg_id_t is not uint16_t"); + static_assert(std::is_same< homestore::chunk_num_t, uint16_t >::value, "chunk_num_t is not uint16_t"); + homestore::chunk_num_t v_chunk_id = v_chunkID.value(); + hints.pdev_id_hint = ((uint32_t)pg_id << 16) | v_chunk_id; return hints; } case ReplicationMessageType::SEAL_SHARD_MSG: { - auto chunk_id = home_object_->get_shard_chunk(msg_header->shard_id); - RELEASE_ASSERT(chunk_id.has_value(), "unknown shard id to get binded chunk"); + auto p_chunkID = home_object_->get_shard_p_chunk_id(msg_header->shard_id); + RELEASE_ASSERT(p_chunkID.has_value(), "unknown shard id to get binded chunk"); homestore::blk_alloc_hints hints; - hints.chunk_id_hint = chunk_id.value(); + hints.chunk_id_hint = p_chunkID.value(); return hints; } diff --git a/src/lib/homestore_backend/tests/hs_pg_tests.cpp b/src/lib/homestore_backend/tests/hs_pg_tests.cpp index b8a4499..3a0dd89 100644 --- a/src/lib/homestore_backend/tests/hs_pg_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_pg_tests.cpp @@ -46,6 +46,44 @@ TEST_F(HomeObjectFixture, PGStatsTest) { LOGINFO("HomeObj stats: {}", stats.to_string()); } +TEST_F(HomeObjectFixture, PGExceedSpaceTest) { + LOGINFO("HomeObject replica={} setup completed", g_helper->replica_num()); + pg_id_t pg_id{1}; + if (0 == g_helper->replica_num()) { // leader + auto memebers = g_helper->members(); + auto name = g_helper->name(); + auto info = homeobject::PGInfo(pg_id); + info.size = 500 * Gi; // execced local available space + for (const auto& member : memebers) { + if (0 == member.second) { + // by default, leader is the first member + info.members.insert(homeobject::PGMember{member.first, name + std::to_string(member.second), 1}); + } else { + info.members.insert(homeobject::PGMember{member.first, name + std::to_string(member.second), 0}); + } + } + auto p = _obj_inst->pg_manager()->create_pg(std::move(info)).get(); + ASSERT_TRUE(p.hasError()); + PGError error = p.error(); + ASSERT_EQ(PGError::NO_SPACE_LEFT, error); + } else { + auto start_time = std::chrono::steady_clock::now(); + bool res = true; + // follower need to wait for pg creation + while (!pg_exist(pg_id)) { + auto current_time = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast< std::chrono::seconds >(current_time - start_time).count(); + if (duration >= 20) { + LOGINFO("Failed to create pg {} at follower", pg_id); + res = false; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + ASSERT_FALSE(res); + } +} + TEST_F(HomeObjectFixture, PGRecoveryTest) { // create 10 pg for (pg_id_t i = 1; i < 11; i++) { diff --git a/src/lib/homestore_backend/tests/hs_shard_tests.cpp b/src/lib/homestore_backend/tests/hs_shard_tests.cpp index b916109..70626b0 100644 --- a/src/lib/homestore_backend/tests/hs_shard_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_shard_tests.cpp @@ -6,10 +6,10 @@ TEST_F(HomeObjectFixture, CreateMultiShards) { auto _shard_1 = create_shard(pg_id, 64 * Mi); auto _shard_2 = create_shard(pg_id, 64 * Mi); - auto chunk_num_1 = _obj_inst->get_shard_chunk(_shard_1.id); + auto chunk_num_1 = _obj_inst->get_shard_p_chunk_id(_shard_1.id); ASSERT_TRUE(chunk_num_1.has_value()); - auto chunk_num_2 = _obj_inst->get_shard_chunk(_shard_2.id); + auto chunk_num_2 = _obj_inst->get_shard_p_chunk_id(_shard_2.id); ASSERT_TRUE(chunk_num_2.has_value()); // check if both chunk is on the same pdev; @@ -30,12 +30,12 @@ TEST_F(HomeObjectFixture, CreateMultiShardsOnMultiPG) { for (const auto pg : pgs) { auto shard_info = create_shard(pg, Mi); - auto chunk_num_1 = _obj_inst->get_shard_chunk(shard_info.id); + auto chunk_num_1 = _obj_inst->get_shard_p_chunk_id(shard_info.id); ASSERT_TRUE(chunk_num_1.has_value()); // create another shard again. shard_info = create_shard(pg, Mi); - auto chunk_num_2 = _obj_inst->get_shard_chunk(shard_info.id); + auto chunk_num_2 = _obj_inst->get_shard_p_chunk_id(shard_info.id); ASSERT_TRUE(chunk_num_2.has_value()); // check if both chunk is on the same pdev; diff --git a/src/lib/homestore_backend/tests/test_heap_chunk_selector.cpp b/src/lib/homestore_backend/tests/test_heap_chunk_selector.cpp index 0f3a1c1..8ce86e2 100644 --- a/src/lib/homestore_backend/tests/test_heap_chunk_selector.cpp +++ b/src/lib/homestore_backend/tests/test_heap_chunk_selector.cpp @@ -40,7 +40,6 @@ class Chunk : public std::enable_shared_from_this< Chunk > { blk_num_t get_total_blks() const { return m_available_blks; } void set_chunk_id(uint16_t chunk_id) { m_chunk_id = chunk_id; } - const std::shared_ptr< Chunk > get_internal_chunk() { return shared_from_this(); } uint64_t size() const { return 1 * Mi; } Chunk(uint32_t pdev_id, uint16_t chunk_id, uint32_t available_blks, uint32_t defrag_nblks) { @@ -75,15 +74,19 @@ blk_num_t VChunk::get_total_blks() const { return m_internal_chunk->get_total_bl uint64_t VChunk::size() const { return m_internal_chunk->size(); } -cshared< Chunk > VChunk::get_internal_chunk() const { return m_internal_chunk->get_internal_chunk(); } +cshared< Chunk > VChunk::get_internal_chunk() const { return m_internal_chunk; } } // namespace homestore using homeobject::csharedChunk; using homeobject::HeapChunkSelector; +using homeobject::pg_id_t; using homestore::Chunk; using homestore::chunk_num_t; +const pg_id_t FAKE_PG_ID = UINT16_MAX; +const chunk_num_t FAKE_CHUNK_ID = UINT16_MAX; + class HeapChunkSelectorTest : public ::testing::Test { protected: void SetUp() override { @@ -112,23 +115,26 @@ class HeapChunkSelectorTest : public ::testing::Test { ASSERT_EQ(pg_heap_it->second->size(), 3); // test chunk_map - auto v2r_chunk_map_it = HCS.m_v2r_chunk_map.find(pg_id); - ASSERT_NE(v2r_chunk_map_it, HCS.m_v2r_chunk_map.end()); - ASSERT_EQ(v2r_chunk_map_it->second->size(), 3); + auto v2p_chunk_map_it = HCS.m_v2p_chunk_map.find(pg_id); + ASSERT_NE(v2p_chunk_map_it, HCS.m_v2p_chunk_map.end()); + ASSERT_EQ(v2p_chunk_map_it->second->size(), 3); + + auto p2v_chunk_map_it = HCS.m_p2v_chunk_map.find(pg_id); + ASSERT_NE(p2v_chunk_map_it, HCS.m_p2v_chunk_map.end()); + ASSERT_EQ(p2v_chunk_map_it->second->size(), 3); - auto r2v_chunk_map_it = HCS.m_r2v_chunk_map.find(pg_id); - ASSERT_NE(r2v_chunk_map_it, HCS.m_r2v_chunk_map.end()); - ASSERT_EQ(r2v_chunk_map_it->second->size(), 3); for (int i = 0; i < 3; ++i) { - auto r_chunk_id = v2r_chunk_map_it->second->at(i); - ASSERT_EQ(i, r2v_chunk_map_it->second->at(r_chunk_id)); - auto pdev_id = HCS.m_chunks[r_chunk_id]->get_pdev_id(); + // test p_chunk_id <-> v_chunk_id + auto p_chunk_id = v2p_chunk_map_it->second->at(i); + ASSERT_EQ(i, p2v_chunk_map_it->second->at(p_chunk_id)); + // test pg chunks must belong to same pdev + auto pdev_id = HCS.m_chunks[p_chunk_id]->get_pdev_id(); if (last_pdev_id != 0) { ASSERT_EQ(last_pdev_id, pdev_id); } else { last_pdev_id = pdev_id; } - + // pdev heap should be empty at this point because all chunks have already been given to pg. auto pdev_it = HCS.m_per_dev_heap.find(pdev_id); ASSERT_NE(pdev_it, HCS.m_per_dev_heap.end()); ASSERT_EQ(pdev_it->second->size(), 0); @@ -136,7 +142,6 @@ class HeapChunkSelectorTest : public ::testing::Test { } } - public: HeapChunkSelector HCS; }; @@ -147,6 +152,34 @@ TEST_F(HeapChunkSelectorTest, test_for_each_chunk) { ASSERT_EQ(size.load(), 18); } +TEST_F(HeapChunkSelectorTest, test_identical_layout) { + const homestore::blk_count_t count = 1; + homestore::blk_alloc_hints hints; + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + chunk_num_t p_chunk_id = 0; + for (int j = 3; j > 0; --j) { + const auto v_chunkID = HCS.pop_top_chunk(pg_id); + ASSERT_TRUE(v_chunkID.has_value()); + auto p_chunkID = HCS.get_physical_chunk_id(pg_id, v_chunkID.value()); + ASSERT_TRUE(p_chunkID.has_value()); + p_chunk_id = p_chunkID.value(); + const auto v_chunkID2 = HCS.get_virtual_chunk_id(pg_id, p_chunkID.value()); + ASSERT_EQ(v_chunkID.value(), v_chunkID2.value()); + hints.pdev_id_hint = ((uint32_t)pg_id << 16) | v_chunkID.value(); + ASSERT_NE(HCS.select_chunk(count, hints), nullptr); + } + + // error handling test + ASSERT_FALSE(HCS.get_virtual_chunk_id(FAKE_PG_ID, 0).has_value()); + ASSERT_FALSE(HCS.get_physical_chunk_id(FAKE_PG_ID, p_chunk_id).has_value()); + + ASSERT_FALSE(HCS.get_virtual_chunk_id(pg_id, FAKE_CHUNK_ID).has_value()); + ASSERT_FALSE(HCS.get_physical_chunk_id(pg_id, FAKE_CHUNK_ID).has_value()); + // all chunks have been given out + ASSERT_FALSE(HCS.pop_top_chunk(pg_id).has_value()); + } +} + TEST_F(HeapChunkSelectorTest, test_select_chunk) { homestore::blk_count_t count = 1; homestore::blk_alloc_hints hints; @@ -154,84 +187,81 @@ TEST_F(HeapChunkSelectorTest, test_select_chunk) { ASSERT_EQ(chunk, nullptr); for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { - hints.pdev_id_hint = pg_id; // tmp bypass using pdev_id_hint present pg_id for (int j = 3; j > 0; --j) { + chunk_num_t v_chunk_id = 3 - j; + hints.pdev_id_hint = ((uint32_t)pg_id << 16) | v_chunk_id; auto chunk = HCS.select_chunk(count, hints); ASSERT_NE(chunk, nullptr); - ASSERT_EQ(chunk->get_pdev_id(), pg_id); + ASSERT_EQ(chunk->get_pdev_id(), pg_id); // in this ut, pg_id is same as pdev id ASSERT_EQ(chunk->available_blks(), j); } } } - TEST_F(HeapChunkSelectorTest, test_select_specific_chunk) { - const uint16_t pg_id = 1; - auto chunk_ids = HCS.get_pg_chunks(pg_id); - ASSERT_NE(chunk_ids, nullptr); - const chunk_num_t chunk_id = chunk_ids->at(0); - - auto chunk = HCS.select_specific_chunk(pg_id, chunk_id); - ASSERT_EQ(chunk->get_chunk_id(), chunk_id); - auto pdev_id = chunk->get_pdev_id(); - - // make sure pg chunk map - auto pg_heap_it = HCS.m_per_pg_heap.find(pg_id); - ASSERT_NE(pg_heap_it, HCS.m_per_pg_heap.end()); - ASSERT_EQ(pg_heap_it->second->size(), 2); - - // test chunk_map stable - auto v2r_chunk_map_it = HCS.m_v2r_chunk_map.find(pg_id); - ASSERT_NE(v2r_chunk_map_it, HCS.m_v2r_chunk_map.end()); - ASSERT_EQ(v2r_chunk_map_it->second->size(), 3); - - auto r2v_chunk_map_it = HCS.m_r2v_chunk_map.find(pg_id); - ASSERT_NE(r2v_chunk_map_it, HCS.m_r2v_chunk_map.end()); - ASSERT_EQ(r2v_chunk_map_it->second->size(), 3); - - // select the rest chunks to make sure specific chunk does not exist in HeapChunkSelector anymore. - homestore::blk_count_t count = 1; - homestore::blk_alloc_hints hints; - hints.pdev_id_hint = pg_id; - for (int j = 2; j > 0; --j) { - auto chunk = HCS.select_chunk(count, hints); - ASSERT_EQ(chunk->get_pdev_id(), pdev_id); + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + auto chunk_ids = HCS.get_pg_chunks(pg_id); + ASSERT_NE(chunk_ids, nullptr); + const chunk_num_t v_chunk_id = 0; + const chunk_num_t p_chunk_id = chunk_ids->at(v_chunk_id); + + auto chunk = HCS.select_specific_chunk(pg_id, v_chunk_id); + ASSERT_EQ(chunk->get_chunk_id(), p_chunk_id); + + auto pg_heap_it = HCS.m_per_pg_heap.find(pg_id); + ASSERT_NE(pg_heap_it, HCS.m_per_pg_heap.end()); + ASSERT_EQ(pg_heap_it->second->size(), 2); + + // test chunk_map stable + auto v2p_chunk_map_it = HCS.m_v2p_chunk_map.find(pg_id); + ASSERT_NE(v2p_chunk_map_it, HCS.m_v2p_chunk_map.end()); + ASSERT_EQ(v2p_chunk_map_it->second->size(), 3); + + auto p2v_chunk_map_it = HCS.m_p2v_chunk_map.find(pg_id); + ASSERT_NE(p2v_chunk_map_it, HCS.m_p2v_chunk_map.end()); + ASSERT_EQ(p2v_chunk_map_it->second->size(), 3); + + // release this chunk to HeapChunkSelector and try again + HCS.release_chunk(pg_id, v_chunk_id); + chunk = HCS.select_specific_chunk(pg_id, v_chunk_id); + ASSERT_EQ(pg_id, chunk->get_pdev_id()); + ASSERT_EQ(p_chunk_id, chunk->get_chunk_id()); } - - // release this chunk to HeapChunkSelector - HCS.release_chunk(pg_id, chunk_id); - chunk = HCS.select_chunk(1, hints); - ASSERT_EQ(1, chunk->get_pdev_id()); - ASSERT_EQ(chunk_id, chunk->get_chunk_id()); - } - TEST_F(HeapChunkSelectorTest, test_release_chunk) { - homestore::blk_count_t count = 1; - homestore::blk_alloc_hints hints; - const uint16_t pg_id = 1; - hints.pdev_id_hint = pg_id; - auto chunk1 = HCS.select_chunk(count, hints); - auto pdev_id = chunk1->get_pdev_id(); - - ASSERT_EQ(chunk1->get_pdev_id(), pdev_id); - ASSERT_EQ(chunk1->available_blks(), 3); - - auto chunk2 = HCS.select_chunk(count, hints); - ASSERT_EQ(chunk2->get_pdev_id(), pdev_id); - ASSERT_EQ(chunk2->available_blks(), 2); - - HCS.release_chunk(pg_id, chunk1->get_chunk_id()); - HCS.release_chunk(pg_id, chunk2->get_chunk_id()); - - chunk1 = HCS.select_chunk(count, hints); - ASSERT_EQ(chunk1->get_pdev_id(), pdev_id); - ASSERT_EQ(chunk1->available_blks(), 3); - - chunk2 = HCS.select_chunk(count, hints); - ASSERT_EQ(chunk2->get_pdev_id(), pdev_id); - ASSERT_EQ(chunk2->available_blks(), 2); + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + auto v_chunkID = HCS.pop_top_chunk(pg_id); + ASSERT_TRUE(v_chunkID.has_value()); + const auto v_chunk_id_1 = v_chunkID.value(); + auto chunk1 = HCS.select_specific_chunk(pg_id, v_chunk_id_1); + ASSERT_NE(chunk1, nullptr); + auto pdev_id = chunk1->get_pdev_id(); + ASSERT_EQ(chunk1->get_pdev_id(), pdev_id); + ASSERT_EQ(chunk1->available_blks(), 3); + + v_chunkID = HCS.pop_top_chunk(pg_id); + ASSERT_TRUE(v_chunkID.has_value()); + const auto v_chunk_id_2 = v_chunkID.value(); + auto chunk2 = HCS.select_specific_chunk(pg_id, v_chunk_id_2); + ASSERT_NE(chunk2, nullptr); + ASSERT_EQ(chunk2->get_pdev_id(), pdev_id); + ASSERT_EQ(chunk2->available_blks(), 2); + + ASSERT_TRUE(HCS.release_chunk(pg_id, v_chunk_id_1)); + ASSERT_TRUE(HCS.release_chunk(pg_id, v_chunk_id_2)); + + chunk1 = HCS.select_specific_chunk(pg_id, v_chunk_id_1); + ASSERT_EQ(chunk1->get_pdev_id(), pdev_id); + ASSERT_EQ(chunk1->available_blks(), 3); + + chunk2 = HCS.select_specific_chunk(pg_id, v_chunk_id_2); + ASSERT_EQ(chunk2->get_pdev_id(), pdev_id); + ASSERT_EQ(chunk2->available_blks(), 2); + + ASSERT_FALSE(HCS.release_chunk(FAKE_PG_ID, FAKE_CHUNK_ID)); + ASSERT_FALSE(HCS.release_chunk(pg_id, FAKE_CHUNK_ID)); + } } TEST_F(HeapChunkSelectorTest, test_recovery) { @@ -242,49 +272,75 @@ TEST_F(HeapChunkSelectorTest, test_recovery) { HCS_recovery.add_chunk(std::make_shared< Chunk >(2, 4, 1, 6)); HCS_recovery.add_chunk(std::make_shared< Chunk >(2, 5, 2, 5)); HCS_recovery.add_chunk(std::make_shared< Chunk >(2, 6, 3, 4)); + HCS_recovery.add_chunk(std::make_shared< Chunk >(3, 7, 1, 3)); + HCS_recovery.add_chunk(std::make_shared< Chunk >(3, 8, 2, 2)); + HCS_recovery.add_chunk(std::make_shared< Chunk >(3, 9, 3, 1)); + + // on_pg_meta_blk_found + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + std::vector< chunk_num_t > chunk_ids{1, 2}; + std::vector< chunk_num_t > chunk_ids_for_twice{1, 2}; + std::vector< chunk_num_t > chunk_ids_not_valid{1, 20}; + std::vector< chunk_num_t > chunk_ids_not_same_pdev{1, 6}; + for (chunk_num_t j = 0; j < 2; ++j) { + chunk_ids[j] += (pg_id - 1) * 3; + chunk_ids_for_twice[j] += (pg_id - 1) * 3; + chunk_ids_not_valid[j] += (pg_id - 1) * 3; + chunk_ids_not_same_pdev[j] += ((pg_id - 1) * 3) % 9; + } - std::vector chunk_ids {1,2,3}; - const uint16_t pg_id = 1; - // test recover chunk map - HCS_recovery.set_pg_chunks(pg_id, std::move(chunk_ids)); - auto v2r_chunk_map_it = HCS_recovery.m_v2r_chunk_map.find(pg_id); - ASSERT_NE(v2r_chunk_map_it, HCS_recovery.m_v2r_chunk_map.end()); - ASSERT_EQ(v2r_chunk_map_it->second->size(), 3); - - auto r2v_chunk_map_it = HCS_recovery.m_r2v_chunk_map.find(pg_id); - ASSERT_NE(r2v_chunk_map_it, HCS_recovery.m_r2v_chunk_map.end()); - ASSERT_EQ(r2v_chunk_map_it->second->size(), 3); - // test recover pdev map + // test recover chunk map + ASSERT_FALSE(HCS_recovery.set_pg_chunks(pg_id, std::move(chunk_ids_not_valid))); + ASSERT_FALSE(HCS_recovery.set_pg_chunks(pg_id, std::move(chunk_ids_not_same_pdev))); + + ASSERT_TRUE(HCS_recovery.set_pg_chunks(pg_id, std::move(chunk_ids))); + // can't set pg chunks twice + ASSERT_FALSE(HCS_recovery.set_pg_chunks(pg_id, std::move(chunk_ids_for_twice))); + + auto v2p_chunk_map_it = HCS_recovery.m_v2p_chunk_map.find(pg_id); + ASSERT_NE(v2p_chunk_map_it, HCS_recovery.m_v2p_chunk_map.end()); + ASSERT_EQ(v2p_chunk_map_it->second->size(), 2); + + auto p2v_chunk_map_it = HCS_recovery.m_p2v_chunk_map.find(pg_id); + ASSERT_NE(p2v_chunk_map_it, HCS_recovery.m_p2v_chunk_map.end()); + ASSERT_EQ(p2v_chunk_map_it->second->size(), 2); + } + + // on_pg_meta_blk_recover_completed HCS_recovery.recover_per_dev_chunk_heap(); - auto pdev_it = HCS_recovery.m_per_dev_heap.find(1); - ASSERT_NE(pdev_it, HCS_recovery.m_per_dev_heap.end()); - ASSERT_EQ(pdev_it->second->size(), 0); - - pdev_it = HCS_recovery.m_per_dev_heap.find(2); - ASSERT_NE(pdev_it, HCS_recovery.m_per_dev_heap.end()); - ASSERT_EQ(pdev_it->second->size(), 3); - auto &pdev_heap = pdev_it->second->m_heap; - auto vchunk = homestore::VChunk(nullptr); - for (int i = 6; i > 3; --i) { + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + // test recover pdev map size + auto pdev_it = HCS_recovery.m_per_dev_heap.find(pg_id); + ASSERT_NE(pdev_it, HCS_recovery.m_per_dev_heap.end()); + ASSERT_EQ(pdev_it->second->size(), 1); // 1 = 3(all) - 2(pg) + + auto& pdev_heap = pdev_it->second->m_heap; + auto vchunk = homestore::VChunk(nullptr); vchunk = pdev_heap.top(); - pdev_heap.pop(); - ASSERT_EQ(vchunk.get_chunk_id(), i); + ASSERT_EQ(vchunk.get_chunk_id(), 3 + (pg_id - 1) * 3); } - // test recover pg heap - std::unordered_set< homestore::chunk_num_t > excluding_chunks; - excluding_chunks.emplace(1); - HCS_recovery.recover_pg_chunk_heap(pg_id, excluding_chunks); - auto pg_heap_it = HCS_recovery.m_per_pg_heap.find(pg_id); - ASSERT_NE(pg_heap_it, HCS_recovery.m_per_pg_heap.end()); - ASSERT_EQ(pg_heap_it->second->size(), 2); - - homestore::blk_alloc_hints hints; - hints.pdev_id_hint = pg_id; - for (int j = 3; j > 1; --j) { - auto chunk = HCS_recovery.select_chunk(1, hints); - ASSERT_EQ(chunk->get_pdev_id(), 1); - ASSERT_EQ(chunk->available_blks(), j); + // on_shard_meta_blk_recover_completed + for (uint16_t pg_id = 1; pg_id < 4; ++pg_id) { + // test recover pg heap + std::unordered_set< homestore::chunk_num_t > excluding_chunks; + excluding_chunks.emplace(0); + + ASSERT_FALSE(HCS_recovery.recover_pg_chunk_heap(FAKE_PG_ID, excluding_chunks)); + ASSERT_TRUE(HCS_recovery.recover_pg_chunk_heap(pg_id, excluding_chunks)); + // can't recover pg chunk heap twice + ASSERT_FALSE(HCS_recovery.recover_pg_chunk_heap(pg_id, excluding_chunks)); + + auto pg_heap_it = HCS_recovery.m_per_pg_heap.find(pg_id); + ASSERT_NE(pg_heap_it, HCS_recovery.m_per_pg_heap.end()); + ASSERT_EQ(pg_heap_it->second->size(), 1); + + const auto p_chunkID = HCS_recovery.pop_top_chunk(pg_id); + ASSERT_TRUE(p_chunkID.has_value()); + auto chunk = HCS_recovery.select_specific_chunk(pg_id, p_chunkID.value()); + ASSERT_NE(chunk, nullptr); + ASSERT_EQ(chunk->get_pdev_id(), pg_id); + ASSERT_EQ(chunk->available_blks(), 2); } }