Skip to content

Commit

Permalink
Implement shard identical layout using blk_alloc_hints and add v_chun…
Browse files Browse the repository at this point in the history 
…k_id field to shard_info_superblk.

Enhance defensive checks in ChunkSelector for input validation and exception handling.
Adapt unit tests and introduce PGExceedSpaceTest.
  • Loading branch information
@Hooper9973
Hooper9973 committed Nov 14, 2024
1 parent 49b1b0c commit 5e8f05b
Show file tree
Hide file tree
Showing 13 changed files with 593 additions and 304 deletions.
2 changes: 1 addition & 1 deletion conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class HomeObjectConan(ConanFile):
name = "homeobject"
version = "2.1.7"
version = "2.1.8"

homepage = "https://github.com/eBay/HomeObject"
description = "Blob Store built on HomeReplication"
Expand Down
3 changes: 2 additions & 1 deletion src/include/homeobject/pg_manager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ struct PGInfo {
pg_id_t id;
mutable MemberSet members;
peer_id_t replica_set_uuid;
u_int64_t size;
uint64_t size;
uint64_t chunk_size;

auto operator<=>(PGInfo const& rhs) const { return id <=> rhs.id; }
auto operator==(PGInfo const& rhs) const { return id == rhs.id; }
Expand Down
2 changes: 1 addition & 1 deletion src/include/homeobject/shard_manager.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
namespace homeobject {

ENUM(ShardError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, NOT_LEADER, UNSUPPORTED_OP, UNKNOWN_PG, UNKNOWN_SHARD,
PG_NOT_READY, CRC_MISMATCH);
PG_NOT_READY, CRC_MISMATCH, NO_SPACE_LEFT);

struct ShardInfo {
enum class State : uint8_t {
Expand Down
291 changes: 192 additions & 99 deletions src/lib/homestore_backend/heap_chunk_selector.cpp
View file

Large diffs are not rendered by default.

61 changes: 40 additions & 21 deletions src/lib/homestore_backend/heap_chunk_selector.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ class HeapChunkSelector : public homestore::ChunkSelector {

using VChunkHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkComparator >;
using VChunkDefragHeap = std::priority_queue< VChunk, std::vector< VChunk >, VChunkDefragComparator >;
using ChunkIdMap = std::unordered_map < homestore::chunk_num_t, homestore::chunk_num_t >; // used for real chunk id -> virtual chunk id map
using ChunkIdMap =
std::unordered_map< homestore::chunk_num_t,
homestore::chunk_num_t >; // used for physical chunk id -> virtual chunk id map
using chunk_num_t = homestore::chunk_num_t;

struct ChunkHeap {
Expand All @@ -52,16 +54,16 @@ class HeapChunkSelector : public homestore::ChunkSelector {

csharedChunk select_chunk([[maybe_unused]] homestore::blk_count_t nblks, const homestore::blk_alloc_hints& hints);

// this function will be used by GC flow or recovery flow to mark one specific chunk to be busy, caller should be
// responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t);
// this function will be used by create shard or recovery flow to mark one specific chunk to be busy, caller should
// be responsible to use release_chunk() interface to release it when no longer to use the chunk anymore.
csharedChunk select_specific_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);

// this function will be used by GC flow to select a chunk for GC
csharedChunk most_defrag_chunk();

// this function is used to return a chunk back to ChunkSelector when sealing a shard, and will only be used by
// Homeobject.
void release_chunk(const pg_id_t pg_id, const chunk_num_t);
// This function returns a chunk back to ChunkSelector.
// It is used in two scenarios: 1. seal shard 2. create shard rollback
bool release_chunk(const pg_id_t pg_id, const chunk_num_t v_chunk_id);

/**
* select chunks for pg, chunks need to be in same pdev.
Expand All @@ -70,26 +72,35 @@ class HeapChunkSelector : public homestore::ChunkSelector {
* @param pg_size The fix pg size.
* @return An optional uint32_t value representing num_chunk, or std::nullopt if no space left.
*/
std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, u_int64_t pg_size);
std::optional< uint32_t > select_chunks_for_pg(pg_id_t pg_id, uint64_t pg_size);

std::shared_ptr< const std::vector <chunk_num_t> > get_pg_chunks(pg_id_t pg_id) const;
// this function is used for pg info superblk persist v_chunk_id <-> p_chunk_id
std::shared_ptr< const std::vector< chunk_num_t > > get_pg_chunks(pg_id_t pg_id) const;

/**
* pop pg top chunk
*
* @param pg_id The ID of the pg.
* @return An optional chunk_num_t value representing v_chunk_id, or std::nullopt if no space left.
*/
std::optional< chunk_num_t > pop_top_chunk(pg_id_t pg_id) const;

// this should be called on each pg meta blk found
void set_pg_chunks(pg_id_t pg_id, std::vector<chunk_num_t>&& chunk_ids);
bool set_pg_chunks(pg_id_t pg_id, std::vector< chunk_num_t >&& p_chunk_ids);

// this should be called after all pg meta blk recovered
void recover_per_dev_chunk_heap();

// this should be called after ShardManager is initialized and get all the open shards
void recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excludingChunks);
bool recover_pg_chunk_heap(pg_id_t pg_id, const std::unordered_set< chunk_num_t >& excluding_v_chunk_ids);

/**
* Retrieves the block allocation hints for a given chunk.
*
* @param chunk_id The ID of the chunk.
* @param p_chunk_id The physical chunk id.
* @return The block allocation hints for the specified chunk.
*/
homestore::blk_alloc_hints chunk_to_hints(chunk_num_t chunk_id) const;
homestore::blk_alloc_hints chunk_to_hints(chunk_num_t p_chunk_id) const;

/**
* Returns the number of available blocks of the given device id.
Expand Down Expand Up @@ -134,25 +145,33 @@ class HeapChunkSelector : public homestore::ChunkSelector {

uint32_t get_chunk_size() const;

private:
std::optional< chunk_num_t > get_virtual_chunk_id(pg_id_t pg_id, chunk_num_t p_chunk_id) const;

std::optional< chunk_num_t > get_physical_chunk_id(pg_id_t pg_id, chunk_num_t v_chunk_id) const;

void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);

void remove_chunk_from_defrag_heap(const chunk_num_t);

private:
std::unordered_map< uint32_t, std::shared_ptr< ChunkHeap > > m_per_dev_heap;
std::unordered_map< pg_id_t, std::shared_ptr< ChunkHeap > > m_per_pg_heap;

// These mappings ensure "identical layout" by providing bidirectional indexing between virtual and real chunk IDs.
// m_v2r_chunk_map: Maps each pg_id to a vector of real chunk IDs (r_chunk_id). The index in the vector corresponds to the virtual chunk ID (v_chunk_id).
std::unordered_map< pg_id_t, std::shared_ptr< std::vector <chunk_num_t> > > m_v2r_chunk_map;
// m_r2v_chunk_map: Maps each pg_id to a map that inversely maps real chunk IDs (r_chunk_id) to virtual chunk IDs (v_chunk_id).
std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_r2v_chunk_map;
// These mappings ensure "identical layout" by providing bidirectional indexing between virtual and physical chunk
// IDs. m_v2p_chunk_map: Maps each pg_id to a vector of physical chunk IDs (p_chunk_id). The index in the vector
// corresponds to the virtual chunk ID (v_chunk_id).
std::unordered_map< pg_id_t, std::shared_ptr< std::vector< chunk_num_t > > > m_v2p_chunk_map;
// m_p2v_chunk_map: Maps each pg_id to a map that inversely maps physical chunk IDs (p_chunk_id) to virtual chunk
// IDs (v_chunk_id).
std::unordered_map< pg_id_t, std::shared_ptr< ChunkIdMap > > m_p2v_chunk_map;

// hold all the chunks , selected or not
std::unordered_map< chunk_num_t, csharedChunk > m_chunks;

mutable std::shared_mutex m_chunk_selector_mtx;
void add_chunk_internal(const chunk_num_t, bool add_to_heap = true);

VChunkDefragHeap m_defrag_heap;
std::mutex m_defrag_mtx;

void remove_chunk_from_defrag_heap(const chunk_num_t);
};
} // namespace homeobject
4 changes: 2 additions & 2 deletions src/lib/homestore_backend/hs_blob_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,10 +351,10 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive<
}

auto hs_shard = d_cast< HS_Shard* >((*shard_iter->second).get());
BLOGD(msg_header->shard_id, "n/a", "Picked chunk_id={}", hs_shard->sb_->chunk_id);
BLOGD(msg_header->shard_id, "n/a", "Picked p_chunk_id={}", hs_shard->sb_->p_chunk_id);

homestore::blk_alloc_hints hints;
hints.chunk_id_hint = hs_shard->sb_->chunk_id;
hints.chunk_id_hint = hs_shard->sb_->p_chunk_id;
return hints;
}

Expand Down
56 changes: 37 additions & 19 deletions src/lib/homestore_backend/hs_homeobject.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,15 @@ class HSHomeObject : public HomeObjectImpl {
// Data layout inside 'data':
// First, an array of 'pg_members' structures:
// | pg_members[0] | pg_members[1] | ... | pg_members[num_members-1] |
// Immediately followed by an array of 'chunk_num_t' values (representing r_chunk_ids):
// Immediately followed by an array of 'chunk_num_t' values (representing physical chunkID):
// | chunk_num_t[0] | chunk_num_t[1] | ... | chunk_num_t[num_chunks-1] |
// Here, 'chunk_num_t[i]' represents the r_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and increases sequentially.
// Here, 'chunk_num_t[i]' represents the p_chunk_id for the v_chunk_id 'i', where v_chunk_id starts from 0 and
// increases sequentially.


uint32_t size() const { return sizeof(pg_info_superblk) - sizeof(char) + num_members * sizeof(pg_members) + num_chunks * sizeof(homestore::chunk_num_t); }
uint32_t size() const {
return sizeof(pg_info_superblk) - sizeof(char) + num_members * sizeof(pg_members) +
num_chunks * sizeof(homestore::chunk_num_t);
}
static std::string name() { return _pg_meta_name; }

pg_info_superblk() = default;
Expand All @@ -117,11 +120,15 @@ class HSHomeObject : public HomeObjectImpl {

void copy(pg_info_superblk const& rhs) { *this = rhs; }

pg_members* get_pg_members_mutable() { return reinterpret_cast<pg_members*>(data); }
const pg_members* get_pg_members() const { return reinterpret_cast<const pg_members*>(data); }
pg_members* get_pg_members_mutable() { return reinterpret_cast< pg_members* >(data); }
const pg_members* get_pg_members() const { return reinterpret_cast< const pg_members* >(data); }

homestore::chunk_num_t* get_chunk_ids_mutable() { return reinterpret_cast<homestore::chunk_num_t*>(data + num_members * sizeof(pg_members)); }
const homestore::chunk_num_t* get_chunk_ids() const { return reinterpret_cast<const homestore::chunk_num_t*>(data + num_members * sizeof(pg_members)); }
homestore::chunk_num_t* get_chunk_ids_mutable() {
return reinterpret_cast< homestore::chunk_num_t* >(data + num_members * sizeof(pg_members));
}
const homestore::chunk_num_t* get_chunk_ids() const {
return reinterpret_cast< const homestore::chunk_num_t* >(data + num_members * sizeof(pg_members));
}
};

struct DataHeader {
Expand All @@ -141,7 +148,8 @@ class HSHomeObject : public HomeObjectImpl {

struct shard_info_superblk : public DataHeader {
ShardInfo info;
homestore::chunk_num_t chunk_id;
homestore::chunk_num_t p_chunk_id;
homestore::chunk_num_t v_chunk_id;
};
#pragma pack()

Expand Down Expand Up @@ -210,11 +218,11 @@ class HSHomeObject : public HomeObjectImpl {
public:
homestore::superblk< pg_info_superblk > pg_sb_;
shared< homestore::ReplDev > repl_dev_;
std::optional< homestore::chunk_num_t > any_allocated_chunk_id_{};
std::shared_ptr< BlobIndexTable > index_table_;
PGMetrics metrics_;

HS_PG(PGInfo info, shared< homestore::ReplDev > rdev, shared< BlobIndexTable > index_table, std::shared_ptr< const std::vector <homestore::chunk_num_t> > pg_chunk_ids);
HS_PG(PGInfo info, shared< homestore::ReplDev > rdev, shared< BlobIndexTable > index_table,
std::shared_ptr< const std::vector< homestore::chunk_num_t > > pg_chunk_ids);
HS_PG(homestore::superblk< pg_info_superblk >&& sb, shared< homestore::ReplDev > rdev);
~HS_PG() override = default;

Expand Down Expand Up @@ -244,12 +252,12 @@ class HSHomeObject : public HomeObjectImpl {

struct HS_Shard : public Shard {
homestore::superblk< shard_info_superblk > sb_;
HS_Shard(ShardInfo info, homestore::chunk_num_t chunk_id);
HS_Shard(ShardInfo info, homestore::chunk_num_t p_chunk_id);
HS_Shard(homestore::superblk< shard_info_superblk >&& sb);
~HS_Shard() override = default;

void update_info(const ShardInfo& info);
auto chunk_id() const { return sb_->chunk_id; }
auto p_chunk_id() const { return sb_->p_chunk_id; }
};

#pragma pack(1)
Expand Down Expand Up @@ -429,9 +437,17 @@ class HSHomeObject : public HomeObjectImpl {
* @brief Retrieves the chunk number associated with the given shard ID.
*
* @param id The ID of the shard to retrieve the chunk number for.
* @return An optional chunk number if the shard ID is valid, otherwise an empty optional.
* @return An optional chunk number values shard p_chunk_id if the shard ID is valid, otherwise an empty optional.
*/
std::optional< homestore::chunk_num_t > get_shard_p_chunk_id(shard_id_t id) const;

/**
* @brief Retrieves the chunk number associated with the given shard ID.
*
* @param id The ID of the shard to retrieve the chunk number for.
* @return An optional chunk number values shard v_chunk_id if the shard ID is valid, otherwise an empty optional.
*/
std::optional< homestore::chunk_num_t > get_shard_chunk(shard_id_t id) const;
std::optional< homestore::chunk_num_t > get_shard_v_chunk_id(shard_id_t id) const;

/**
* @brief recover PG and shard from the superblock.
Expand All @@ -440,12 +456,14 @@ class HSHomeObject : public HomeObjectImpl {
void on_replica_restart();

/**
* @brief Returns any chunk number for the given pg ID.
* @brief Extracts the physical chunk ID for create shard from the message.
*
* @param pg The pg ID to get the chunk number for.
* @return A tuple of <if pg exist, if shard exist, chunk number if both exist>.
* @param header The message header that includes the shard_info_superblk, which contains the data necessary for
* extracting and mapping the chunk ID.
* @return An optional virtual chunk id if the extraction and mapping process is successful, otherwise an empty
* optional.
*/
std::tuple< bool, bool, homestore::chunk_num_t > get_any_chunk_id(pg_id_t pg);
std::optional< homestore::chunk_num_t > resolve_v_chunk_id_from_msg(sisl::blob const& header);

cshared< HeapChunkSelector > chunk_selector() const { return chunk_selector_; }

Expand Down
38 changes: 27 additions & 11 deletions src/lib/homestore_backend/hs_pg_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ PGManager::NullAsyncResult HSHomeObject::_create_pg(PGInfo&& pg_info, std::set<
return folly::makeUnexpected(PGError::NO_SPACE_LEFT);
}

pg_info.chunk_size = chunk_size;
pg_info.replica_set_uuid = boost::uuids::random_generator()();
return hs_repl_service()
.create_repl_dev(pg_info.replica_set_uuid, peers)
Expand Down Expand Up @@ -136,15 +137,27 @@ void HSHomeObject::on_create_pg_message_commit(int64_t lsn, sisl::blob const& he
return;
}

auto local_chunk_size = chunk_selector()->get_chunk_size();
if (pg_info.chunk_size != local_chunk_size) {
LOGE("Chunk sizes are inconsistent, leader_chunk_size={}, local_chunk_size={}", pg_info.chunk_size,
local_chunk_size);
if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::UNKNOWN)); }
return;
}

// select chunks for pg
auto const num_chunk = chunk_selector()->select_chunks_for_pg(pg_id, pg_info.size);
if (!num_chunk.has_value()) {
LOGW("select chunks for pg failed, pg_id {}", pg_id);
LOGW("Failed to select chunks for pg {}", pg_id);
if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); }
return;
}
auto chunk_ids = chunk_selector()->get_pg_chunks(pg_id);

if (chunk_ids == nullptr) {
LOGW("Failed to get pg chunks, pg_id {}", pg_id);
if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(PGError::NO_SPACE_LEFT)); }
return;
}
// create index table and pg
// TODO create index table during create shard.
auto index_table = create_index_table();
Expand Down Expand Up @@ -245,6 +258,7 @@ std::string HSHomeObject::serialize_pg_info(const PGInfo& pginfo) {
nlohmann::json j;
j["pg_info"]["pg_id_t"] = pginfo.id;
j["pg_info"]["pg_size"] = pginfo.size;
j["pg_info"]["chunk_size"] = pginfo.chunk_size;
j["pg_info"]["repl_uuid"] = boost::uuids::to_string(pginfo.replica_set_uuid);

nlohmann::json members_j{};
Expand All @@ -263,7 +277,8 @@ PGInfo HSHomeObject::deserialize_pg_info(const unsigned char* json_str, size_t s
auto pg_json = nlohmann::json::parse(json_str, json_str + size);

PGInfo pg_info(pg_json["pg_info"]["pg_id_t"].get< pg_id_t >());
pg_info.size = pg_json["pg_info"]["pg_size"].get< u_int64_t >();
pg_info.size = pg_json["pg_info"]["pg_size"].get< uint64_t >();
pg_info.chunk_size = pg_json["pg_info"]["chunk_size"].get< uint64_t >();
pg_info.replica_set_uuid = boost::uuids::string_generator()(pg_json["pg_info"]["repl_uuid"].get< std::string >());

for (auto const& m : pg_json["pg_info"]["members"]) {
Expand All @@ -287,8 +302,9 @@ void HSHomeObject::on_pg_meta_blk_found(sisl::byte_view const& buf, void* meta_c
return;
}
auto pg_id = pg_sb->id;
std::vector<chunk_num_t> chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks);
chunk_selector_->set_pg_chunks(pg_id, std::move(chunk_ids));
std::vector< chunk_num_t > p_chunk_ids(pg_sb->get_chunk_ids(), pg_sb->get_chunk_ids() + pg_sb->num_chunks);
bool set_pg_chunks_res = chunk_selector_->set_pg_chunks(pg_id, std::move(p_chunk_ids));
RELEASE_ASSERT(set_pg_chunks_res, "Failed to set pg={} chunks", pg_id);
auto uuid_str = boost::uuids::to_string(pg_sb->index_table_uuid);
auto hs_pg = std::make_unique< HS_PG >(std::move(pg_sb), std::move(v.value()));
// During PG recovery check if index is already recoverd else
Expand All @@ -302,9 +318,7 @@ void HSHomeObject::on_pg_meta_blk_found(sisl::byte_view const& buf, void* meta_c
add_pg_to_map(std::move(hs_pg));
}

void HSHomeObject::on_pg_meta_blk_recover_completed(bool success) {
chunk_selector_->recover_per_dev_chunk_heap();
}
void HSHomeObject::on_pg_meta_blk_recover_completed(bool success) { chunk_selector_->recover_per_dev_chunk_heap(); }

PGInfo HSHomeObject::HS_PG::pg_info_from_sb(homestore::superblk< pg_info_superblk > const& sb) {
PGInfo pginfo{sb->id};
Expand All @@ -317,15 +331,17 @@ PGInfo HSHomeObject::HS_PG::pg_info_from_sb(homestore::superblk< pg_info_superbl
return pginfo;
}

HSHomeObject::HS_PG::HS_PG(PGInfo info, shared< homestore::ReplDev > rdev, shared< BlobIndexTable > index_table, std::shared_ptr< const std::vector <chunk_num_t> > pg_chunk_ids) :
HSHomeObject::HS_PG::HS_PG(PGInfo info, shared< homestore::ReplDev > rdev, shared< BlobIndexTable > index_table,
std::shared_ptr< const std::vector< chunk_num_t > > pg_chunk_ids) :
PG{std::move(info)},
pg_sb_{_pg_meta_name},
repl_dev_{std::move(rdev)},
index_table_{std::move(index_table)},
metrics_{*this} {
RELEASE_ASSERT(pg_chunk_ids != nullptr, "PG chunks null");
const uint32_t num_chunks = pg_chunk_ids->size();
pg_sb_.create(sizeof(pg_info_superblk) - sizeof(char) + pg_info_.members.size() * sizeof(pg_members)+ num_chunks * sizeof(homestore::chunk_num_t));
pg_sb_.create(sizeof(pg_info_superblk) - sizeof(char) + pg_info_.members.size() * sizeof(pg_members) +
num_chunks * sizeof(homestore::chunk_num_t));
pg_sb_->id = pg_info_.id;
pg_sb_->num_members = pg_info_.members.size();
pg_sb_->num_chunks = num_chunks;
Expand Down Expand Up @@ -368,7 +384,7 @@ uint32_t HSHomeObject::HS_PG::open_shards() const {
std::optional< uint32_t > HSHomeObject::HS_PG::dev_hint(cshared< HeapChunkSelector > chunk_sel) const {
if (shards_.empty()) { return std::nullopt; }
auto const hs_shard = d_cast< HS_Shard* >(shards_.front().get());
auto const hint = chunk_sel->chunk_to_hints(hs_shard->chunk_id());
auto const hint = chunk_sel->chunk_to_hints(hs_shard->p_chunk_id());
return hint.pdev_id_hint;
}

Expand Down
Loading

0 comments on commit 5e8f05b

Please sign in to comment.