Skip to content

Commit

Permalink
add pg recovery test
Browse files Browse the repository at this point in the history
  • Loading branch information
JacksonYao287 committed Jun 5, 2024
1 parent acb04e8 commit c268e57
Show file tree
Hide file tree
Showing 8 changed files with 136 additions and 100 deletions.
2 changes: 1 addition & 1 deletion conanfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class HomeObjectConan(ConanFile):
name = "homeobject"
version = "2.0.2"
version = "2.0.3"
homepage = "https://github.com/eBay/HomeObject"
description = "Blob Store built on HomeReplication"
topics = ("ebay")
Expand Down
1 change: 1 addition & 0 deletions src/lib/homestore_backend/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ include_directories (BEFORE .)
list(APPEND TEST_SOURCES
hs_shard_tests.cpp
hs_blob_tests.cpp
hs_pg_tests.cpp
homeobj_cp_tests.cpp
)

Expand Down
14 changes: 5 additions & 9 deletions src/lib/homestore_backend/tests/homeobj_cp_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#include "homeobj_fixture.hpp"

#if 0
TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) {
// Step-1: create a PG and a shard
std::vector< std::pair< pg_id_t, shard_id_t > > pg_shard_id_vec;
Expand Down Expand Up @@ -30,22 +29,19 @@ TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) {
// Step-3: trigger a cp;
trigger_cp(true /* wait */);

// TODO:enable this after we have recovery ability for raft repl dev
/*
_obj_inst.reset();
_obj_inst.reset();

// Step-4: re-create the homeobject and pg infos and shard infos will be recover automatically.
_obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app));

// Step-4: re-create the homeobject and pg infos and shard infos will be recover automatically.
_obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app));
*/
ho = dynamic_cast< homeobject::HSHomeObject* >(_obj_inst.get());

EXPECT_TRUE(ho->_pg_map.size() == 1);
{
auto lg = std::shared_lock(ho->_pg_lock);
for (auto& [_, pg] : ho->_pg_map) {
auto hs_pg = static_cast< HSHomeObject::HS_PG* >(pg.get());
EXPECT_EQ(hs_pg->cache_pg_sb_->blob_sequence_num, 12345);
EXPECT_EQ(hs_pg->durable_entities_.blob_sequence_num, 12345);
}
}
}
#endif
17 changes: 17 additions & 0 deletions src/lib/homestore_backend/tests/homeobj_fixture.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,23 @@ class HomeObjectFixture : public ::testing::Test {
}
}

void verify_pg_sb(const HSHomeObject::pg_info_superblk* lhs, const HSHomeObject::pg_info_superblk* rhs) {
EXPECT_EQ(lhs->id, rhs->id);
EXPECT_EQ(lhs->num_members, rhs->num_members);
EXPECT_EQ(lhs->replica_set_uuid, rhs->replica_set_uuid);
EXPECT_EQ(lhs->index_table_uuid, rhs->index_table_uuid);
EXPECT_EQ(lhs->blob_sequence_num, rhs->blob_sequence_num);
EXPECT_EQ(lhs->active_blob_count, rhs->active_blob_count);
EXPECT_EQ(lhs->tombstone_blob_count, rhs->tombstone_blob_count);
EXPECT_EQ(lhs->total_occupied_blk_count, rhs->total_occupied_blk_count);
EXPECT_EQ(lhs->tombstone_blob_count, rhs->tombstone_blob_count);
for (uint32_t i = 0; i < lhs->num_members; i++) {
EXPECT_EQ(lhs->members[i].id, rhs->members[i].id);
EXPECT_EQ(lhs->members[i].priority, rhs->members[i].priority);
EXPECT_EQ(0, std::strcmp(lhs->members[i].name, rhs->members[i].name));
}
}

void restart() {
LOGINFO("Restarting homeobject.");
_obj_inst.reset();
Expand Down
53 changes: 0 additions & 53 deletions src/lib/homestore_backend/tests/hs_blob_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -181,56 +181,3 @@ TEST_F(HomeObjectFixture, SealShardWithRestart) {
ASSERT_EQ(b.error(), BlobError::SEALED_SHARD);
LOGINFO("Put blob {}", b.error());
}

TEST_F(HomeObjectFixture, PGStatsTest) {
// Create a pg, shard, put blob should succeed.
pg_id_t pg_id{1};
create_pg(pg_id);

auto s = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get();
ASSERT_TRUE(!!s);
auto shard_info = s.value();
auto shard_id = shard_info.id;
s = _obj_inst->shard_manager()->get_shard(shard_id).get();
ASSERT_TRUE(!!s);

LOGINFO("Got shard {}", shard_id);
shard_info = s.value();
EXPECT_EQ(shard_info.id, shard_id);
EXPECT_EQ(shard_info.placement_group, pg_id);
EXPECT_EQ(shard_info.state, ShardInfo::State::OPEN);
auto b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get();
ASSERT_TRUE(!!b);
LOGINFO("Put blob {}", b.value());

// create a shard
s = _obj_inst->shard_manager()->seal_shard(shard_id).get();
ASSERT_TRUE(!!s);
shard_info = s.value();
EXPECT_EQ(shard_info.id, shard_id);
EXPECT_EQ(shard_info.placement_group, pg_id);
EXPECT_EQ(shard_info.state, ShardInfo::State::SEALED);
LOGINFO("Sealed shard {}", shard_id);

// create a 2nd shard
auto s2 = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get();
auto shard_info2 = s2.value();
auto shard_id2 = shard_info2.id;
s2 = _obj_inst->shard_manager()->get_shard(shard_id2).get();
ASSERT_TRUE(!!s);
LOGINFO("Got shard {}", shard_id2);

PGStats pg_stats;
auto res = _obj_inst->pg_manager()->get_stats(pg_id, pg_stats);
LOGINFO("stats: {}", pg_stats.to_string());

EXPECT_EQ(res, true);
EXPECT_EQ(pg_stats.id, pg_id);
EXPECT_EQ(pg_stats.total_shards, 2);
EXPECT_EQ(pg_stats.open_shards, 1);
// TODO: EXPECT_EQ(pg_stats.num_members, 1) after having real 3-replica repl dev in test
EXPECT_EQ(pg_stats.num_members, 1);

auto stats = _obj_inst->get_stats();
LOGINFO("HomeObj stats: {}", stats.to_string());
}
89 changes: 89 additions & 0 deletions src/lib/homestore_backend/tests/hs_pg_tests.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#include "homeobj_fixture.hpp"

TEST_F(HomeObjectFixture, PGStatsTest) {
// Create a pg, shard, put blob should succeed.
pg_id_t pg_id{1};
create_pg(pg_id);

auto s = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get();
ASSERT_TRUE(!!s);
auto shard_info = s.value();
auto shard_id = shard_info.id;
s = _obj_inst->shard_manager()->get_shard(shard_id).get();
ASSERT_TRUE(!!s);

LOGINFO("Got shard {}", shard_id);
shard_info = s.value();
EXPECT_EQ(shard_info.id, shard_id);
EXPECT_EQ(shard_info.placement_group, pg_id);
EXPECT_EQ(shard_info.state, ShardInfo::State::OPEN);
auto b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get();
ASSERT_TRUE(!!b);
LOGINFO("Put blob {}", b.value());

// create a shard
s = _obj_inst->shard_manager()->seal_shard(shard_id).get();
ASSERT_TRUE(!!s);
shard_info = s.value();
EXPECT_EQ(shard_info.id, shard_id);
EXPECT_EQ(shard_info.placement_group, pg_id);
EXPECT_EQ(shard_info.state, ShardInfo::State::SEALED);
LOGINFO("Sealed shard {}", shard_id);

// create a 2nd shard
auto s2 = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get();
auto shard_info2 = s2.value();
auto shard_id2 = shard_info2.id;
s2 = _obj_inst->shard_manager()->get_shard(shard_id2).get();
ASSERT_TRUE(!!s);
LOGINFO("Got shard {}", shard_id2);

PGStats pg_stats;
auto res = _obj_inst->pg_manager()->get_stats(pg_id, pg_stats);
LOGINFO("stats: {}", pg_stats.to_string());

EXPECT_EQ(res, true);
EXPECT_EQ(pg_stats.id, pg_id);
EXPECT_EQ(pg_stats.total_shards, 2);
EXPECT_EQ(pg_stats.open_shards, 1);
// TODO: EXPECT_EQ(pg_stats.num_members, 1) after having real 3-replica repl dev in test
EXPECT_EQ(pg_stats.num_members, 1);

auto stats = _obj_inst->get_stats();
LOGINFO("HomeObj stats: {}", stats.to_string());
}

TEST_F(HomeObjectFixture, PGRecoveryTest) {
// create 10 pg
for (pg_id_t i = 0; i < 10; i++) {
pg_id_t pg_id{i};
create_pg(pg_id);
}

// get pg map
HSHomeObject* ho = dynamic_cast< HSHomeObject* >(_obj_inst.get());
std::map< pg_id_t, std::unique_ptr< PG > > pg_map;
pg_map.swap(ho->_pg_map);

// get uuid
auto id = ho->our_uuid();

// restart
restart();

ho = dynamic_cast< HSHomeObject* >(_obj_inst.get());
// verify uuid
EXPECT_EQ(id, ho->our_uuid());

// verify pg map
EXPECT_EQ(10, ho->_pg_map.size());

for (auto const& [id, pg] : ho->_pg_map) {
EXPECT_TRUE(pg_map.contains(id));
auto reserved_pg = dynamic_cast< HSHomeObject::HS_PG* >(pg_map[id].get());
auto recovered_pg = dynamic_cast< HSHomeObject::HS_PG* >(pg.get());
EXPECT_TRUE(reserved_pg);
EXPECT_TRUE(recovered_pg);
verify_pg_sb(reserved_pg->pg_sb_.get(), recovered_pg->pg_sb_.get());
}
}
58 changes: 22 additions & 36 deletions src/lib/homestore_backend/tests/hs_shard_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
#include "lib/homestore_backend/hs_homeobject.hpp"
#include "lib/homestore_backend/replication_message.hpp"
#include "lib/homestore_backend/replication_state_machine.hpp"

#include "lib/tests/fixture_app.hpp"

using homeobject::shard_id_t;
Expand Down Expand Up @@ -72,10 +71,10 @@ TEST_F(TestFixture, CreateMultiShardsOnMultiPG) {
}
}

#if 0
TEST_F(TestFixture, MockSealShard) {
ShardInfo shard_info = _shard_1;
shard_info.state = ShardInfo::State::SEALED;
using shard_info_superblk = homeobject::HSHomeObject::shard_info_superblk;

nlohmann::json j;
j["shard_info"]["shard_id_t"] = shard_info.id;
Expand All @@ -91,25 +90,27 @@ TEST_F(TestFixture, MockSealShard) {
homeobject::HSHomeObject* ho = dynamic_cast< homeobject::HSHomeObject* >(homeobj_.get());
auto* pg = s_cast< homeobject::HSHomeObject::HS_PG* >(ho->_pg_map[_pg_id].get());
auto repl_dev = pg->repl_dev_;
const auto msg_size = sisl::round_up(seal_shard_msg.size(), repl_dev->get_blk_size());
auto req = homeobject::repl_result_ctx< ShardManager::Result< ShardInfo > >::make(msg_size, 512 /*alignment*/);
auto buf_ptr = req->hdr_buf_.bytes();
std::memset(buf_ptr, 0, msg_size);
std::memcpy(buf_ptr, seal_shard_msg.c_str(), seal_shard_msg.size());

req->header_.msg_type = homeobject::ReplicationMessageType::SEAL_SHARD_MSG;
req->header_.pg_id = _pg_id;
req->header_.shard_id = shard_info.id;
req->header_.payload_size = msg_size;
req->header_.payload_crc = crc32_ieee(homeobject::init_crc32, buf_ptr, msg_size);
req->header_.seal();
sisl::blob header;
header.set_bytes(r_cast< uint8_t* >(&req->header_));
header.set_size(sizeof(req->header_));
sisl::sg_list value;
value.size = msg_size;
value.iovs.push_back(iovec(buf_ptr, msg_size));
repl_dev->async_alloc_write(header, sisl::blob{buf_ptr, (uint32_t)msg_size}, value, req);

sisl::io_blob_safe sb_blob(sisl::round_up(sizeof(shard_info_superblk), repl_dev->get_blk_size()),
homeobject::io_align);
shard_info_superblk* sb = new (sb_blob.bytes()) shard_info_superblk();
sb->type = homeobject::HSHomeObject::DataHeader::data_type_t::SHARD_INFO;
sb->info = shard_info;
sb->chunk_id = 0;

auto req = homeobject::repl_result_ctx< ShardManager::Result< ShardInfo > >::make(sizeof(shard_info_superblk), 0u);

req->header()->msg_type = homeobject::ReplicationMessageType::SEAL_SHARD_MSG;
req->header()->pg_id = _pg_id;
req->header()->shard_id = shard_info.id;
req->header()->payload_size = sizeof(shard_info_superblk);
req->header()->payload_crc = crc32_ieee(homeobject::init_crc32, sb_blob.cbytes(), sizeof(shard_info_superblk));
req->header()->seal();

std::memcpy(req->header_extn(), sb_blob.cbytes(), sizeof(shard_info_superblk));
req->add_data_sg(std::move(sb_blob));

repl_dev->async_alloc_write(req->cheader_buf(), sisl::blob{}, req->data_sgs(), req);
auto info = req->result().get();
EXPECT_TRUE(info);
EXPECT_TRUE(info.value().id == shard_info.id);
Expand All @@ -123,7 +124,6 @@ TEST_F(TestFixture, MockSealShard) {
auto& check_shard = pg_result->shards_.front();
EXPECT_EQ(ShardInfo::State::SEALED, check_shard->info.state);
}
#endif

class ShardManagerTestingRecovery : public ::testing::Test {
public:
Expand Down Expand Up @@ -179,13 +179,6 @@ TEST_F(ShardManagerTestingRecovery, ShardManagerRecovery) {
auto hs_shard = d_cast< homeobject::HSHomeObject::HS_Shard* >(check_shard);
auto& recovered_shard_info = hs_shard->info;
EXPECT_TRUE(recovered_shard_info == shard_info);
EXPECT_TRUE(recovered_shard_info.placement_group == shard_info.placement_group);
EXPECT_TRUE(recovered_shard_info.state == shard_info.state);
EXPECT_TRUE(recovered_shard_info.created_time == shard_info.created_time);
EXPECT_TRUE(recovered_shard_info.last_modified_time == shard_info.last_modified_time);
EXPECT_TRUE(recovered_shard_info.available_capacity_bytes == shard_info.available_capacity_bytes);
EXPECT_TRUE(recovered_shard_info.total_capacity_bytes == shard_info.total_capacity_bytes);
EXPECT_TRUE(recovered_shard_info.deleted_capacity_bytes == shard_info.deleted_capacity_bytes);

// seal the shard when shard is recovery
e = _home_object->shard_manager()->seal_shard(shard_id).get();
Expand Down Expand Up @@ -262,13 +255,6 @@ TEST_F(ShardManagerTestingRecovery, SealedShardRecovery) {
auto hs_shard = d_cast< homeobject::HSHomeObject::HS_Shard* >(pg_iter->second->shards_.front().get());
auto& recovered_shard_info = hs_shard->info;
EXPECT_TRUE(recovered_shard_info == shard_info);
EXPECT_TRUE(recovered_shard_info.placement_group == shard_info.placement_group);
EXPECT_TRUE(recovered_shard_info.state == shard_info.state);
EXPECT_TRUE(recovered_shard_info.created_time == shard_info.created_time);
EXPECT_TRUE(recovered_shard_info.last_modified_time == shard_info.last_modified_time);
EXPECT_TRUE(recovered_shard_info.available_capacity_bytes == shard_info.available_capacity_bytes);
EXPECT_TRUE(recovered_shard_info.total_capacity_bytes == shard_info.total_capacity_bytes);
EXPECT_TRUE(recovered_shard_info.deleted_capacity_bytes == shard_info.deleted_capacity_bytes);
// finally close the homeobject and homestore.
_home_object.reset();
}
2 changes: 1 addition & 1 deletion src/lib/tests/fixture_app.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class FixtureApp : public homeobject::HomeObjectApplication {
bool is_hybrid_{false};

public:
FixtureApp(bool is_hybrid=false);
FixtureApp(bool is_hybrid = false);
~FixtureApp() = default;

bool spdk_mode() const override { return false; }
Expand Down

0 comments on commit c268e57

Please sign in to comment.