From c268e57b3a20f99a6339bedb67e03919b0bbdb09 Mon Sep 17 00:00:00 2001 From: Jie Yao Date: Tue, 4 Jun 2024 20:39:51 -0700 Subject: [PATCH] add pg recovery test --- conanfile.py | 2 +- .../homestore_backend/tests/CMakeLists.txt | 1 + .../tests/homeobj_cp_tests.cpp | 14 ++- .../tests/homeobj_fixture.hpp | 17 ++++ .../homestore_backend/tests/hs_blob_tests.cpp | 53 ----------- .../homestore_backend/tests/hs_pg_tests.cpp | 89 +++++++++++++++++++ .../tests/hs_shard_tests.cpp | 58 +++++------- src/lib/tests/fixture_app.hpp | 2 +- 8 files changed, 136 insertions(+), 100 deletions(-) create mode 100644 src/lib/homestore_backend/tests/hs_pg_tests.cpp diff --git a/conanfile.py b/conanfile.py index 016b3c89..0fcb6652 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "2.0.2" + version = "2.0.3" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" topics = ("ebay") diff --git a/src/lib/homestore_backend/tests/CMakeLists.txt b/src/lib/homestore_backend/tests/CMakeLists.txt index b8a4f0f9..7eda153a 100644 --- a/src/lib/homestore_backend/tests/CMakeLists.txt +++ b/src/lib/homestore_backend/tests/CMakeLists.txt @@ -5,6 +5,7 @@ include_directories (BEFORE .) list(APPEND TEST_SOURCES hs_shard_tests.cpp hs_blob_tests.cpp + hs_pg_tests.cpp homeobj_cp_tests.cpp ) diff --git a/src/lib/homestore_backend/tests/homeobj_cp_tests.cpp b/src/lib/homestore_backend/tests/homeobj_cp_tests.cpp index 48b1857f..505a715a 100644 --- a/src/lib/homestore_backend/tests/homeobj_cp_tests.cpp +++ b/src/lib/homestore_backend/tests/homeobj_cp_tests.cpp @@ -1,6 +1,5 @@ #include "homeobj_fixture.hpp" -#if 0 TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) { // Step-1: create a PG and a shard std::vector< std::pair< pg_id_t, shard_id_t > > pg_shard_id_vec; @@ -30,13 +29,11 @@ TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) { // Step-3: trigger a cp; trigger_cp(true /* wait */); - // TODO:enable this after we have recovery ability for raft repl dev - /* - _obj_inst.reset(); + _obj_inst.reset(); + + // Step-4: re-create the homeobject and pg infos and shard infos will be recover automatically. + _obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app)); - // Step-4: re-create the homeobject and pg infos and shard infos will be recover automatically. - _obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app)); - */ ho = dynamic_cast< homeobject::HSHomeObject* >(_obj_inst.get()); EXPECT_TRUE(ho->_pg_map.size() == 1); @@ -44,8 +41,7 @@ TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) { auto lg = std::shared_lock(ho->_pg_lock); for (auto& [_, pg] : ho->_pg_map) { auto hs_pg = static_cast< HSHomeObject::HS_PG* >(pg.get()); - EXPECT_EQ(hs_pg->cache_pg_sb_->blob_sequence_num, 12345); + EXPECT_EQ(hs_pg->durable_entities_.blob_sequence_num, 12345); } } } -#endif \ No newline at end of file diff --git a/src/lib/homestore_backend/tests/homeobj_fixture.hpp b/src/lib/homestore_backend/tests/homeobj_fixture.hpp index 420feb71..cef94ee6 100644 --- a/src/lib/homestore_backend/tests/homeobj_fixture.hpp +++ b/src/lib/homestore_backend/tests/homeobj_fixture.hpp @@ -157,6 +157,23 @@ class HomeObjectFixture : public ::testing::Test { } } + void verify_pg_sb(const HSHomeObject::pg_info_superblk* lhs, const HSHomeObject::pg_info_superblk* rhs) { + EXPECT_EQ(lhs->id, rhs->id); + EXPECT_EQ(lhs->num_members, rhs->num_members); + EXPECT_EQ(lhs->replica_set_uuid, rhs->replica_set_uuid); + EXPECT_EQ(lhs->index_table_uuid, rhs->index_table_uuid); + EXPECT_EQ(lhs->blob_sequence_num, rhs->blob_sequence_num); + EXPECT_EQ(lhs->active_blob_count, rhs->active_blob_count); + EXPECT_EQ(lhs->tombstone_blob_count, rhs->tombstone_blob_count); + EXPECT_EQ(lhs->total_occupied_blk_count, rhs->total_occupied_blk_count); + EXPECT_EQ(lhs->tombstone_blob_count, rhs->tombstone_blob_count); + for (uint32_t i = 0; i < lhs->num_members; i++) { + EXPECT_EQ(lhs->members[i].id, rhs->members[i].id); + EXPECT_EQ(lhs->members[i].priority, rhs->members[i].priority); + EXPECT_EQ(0, std::strcmp(lhs->members[i].name, rhs->members[i].name)); + } + } + void restart() { LOGINFO("Restarting homeobject."); _obj_inst.reset(); diff --git a/src/lib/homestore_backend/tests/hs_blob_tests.cpp b/src/lib/homestore_backend/tests/hs_blob_tests.cpp index 57615b9e..78922791 100644 --- a/src/lib/homestore_backend/tests/hs_blob_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_blob_tests.cpp @@ -181,56 +181,3 @@ TEST_F(HomeObjectFixture, SealShardWithRestart) { ASSERT_EQ(b.error(), BlobError::SEALED_SHARD); LOGINFO("Put blob {}", b.error()); } - -TEST_F(HomeObjectFixture, PGStatsTest) { - // Create a pg, shard, put blob should succeed. - pg_id_t pg_id{1}; - create_pg(pg_id); - - auto s = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get(); - ASSERT_TRUE(!!s); - auto shard_info = s.value(); - auto shard_id = shard_info.id; - s = _obj_inst->shard_manager()->get_shard(shard_id).get(); - ASSERT_TRUE(!!s); - - LOGINFO("Got shard {}", shard_id); - shard_info = s.value(); - EXPECT_EQ(shard_info.id, shard_id); - EXPECT_EQ(shard_info.placement_group, pg_id); - EXPECT_EQ(shard_info.state, ShardInfo::State::OPEN); - auto b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get(); - ASSERT_TRUE(!!b); - LOGINFO("Put blob {}", b.value()); - - // create a shard - s = _obj_inst->shard_manager()->seal_shard(shard_id).get(); - ASSERT_TRUE(!!s); - shard_info = s.value(); - EXPECT_EQ(shard_info.id, shard_id); - EXPECT_EQ(shard_info.placement_group, pg_id); - EXPECT_EQ(shard_info.state, ShardInfo::State::SEALED); - LOGINFO("Sealed shard {}", shard_id); - - // create a 2nd shard - auto s2 = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get(); - auto shard_info2 = s2.value(); - auto shard_id2 = shard_info2.id; - s2 = _obj_inst->shard_manager()->get_shard(shard_id2).get(); - ASSERT_TRUE(!!s); - LOGINFO("Got shard {}", shard_id2); - - PGStats pg_stats; - auto res = _obj_inst->pg_manager()->get_stats(pg_id, pg_stats); - LOGINFO("stats: {}", pg_stats.to_string()); - - EXPECT_EQ(res, true); - EXPECT_EQ(pg_stats.id, pg_id); - EXPECT_EQ(pg_stats.total_shards, 2); - EXPECT_EQ(pg_stats.open_shards, 1); - // TODO: EXPECT_EQ(pg_stats.num_members, 1) after having real 3-replica repl dev in test - EXPECT_EQ(pg_stats.num_members, 1); - - auto stats = _obj_inst->get_stats(); - LOGINFO("HomeObj stats: {}", stats.to_string()); -} diff --git a/src/lib/homestore_backend/tests/hs_pg_tests.cpp b/src/lib/homestore_backend/tests/hs_pg_tests.cpp new file mode 100644 index 00000000..35b18de1 --- /dev/null +++ b/src/lib/homestore_backend/tests/hs_pg_tests.cpp @@ -0,0 +1,89 @@ +#include "homeobj_fixture.hpp" + +TEST_F(HomeObjectFixture, PGStatsTest) { + // Create a pg, shard, put blob should succeed. + pg_id_t pg_id{1}; + create_pg(pg_id); + + auto s = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get(); + ASSERT_TRUE(!!s); + auto shard_info = s.value(); + auto shard_id = shard_info.id; + s = _obj_inst->shard_manager()->get_shard(shard_id).get(); + ASSERT_TRUE(!!s); + + LOGINFO("Got shard {}", shard_id); + shard_info = s.value(); + EXPECT_EQ(shard_info.id, shard_id); + EXPECT_EQ(shard_info.placement_group, pg_id); + EXPECT_EQ(shard_info.state, ShardInfo::State::OPEN); + auto b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get(); + ASSERT_TRUE(!!b); + LOGINFO("Put blob {}", b.value()); + + // create a shard + s = _obj_inst->shard_manager()->seal_shard(shard_id).get(); + ASSERT_TRUE(!!s); + shard_info = s.value(); + EXPECT_EQ(shard_info.id, shard_id); + EXPECT_EQ(shard_info.placement_group, pg_id); + EXPECT_EQ(shard_info.state, ShardInfo::State::SEALED); + LOGINFO("Sealed shard {}", shard_id); + + // create a 2nd shard + auto s2 = _obj_inst->shard_manager()->create_shard(pg_id, 64 * Mi).get(); + auto shard_info2 = s2.value(); + auto shard_id2 = shard_info2.id; + s2 = _obj_inst->shard_manager()->get_shard(shard_id2).get(); + ASSERT_TRUE(!!s); + LOGINFO("Got shard {}", shard_id2); + + PGStats pg_stats; + auto res = _obj_inst->pg_manager()->get_stats(pg_id, pg_stats); + LOGINFO("stats: {}", pg_stats.to_string()); + + EXPECT_EQ(res, true); + EXPECT_EQ(pg_stats.id, pg_id); + EXPECT_EQ(pg_stats.total_shards, 2); + EXPECT_EQ(pg_stats.open_shards, 1); + // TODO: EXPECT_EQ(pg_stats.num_members, 1) after having real 3-replica repl dev in test + EXPECT_EQ(pg_stats.num_members, 1); + + auto stats = _obj_inst->get_stats(); + LOGINFO("HomeObj stats: {}", stats.to_string()); +} + +TEST_F(HomeObjectFixture, PGRecoveryTest) { + // create 10 pg + for (pg_id_t i = 0; i < 10; i++) { + pg_id_t pg_id{i}; + create_pg(pg_id); + } + + // get pg map + HSHomeObject* ho = dynamic_cast< HSHomeObject* >(_obj_inst.get()); + std::map< pg_id_t, std::unique_ptr< PG > > pg_map; + pg_map.swap(ho->_pg_map); + + // get uuid + auto id = ho->our_uuid(); + + // restart + restart(); + + ho = dynamic_cast< HSHomeObject* >(_obj_inst.get()); + // verify uuid + EXPECT_EQ(id, ho->our_uuid()); + + // verify pg map + EXPECT_EQ(10, ho->_pg_map.size()); + + for (auto const& [id, pg] : ho->_pg_map) { + EXPECT_TRUE(pg_map.contains(id)); + auto reserved_pg = dynamic_cast< HSHomeObject::HS_PG* >(pg_map[id].get()); + auto recovered_pg = dynamic_cast< HSHomeObject::HS_PG* >(pg.get()); + EXPECT_TRUE(reserved_pg); + EXPECT_TRUE(recovered_pg); + verify_pg_sb(reserved_pg->pg_sb_.get(), recovered_pg->pg_sb_.get()); + } +} \ No newline at end of file diff --git a/src/lib/homestore_backend/tests/hs_shard_tests.cpp b/src/lib/homestore_backend/tests/hs_shard_tests.cpp index 19efa83f..de604f09 100644 --- a/src/lib/homestore_backend/tests/hs_shard_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_shard_tests.cpp @@ -13,7 +13,6 @@ #include "lib/homestore_backend/hs_homeobject.hpp" #include "lib/homestore_backend/replication_message.hpp" #include "lib/homestore_backend/replication_state_machine.hpp" - #include "lib/tests/fixture_app.hpp" using homeobject::shard_id_t; @@ -72,10 +71,10 @@ TEST_F(TestFixture, CreateMultiShardsOnMultiPG) { } } -#if 0 TEST_F(TestFixture, MockSealShard) { ShardInfo shard_info = _shard_1; shard_info.state = ShardInfo::State::SEALED; + using shard_info_superblk = homeobject::HSHomeObject::shard_info_superblk; nlohmann::json j; j["shard_info"]["shard_id_t"] = shard_info.id; @@ -91,25 +90,27 @@ TEST_F(TestFixture, MockSealShard) { homeobject::HSHomeObject* ho = dynamic_cast< homeobject::HSHomeObject* >(homeobj_.get()); auto* pg = s_cast< homeobject::HSHomeObject::HS_PG* >(ho->_pg_map[_pg_id].get()); auto repl_dev = pg->repl_dev_; - const auto msg_size = sisl::round_up(seal_shard_msg.size(), repl_dev->get_blk_size()); - auto req = homeobject::repl_result_ctx< ShardManager::Result< ShardInfo > >::make(msg_size, 512 /*alignment*/); - auto buf_ptr = req->hdr_buf_.bytes(); - std::memset(buf_ptr, 0, msg_size); - std::memcpy(buf_ptr, seal_shard_msg.c_str(), seal_shard_msg.size()); - - req->header_.msg_type = homeobject::ReplicationMessageType::SEAL_SHARD_MSG; - req->header_.pg_id = _pg_id; - req->header_.shard_id = shard_info.id; - req->header_.payload_size = msg_size; - req->header_.payload_crc = crc32_ieee(homeobject::init_crc32, buf_ptr, msg_size); - req->header_.seal(); - sisl::blob header; - header.set_bytes(r_cast< uint8_t* >(&req->header_)); - header.set_size(sizeof(req->header_)); - sisl::sg_list value; - value.size = msg_size; - value.iovs.push_back(iovec(buf_ptr, msg_size)); - repl_dev->async_alloc_write(header, sisl::blob{buf_ptr, (uint32_t)msg_size}, value, req); + + sisl::io_blob_safe sb_blob(sisl::round_up(sizeof(shard_info_superblk), repl_dev->get_blk_size()), + homeobject::io_align); + shard_info_superblk* sb = new (sb_blob.bytes()) shard_info_superblk(); + sb->type = homeobject::HSHomeObject::DataHeader::data_type_t::SHARD_INFO; + sb->info = shard_info; + sb->chunk_id = 0; + + auto req = homeobject::repl_result_ctx< ShardManager::Result< ShardInfo > >::make(sizeof(shard_info_superblk), 0u); + + req->header()->msg_type = homeobject::ReplicationMessageType::SEAL_SHARD_MSG; + req->header()->pg_id = _pg_id; + req->header()->shard_id = shard_info.id; + req->header()->payload_size = sizeof(shard_info_superblk); + req->header()->payload_crc = crc32_ieee(homeobject::init_crc32, sb_blob.cbytes(), sizeof(shard_info_superblk)); + req->header()->seal(); + + std::memcpy(req->header_extn(), sb_blob.cbytes(), sizeof(shard_info_superblk)); + req->add_data_sg(std::move(sb_blob)); + + repl_dev->async_alloc_write(req->cheader_buf(), sisl::blob{}, req->data_sgs(), req); auto info = req->result().get(); EXPECT_TRUE(info); EXPECT_TRUE(info.value().id == shard_info.id); @@ -123,7 +124,6 @@ TEST_F(TestFixture, MockSealShard) { auto& check_shard = pg_result->shards_.front(); EXPECT_EQ(ShardInfo::State::SEALED, check_shard->info.state); } -#endif class ShardManagerTestingRecovery : public ::testing::Test { public: @@ -179,13 +179,6 @@ TEST_F(ShardManagerTestingRecovery, ShardManagerRecovery) { auto hs_shard = d_cast< homeobject::HSHomeObject::HS_Shard* >(check_shard); auto& recovered_shard_info = hs_shard->info; EXPECT_TRUE(recovered_shard_info == shard_info); - EXPECT_TRUE(recovered_shard_info.placement_group == shard_info.placement_group); - EXPECT_TRUE(recovered_shard_info.state == shard_info.state); - EXPECT_TRUE(recovered_shard_info.created_time == shard_info.created_time); - EXPECT_TRUE(recovered_shard_info.last_modified_time == shard_info.last_modified_time); - EXPECT_TRUE(recovered_shard_info.available_capacity_bytes == shard_info.available_capacity_bytes); - EXPECT_TRUE(recovered_shard_info.total_capacity_bytes == shard_info.total_capacity_bytes); - EXPECT_TRUE(recovered_shard_info.deleted_capacity_bytes == shard_info.deleted_capacity_bytes); // seal the shard when shard is recovery e = _home_object->shard_manager()->seal_shard(shard_id).get(); @@ -262,13 +255,6 @@ TEST_F(ShardManagerTestingRecovery, SealedShardRecovery) { auto hs_shard = d_cast< homeobject::HSHomeObject::HS_Shard* >(pg_iter->second->shards_.front().get()); auto& recovered_shard_info = hs_shard->info; EXPECT_TRUE(recovered_shard_info == shard_info); - EXPECT_TRUE(recovered_shard_info.placement_group == shard_info.placement_group); - EXPECT_TRUE(recovered_shard_info.state == shard_info.state); - EXPECT_TRUE(recovered_shard_info.created_time == shard_info.created_time); - EXPECT_TRUE(recovered_shard_info.last_modified_time == shard_info.last_modified_time); - EXPECT_TRUE(recovered_shard_info.available_capacity_bytes == shard_info.available_capacity_bytes); - EXPECT_TRUE(recovered_shard_info.total_capacity_bytes == shard_info.total_capacity_bytes); - EXPECT_TRUE(recovered_shard_info.deleted_capacity_bytes == shard_info.deleted_capacity_bytes); // finally close the homeobject and homestore. _home_object.reset(); } diff --git a/src/lib/tests/fixture_app.hpp b/src/lib/tests/fixture_app.hpp index 9dfcd2c9..b5915629 100644 --- a/src/lib/tests/fixture_app.hpp +++ b/src/lib/tests/fixture_app.hpp @@ -24,7 +24,7 @@ class FixtureApp : public homeobject::HomeObjectApplication { bool is_hybrid_{false}; public: - FixtureApp(bool is_hybrid=false); + FixtureApp(bool is_hybrid = false); ~FixtureApp() = default; bool spdk_mode() const override { return false; }