From 50c5f0c91845444b1b91939f9ef3ab60660b88e6 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Wed, 4 Sep 2024 00:28:22 +0800 Subject: [PATCH] Add leader_id into error. Signed-off-by: Xiaoxi Chen --- conanfile.py | 2 +- src/include/homeobject/blob_manager.hpp | 35 +++++++++++- src/lib/blob_manager.cpp | 10 ++-- src/lib/homestore_backend/hs_blob_manager.cpp | 54 +++++++++++-------- src/lib/homestore_backend/index_kv.cpp | 8 +-- .../tests/homeobj_fixture.hpp | 2 +- .../homestore_backend/tests/hs_blob_tests.cpp | 6 +-- src/lib/memory_backend/mem_blob_manager.cpp | 2 +- src/lib/tests/BlobManagerTest.cpp | 11 ++-- src/lib/tests/fixture_app.cpp | 2 +- 10 files changed, 89 insertions(+), 43 deletions(-) diff --git a/conanfile.py b/conanfile.py index 854b96a3..c4c07f99 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "2.0.14" + version = "2.1.0" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" diff --git a/src/include/homeobject/blob_manager.hpp b/src/include/homeobject/blob_manager.hpp index 18d1a402..b1aa8589 100644 --- a/src/include/homeobject/blob_manager.hpp +++ b/src/include/homeobject/blob_manager.hpp @@ -9,12 +9,25 @@ namespace homeobject { -ENUM(BlobError, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, UNSUPPORTED_OP, NOT_LEADER, REPLICATION_ERROR, +ENUM(BlobErrorCode, uint16_t, UNKNOWN = 1, TIMEOUT, INVALID_ARG, UNSUPPORTED_OP, NOT_LEADER, REPLICATION_ERROR, UNKNOWN_SHARD, UNKNOWN_BLOB, CHECKSUM_MISMATCH, READ_FAILED, INDEX_ERROR, SEALED_SHARD); +struct BlobError { + BlobErrorCode code; + // set when we are not the current leader of the PG. + std::optional< peer_id_t > current_leader{std::nullopt}; + BlobError(BlobErrorCode _code) { code = _code; } + + BlobError(BlobErrorCode _code, peer_id_t _leader) { + code = _code; + current_leader = std::optional< peer_id_t >{_leader}; + } + BlobErrorCode getCode() const { return code; } +}; struct Blob { Blob() = default; Blob(sisl::io_blob_safe b, std::string const& u, uint64_t o) : body(std::move(b)), user_key(u), object_off(o) {} + Blob(sisl::io_blob_safe b, std::string const& u, uint64_t o, peer_id_t l) : body(std::move(b)), user_key(u), object_off(o), current_leader(l) {} Blob clone() const; @@ -33,3 +46,23 @@ class BlobManager : public Manager< BlobError > { }; } // namespace homeobject + +namespace fmt { +template <> +struct formatter< homeobject::BlobError > { + template < typename ParseContext > + constexpr auto parse(ParseContext& ctx) { + return ctx.begin(); + } + template < typename FormatContext > + auto format(homeobject::BlobError const& err, FormatContext& ctx) { + if (err.current_leader.has_value()) { + return fmt::format_to(ctx.out(), "Code: {}, Leader: {}", err.code, err.current_leader.value()); + } else { + return fmt::format_to(ctx.out(), "Code: {}", err.code); + } + } +}; +template <> +struct formatter< boost::uuids::uuid > : ostream_formatter {}; +} // namespace fmt \ No newline at end of file diff --git a/src/lib/blob_manager.cpp b/src/lib/blob_manager.cpp index e8678a5d..c2829861 100644 --- a/src/lib/blob_manager.cpp +++ b/src/lib/blob_manager.cpp @@ -7,7 +7,7 @@ std::shared_ptr< BlobManager > HomeObjectImpl::blob_manager() { return shared_fr BlobManager::AsyncResult< Blob > HomeObjectImpl::get(shard_id_t shard, blob_id_t const& blob_id, uint64_t off, uint64_t len) const { return _get_shard(shard).thenValue([this, blob_id, off, len](auto const e) -> BlobManager::AsyncResult< Blob > { - if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD); + if (!e) return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_SHARD)); return _get_blob(e.value(), blob_id, off, len); }); } @@ -15,16 +15,16 @@ BlobManager::AsyncResult< Blob > HomeObjectImpl::get(shard_id_t shard, blob_id_t BlobManager::AsyncResult< blob_id_t > HomeObjectImpl::put(shard_id_t shard, Blob&& blob) { return _get_shard(shard).thenValue( [this, blob = std::move(blob)](auto const e) mutable -> BlobManager::AsyncResult< blob_id_t > { - if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD); - if (ShardInfo::State::SEALED == e.value().state) return folly::makeUnexpected(BlobError::SEALED_SHARD); - if (blob.body.size() == 0) return folly::makeUnexpected(BlobError::INVALID_ARG); + if (!e) return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_SHARD)); + if (ShardInfo::State::SEALED == e.value().state) return folly::makeUnexpected(BlobError(BlobErrorCode::SEALED_SHARD)); + if (blob.body.size() == 0) return folly::makeUnexpected(BlobError(BlobErrorCode::INVALID_ARG)); return _put_blob(e.value(), std::move(blob)); }); } BlobManager::NullAsyncResult HomeObjectImpl::del(shard_id_t shard, blob_id_t const& blob) { return _get_shard(shard).thenValue([this, blob](auto const e) mutable -> BlobManager::NullAsyncResult { - if (!e) return folly::makeUnexpected(BlobError::UNKNOWN_SHARD); + if (!e) return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_SHARD)); return _del_blob(e.value(), blob); }); } diff --git a/src/lib/homestore_backend/hs_blob_manager.cpp b/src/lib/homestore_backend/hs_blob_manager.cpp index 6e791504..bfcb1ee9 100644 --- a/src/lib/homestore_backend/hs_blob_manager.cpp +++ b/src/lib/homestore_backend/hs_blob_manager.cpp @@ -37,20 +37,20 @@ BlobError toBlobError(ReplServiceError const& e) { case ReplServiceError::RESULT_NOT_EXIST_YET: [[fallthrough]]; case ReplServiceError::TERM_MISMATCH: - return BlobError::REPLICATION_ERROR; + return BlobError(BlobErrorCode::REPLICATION_ERROR); case ReplServiceError::NOT_LEADER: - return BlobError::NOT_LEADER; + return BlobError(BlobErrorCode::NOT_LEADER); case ReplServiceError::TIMEOUT: - return BlobError::TIMEOUT; + return BlobError(BlobErrorCode::TIMEOUT); case ReplServiceError::NOT_IMPLEMENTED: - return BlobError::UNSUPPORTED_OP; + return BlobError(BlobErrorCode::UNSUPPORTED_OP); case ReplServiceError::OK: DEBUG_ASSERT(false, "Should not process OK!"); [[fallthrough]]; case ReplServiceError::FAILED: - return BlobError::UNKNOWN; + return BlobError(BlobErrorCode::UNKNOWN); default: - return BlobError::UNKNOWN; + return BlobError(BlobErrorCode::UNKNOWN); } } @@ -159,8 +159,14 @@ BlobManager::AsyncResult< blob_id_t > HSHomeObject::_put_blob(ShardInfo const& s req->blob_header()->to_string(), req->data_sgs_string()); repl_dev->async_alloc_write(req->cheader_buf(), req->ckey_buf(), req->data_sgs(), req); - return req->result().deferValue([this, req](const auto& result) -> BlobManager::AsyncResult< blob_id_t > { - if (result.hasError()) { return folly::makeUnexpected(result.error()); } + return req->result().deferValue([this, req, repl_dev](const auto& result) -> BlobManager::AsyncResult< blob_id_t > { + if (result.hasError()) { + auto err = result.error(); + if (err.getCode() == BlobErrorCode::NOT_LEADER) { + err.current_leader = repl_dev->get_leader_id(); + } + return folly::makeUnexpected(err); + } auto blob_info = result.value(); BLOGT(blob_info.shard_id, blob_info.blob_id, "Put blob success blkid=[{}]", blob_info.pbas.to_string()); return blob_info.blob_id; @@ -178,7 +184,7 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis auto msg_header = r_cast< ReplicationMessageHeader const* >(header.cbytes()); if (msg_header->corrupted()) { LOGE("replication message header is corrupted with crc error, lsn:{}", lsn); - if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH)); } + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH))); } return; } @@ -205,7 +211,7 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis if (!exist_already) { if (status != homestore::btree_status_t::success) { LOGE("Failed to insert into index table for blob {} err {}", lsn, enum_name(status)); - if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::INDEX_ERROR)); } + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::INDEX_ERROR))); } return; } else { // The PG superblock (durable entities) will be persisted as part of HS_CLIENT Checkpoint, which is always @@ -261,21 +267,21 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard, BLOGT(shard.id, blob_id, "Blob get request: blkid={}, buf={}", blkid.to_string(), (void*)read_buf.bytes()); return repl_dev->async_read(blkid, sgs, total_size) .thenValue([this, blob_id, shard_id = shard.id, req_len, req_offset, blkid, - read_buf = std::move(read_buf)](auto&& result) mutable -> BlobManager::AsyncResult< Blob > { + read_buf = std::move(read_buf), repl_dev](auto&& result) mutable -> BlobManager::AsyncResult< Blob > { if (result) { BLOGE(shard_id, blob_id, "Failed to get blob: err={}", blob_id, shard_id, result.value()); - return folly::makeUnexpected(BlobError::READ_FAILED); + return folly::makeUnexpected(BlobError(BlobErrorCode::READ_FAILED)); } BlobHeader const* header = r_cast< BlobHeader const* >(read_buf.cbytes()); if (!header->valid()) { BLOGE(shard_id, blob_id, "Invalid header found: [header={}]", header->to_string()); - return folly::makeUnexpected(BlobError::READ_FAILED); + return folly::makeUnexpected(BlobError(BlobErrorCode::READ_FAILED)); } if (header->shard_id != shard_id) { BLOGE(shard_id, blob_id, "Invalid shard_id in header: [header={}]", header->to_string()); - return folly::makeUnexpected(BlobError::READ_FAILED); + return folly::makeUnexpected(BlobError(BlobErrorCode::READ_FAILED)); } // Metadata start offset is just after blob header @@ -291,13 +297,13 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard, if (std::memcmp(computed_hash, header->hash, BlobHeader::blob_max_hash_len) != 0) { BLOGE(shard_id, blob_id, "Hash mismatch header [{}] [computed={:np}]", header->to_string(), spdlog::to_hex(computed_hash, computed_hash + BlobHeader::blob_max_hash_len)); - return folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH); + return folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH)); } if (req_offset + req_len > header->blob_size) { BLOGE(shard_id, blob_id, "Invalid offset length requested in get blob offset={} len={} size={}", req_offset, req_len, header->blob_size); - return folly::makeUnexpected(BlobError::INVALID_ARG); + return folly::makeUnexpected(BlobError(BlobErrorCode::INVALID_ARG)); } // Copy the blob bytes from the offset. If request len is 0, take the @@ -307,7 +313,7 @@ BlobManager::AsyncResult< Blob > HSHomeObject::_get_blob(ShardInfo const& shard, std::memcpy(body.bytes(), blob_bytes + req_offset, res_len); BLOGT(blob_id, shard_id, "Blob get success: blkid={}", blkid.to_string()); - return Blob(std::move(body), std::move(user_key), header->object_offset); + return Blob(std::move(body), std::move(user_key), header->object_offset, repl_dev->get_leader_id()); }); } @@ -321,7 +327,7 @@ HSHomeObject::blob_put_get_blk_alloc_hints(sisl::blob const& header, cintrusive< auto msg_header = r_cast< ReplicationMessageHeader* >(const_cast< uint8_t* >(header.cbytes())); if (msg_header->corrupted()) { LOGE("replication message header is corrupted with crc error shard:{}", msg_header->shard_id); - if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH)); } + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH))); } return folly::makeUnexpected(homestore::ReplServiceError::FAILED); } @@ -367,8 +373,14 @@ BlobManager::NullAsyncResult HSHomeObject::_del_blob(ShardInfo const& shard, blo std::memcpy(req->key_buf().bytes(), &blob_id, sizeof(blob_id_t)); repl_dev->async_alloc_write(req->cheader_buf(), req->ckey_buf(), sisl::sg_list{}, req); - return req->result().deferValue([](const auto& result) -> folly::Expected< folly::Unit, BlobError > { - if (result.hasError()) { return folly::makeUnexpected(result.error()); } + return req->result().deferValue([repl_dev](const auto& result) -> folly::Expected< folly::Unit, BlobError > { + if (result.hasError()) { + auto err = result.error(); + if (err.getCode() == BlobErrorCode::NOT_LEADER) { + err.current_leader = repl_dev->get_leader_id(); + } + return folly::makeUnexpected(err); + } auto blob_info = result.value(); BLOGT(blob_info.shard_id, blob_info.blob_id, "Delete blob successful"); return folly::Unit(); @@ -387,7 +399,7 @@ void HSHomeObject::on_blob_del_commit(int64_t lsn, sisl::blob const& header, sis BLOGE(msg_header->shard_id, *r_cast< blob_id_t const* >(key.cbytes()), "replication message header is corrupted with crc error, lsn={} header=[{}]", lsn, msg_header->to_string()); - if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError::CHECKSUM_MISMATCH)); } + if (ctx) { ctx->promise_.setValue(folly::makeUnexpected(BlobError(BlobErrorCode::CHECKSUM_MISMATCH))); } return; } diff --git a/src/lib/homestore_backend/index_kv.cpp b/src/lib/homestore_backend/index_kv.cpp index a4fd158b..e95bacd0 100644 --- a/src/lib/homestore_backend/index_kv.cpp +++ b/src/lib/homestore_backend/index_kv.cpp @@ -67,12 +67,12 @@ HSHomeObject::get_blob_from_index_table(shared< BlobIndexTable > index_table, sh if (homestore::btree_status_t::success != index_table->get(get_req)) { LOGDEBUG("Failed to get from index table [route={}]", index_key); - return folly::makeUnexpected(BlobError::UNKNOWN_BLOB); + return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_BLOB)); } // blob get API if (const auto& pbas = index_value.pbas(); pbas != tombstone_pbas) return pbas; - return folly::makeUnexpected(BlobError::UNKNOWN_BLOB); + return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_BLOB)); } BlobManager::Result< homestore::MultiBlkId > HSHomeObject::move_to_tombstone(shared< BlobIndexTable > index_table, @@ -83,7 +83,7 @@ BlobManager::Result< homestore::MultiBlkId > HSHomeObject::move_to_tombstone(sha auto status = index_table->get(get_req); if (status != homestore::btree_status_t::success) { LOGE("Failed to get from index table [route={}]", index_key); - return folly::makeUnexpected(BlobError::UNKNOWN_BLOB); + return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_BLOB)); } BlobRouteValue index_value_put{tombstone_pbas}; @@ -91,7 +91,7 @@ BlobManager::Result< homestore::MultiBlkId > HSHomeObject::move_to_tombstone(sha status = index_table->put(put_req); if (status != homestore::btree_status_t::success) { LOGDEBUG("Failed to move blob to tombstone in index table [route={}]", index_key); - return folly::makeUnexpected(BlobError::INDEX_ERROR); + return folly::makeUnexpected(BlobError(BlobErrorCode::INDEX_ERROR)); } return index_value_get.pbas(); diff --git a/src/lib/homestore_backend/tests/homeobj_fixture.hpp b/src/lib/homestore_backend/tests/homeobj_fixture.hpp index 33603f49..a3e2e61f 100644 --- a/src/lib/homestore_backend/tests/homeobj_fixture.hpp +++ b/src/lib/homestore_backend/tests/homeobj_fixture.hpp @@ -95,7 +95,7 @@ class HomeObjectFixture : public ::testing::Test { retry: auto b = _obj_inst->blob_manager()->put(shard_id, std::move(put_blob)).get(); - if (!b && b.error() == BlobError::NOT_LEADER) { + if (!b && b.error().getCode() == BlobErrorCode::NOT_LEADER) { LOGINFO("Failed to put blob due to not leader, sleep 1s and retry put", pg_id, shard_id); put_blob = homeobject::Blob{sisl::io_blob_safe(blob_size, alignment), user_key, 42ul}; std::memcpy(put_blob.body.bytes(), clone.body.bytes(), clone.body.size()); diff --git a/src/lib/homestore_backend/tests/hs_blob_tests.cpp b/src/lib/homestore_backend/tests/hs_blob_tests.cpp index 78922791..3d580e57 100644 --- a/src/lib/homestore_backend/tests/hs_blob_tests.cpp +++ b/src/lib/homestore_backend/tests/hs_blob_tests.cpp @@ -108,7 +108,7 @@ TEST_F(HomeObjectFixture, BasicPutGetDelBlobWRestart) { auto g = hs_homeobject->get_blob_from_index_table(index_table, shard_id, blob_id); ASSERT_FALSE(!!g); - EXPECT_EQ(BlobError::UNKNOWN_BLOB, g.error()); + EXPECT_EQ(BlobErrorCode::UNKNOWN_BLOB, g.error().getCode()); } LOGINFO("Flushing CP."); @@ -160,7 +160,7 @@ TEST_F(HomeObjectFixture, SealShardWithRestart) { b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get(); ASSERT_TRUE(!b); - ASSERT_EQ(b.error(), BlobError::SEALED_SHARD); + ASSERT_EQ(b.error().getCode(), BlobErrorCode::SEALED_SHARD); LOGINFO("Put blob {}", b.error()); // Restart homeobject @@ -178,6 +178,6 @@ TEST_F(HomeObjectFixture, SealShardWithRestart) { b = _obj_inst->blob_manager()->put(shard_id, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}).get(); ASSERT_TRUE(!b); - ASSERT_EQ(b.error(), BlobError::SEALED_SHARD); + ASSERT_EQ(b.error().getCode(), BlobErrorCode::SEALED_SHARD); LOGINFO("Put blob {}", b.error()); } diff --git a/src/lib/memory_backend/mem_blob_manager.cpp b/src/lib/memory_backend/mem_blob_manager.cpp index 0988483d..5d940604 100644 --- a/src/lib/memory_backend/mem_blob_manager.cpp +++ b/src/lib/memory_backend/mem_blob_manager.cpp @@ -41,7 +41,7 @@ BlobManager::AsyncResult< Blob > MemoryHomeObject::_get_blob(ShardInfo const& _s WITH_SHARD WITH_ROUTE(_blob) IF_BLOB_ALIVE { return blob_it->second.blob_->clone(); } - return folly::makeUnexpected(BlobError::UNKNOWN_BLOB); + return folly::makeUnexpected(BlobError(BlobErrorCode::UNKNOWN_BLOB)); } // Tombstone BlobExt entry diff --git a/src/lib/tests/BlobManagerTest.cpp b/src/lib/tests/BlobManagerTest.cpp index 7d81fdb1..7d89d129 100644 --- a/src/lib/tests/BlobManagerTest.cpp +++ b/src/lib/tests/BlobManagerTest.cpp @@ -6,6 +6,7 @@ using homeobject::Blob; using homeobject::BlobError; +using homeobject::BlobErrorCode; TEST_F(TestFixture, BasicBlobTests) { auto const batch_sz = 4; @@ -33,7 +34,7 @@ TEST_F(TestFixture, BasicBlobTests) { our_calls.push_back( homeobj_->blob_manager() ->put(i, Blob{sisl::io_blob_safe(512u, 512u), "test_blob", 0ul}) - .deferValue([](auto const& e) { EXPECT_EQ(BlobError::UNKNOWN_SHARD, e.error()); })); + .deferValue([](auto const& e) { EXPECT_EQ(BlobErrorCode::UNKNOWN_SHARD, e.error().code); })); LOGINFO("Calling to put blob, shard {}", _shard_1.id); our_calls.push_back(homeobj_->blob_manager() ->put(_shard_1.id, Blob{sisl::io_blob_safe(4 * Ki, 512u), "test_blob", 4 * Mi}) @@ -50,7 +51,7 @@ TEST_F(TestFixture, BasicBlobTests) { .deferValue([](auto const& e) { EXPECT_TRUE(!!e); })); our_calls.push_back(homeobj_->blob_manager()->del(i, _blob_id).deferValue([](auto const& e) { EXPECT_FALSE(!!e); - EXPECT_EQ(BlobError::UNKNOWN_SHARD, e.error()); + EXPECT_EQ(BlobErrorCode::UNKNOWN_SHARD, e.error().getCode()); })); LOGINFO("Calling to Deleting blob, shard {}, blobID {}", _shard_1.id, (i - _shard_2.id)); our_calls.push_back( @@ -62,7 +63,7 @@ TEST_F(TestFixture, BasicBlobTests) { return; } else { EXPECT_FALSE(!!e); - EXPECT_EQ(BlobError::UNKNOWN_BLOB, e.error()); + EXPECT_EQ(BlobErrorCode::UNKNOWN_BLOB, e.error().getCode()); } })); } @@ -78,7 +79,7 @@ TEST_F(TestFixture, BasicBlobTests) { auto p_e = homeobj_->blob_manager()->put(_shard_1.id, Blob{sisl::io_blob_safe(4 * Ki, 512u), "test_blob", 4 * Mi}).get(); ASSERT_FALSE(!!p_e); - EXPECT_EQ(BlobError::SEALED_SHARD, p_e.error()); + EXPECT_EQ(BlobErrorCode::SEALED_SHARD, p_e.error().getCode()); // BLOB exists EXPECT_TRUE(homeobj_->blob_manager()->get(_shard_1.id, _blob_id).get()); @@ -89,7 +90,7 @@ TEST_F(TestFixture, BasicBlobTests) { // BLOB is now unknown auto g_e = homeobj_->blob_manager()->get(_shard_1.id, _blob_id).get(); ASSERT_FALSE(!!g_e); - EXPECT_EQ(BlobError::UNKNOWN_BLOB, g_e.error()); + EXPECT_EQ(BlobErrorCode::UNKNOWN_BLOB, g_e.error().getCode()); // Delete is Idempotent EXPECT_TRUE(homeobj_->blob_manager()->del(_shard_1.id, _blob_id).get()); diff --git a/src/lib/tests/fixture_app.cpp b/src/lib/tests/fixture_app.cpp index 8126e943..931fd4bb 100644 --- a/src/lib/tests/fixture_app.cpp +++ b/src/lib/tests/fixture_app.cpp @@ -60,7 +60,7 @@ void TestFixture::SetUp() { LOGDEBUG("Get on empty Shard: {}", _shard_1.id); auto g_e = homeobj_->blob_manager()->get(_shard_1.id, 0).get(); ASSERT_FALSE(g_e); - EXPECT_EQ(homeobject::BlobError::UNKNOWN_BLOB, g_e.error()); + EXPECT_EQ(homeobject::BlobErrorCode::UNKNOWN_BLOB, g_e.error().getCode()); LOGDEBUG("Insert Blob to: {}", _shard_1.id); auto o_e = homeobj_->blob_manager()