raft test framework for homeobject

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomeObjectConan(ConanFile):
     name = "homeobject"
-    version = "2.1.5"
+    version = "2.1.6"
 
     homepage = "https://github.com/eBay/HomeObject"
     description = "Blob Store built on HomeReplication"

src/lib/homestore_backend/CMakeLists.txt

@@ -50,7 +50,6 @@ add_subdirectory(tests)
 add_executable (homestore_test)
 target_sources(homestore_test PRIVATE
     $<TARGET_OBJECTS:homestore_tests>
-    $<TARGET_OBJECTS:test_fixture>
 )
 target_link_libraries(homestore_test PUBLIC
     homeobject_homestore

src/lib/homestore_backend/hs_blob_manager.cpp

@@ -226,8 +226,11 @@ void HSHomeObject::on_blob_put_commit(int64_t lsn, sisl::blob const& header, sis
             // number is already updated in the put_blob call.
             hs_pg->durable_entities_update([&blob_id, &pbas](auto& de) {
                 auto existing_blob_id = de.blob_sequence_num.load();
-                while ((blob_id > existing_blob_id) &&
-                       !de.blob_sequence_num.compare_exchange_weak(existing_blob_id, blob_id)) {}
+                auto next_blob_id = blob_id + 1;
+                while ((next_blob_id > existing_blob_id) &&
+                       // we need update the blob_sequence_num to existing_blob_id+1 so that if leader changes, we can
+                       // still get the up-to-date blob_sequence_num
+                       !de.blob_sequence_num.compare_exchange_weak(existing_blob_id, next_blob_id)) {}
                 de.active_blob_count.fetch_add(1, std::memory_order_relaxed);
                 de.total_occupied_blk_count.fetch_add(pbas.blk_count(), std::memory_order_relaxed);
             });

src/lib/homestore_backend/hs_homeobject.cpp

@@ -177,7 +177,8 @@ void HSHomeObject::init_homestore() {
             auto run_on_type = has_fast_dev ? homestore::HSDevType::Fast : homestore::HSDevType::Data;
             LOGD("Running with Single mode, all service on {}", run_on_type);
             HomeStore::instance()->format_and_start({
-	        // FIXME:  this is to work around the issue in HS that varsize allocator doesnt work with small chunk size.
+                // FIXME:  this is to work around the issue in HS that varsize allocator doesnt work with small chunk
+                // size.
                 {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0, .num_chunks = 1}},
                 {HS_SERVICE::LOG, hs_format_params{.dev_type = run_on_type, .size_pct = 10.0, .chunk_size = 32 * Mi}},
                 {HS_SERVICE::INDEX, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0, .num_chunks = 1}},
@@ -280,6 +281,7 @@ void HSHomeObject::register_homestore_metablk_callback() {
 }
 
 HSHomeObject::~HSHomeObject() {
+    LOGI("HomeObject start destructing");
 #if 0
     if (ho_timer_thread_handle_.first) {
         iomanager.cancel_timer(ho_timer_thread_handle_, true);

src/lib/homestore_backend/hs_http_manager.cpp

@@ -45,8 +45,8 @@ HttpManager::HttpManager(HSHomeObject& ho) : ho_(ho) {
         return;
     }
     try {
-         http_server->setup_routes(routes);
-    } catch (std::runtime_error const& e) { LOGERROR("setup routes failed, {}", e.what()) }
+        http_server->setup_routes(routes);
+    } catch (std::runtime_error const& e) { LOGERROR("setup routes failed, {}", e.what()); }
 }
 
 void HttpManager::get_obj_life(const Pistache::Rest::Request& request, Pistache::Http::ResponseWriter response) {

src/lib/homestore_backend/tests/CMakeLists.txt

@@ -7,6 +7,7 @@ list(APPEND TEST_SOURCES
     hs_blob_tests.cpp
     hs_pg_tests.cpp
     homeobj_cp_tests.cpp
+    test_homestore_backend.cpp
     )
 
 add_library(homestore_tests OBJECT)

src/lib/homestore_backend/tests/bits_generator.hpp

@@ -20,7 +20,7 @@
 #include <type_traits>
 #include <cstdint>
 #include <random>
-#include <iostream>
+#include <limits>
 
 namespace homeobject {
 
@@ -29,14 +29,27 @@ class BitsGenerator {
     static void gen_random_bits(size_t size, uint8_t* buf) {
         std::random_device rd;
         std::default_random_engine g(rd());
-        std::uniform_int_distribution< unsigned long long > dis(std::numeric_limits< std::uint8_t >::min(),
-                                                                std::numeric_limits< std::uint8_t >::max());
+        std::uniform_int_distribution< std::uint8_t > dis(std::numeric_limits< std::uint8_t >::min(),
+                                                          std::numeric_limits< std::uint8_t >::max());
         for (size_t i = 0; i < size; ++i) {
             buf[i] = dis(g);
         }
     }
 
     static void gen_random_bits(sisl::blob& b) { gen_random_bits(b.size(), b.bytes()); }
+
+    // this function guarantees that the generated bits will be identical for the identical input of blob_id
+    // and size.
+    static void gen_blob_bits(size_t size, uint8_t* buf, blob_id_t blob_id) {
+        std::mt19937_64 rng(blob_id);
+        std::uniform_int_distribution< std::uint8_t > dist(std::numeric_limits< std::uint8_t >::min(),
+                                                           std::numeric_limits< std::uint8_t >::max());
+
+        for (size_t i = 0; i < size;)
+            buf[i++] = dist(rng);
+    }
+
+    static void gen_blob_bits(sisl::blob& b, blob_id_t blob_id) { gen_blob_bits(b.size(), b.bytes(), blob_id); }
 };
 
 }; // namespace homeobject

src/lib/homestore_backend/tests/homeobj_cp_tests.cpp

@@ -4,17 +4,13 @@ TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) {
     // Step-1: create a PG and a shard
     std::vector< std::pair< pg_id_t, shard_id_t > > pg_shard_id_vec;
     create_pg(1 /* pg_id */);
-    auto shard = _obj_inst->shard_manager()->create_shard(1 /* pg_id */, 64 * Mi).get();
-    ASSERT_TRUE(!!shard);
-    pg_shard_id_vec.emplace_back(1 /* pg_id */, shard->id);
-    LOGINFO("pg {} shard {}", 1, shard->id);
-
-    using namespace homestore;
-    auto ho = dynamic_cast< HSHomeObject* >(_obj_inst.get());
+    auto shard_info = create_shard(1 /* pg_id */, 64 * Mi);
+    pg_shard_id_vec.emplace_back(1 /* pg_id */, shard_info.id);
+    LOGINFO("pg {} shard {}", 1, shard_info.id);
     {
         // Step-2: write some dirty pg information and add to dirt list;
-        auto lg = std::unique_lock(ho->_pg_lock);
-        for (auto& [_, pg] : ho->_pg_map) {
+        auto lg = std::unique_lock(_obj_inst->_pg_lock);
+        for (auto& [_, pg] : _obj_inst->_pg_map) {
             auto hs_pg = static_cast< HSHomeObject::HS_PG* >(pg.get());
             hs_pg->durable_entities_.blob_sequence_num = 54321; // fake some random blob seq number to make it dirty;
             hs_pg->is_dirty_.store(true);
@@ -26,20 +22,12 @@ TEST_F(HomeObjectFixture, HSHomeObjectCPTestBasic) {
         }
     }
 
-    // Step-3: trigger a cp;
-    trigger_cp(true /* wait */);
-
-    _obj_inst.reset();
-
-    // Step-4: re-create the homeobject and pg infos and shard infos will be recover automatically.
-    _obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app));
-
-    ho = dynamic_cast< homeobject::HSHomeObject* >(_obj_inst.get());
+    restart();
 
-    EXPECT_TRUE(ho->_pg_map.size() == 1);
+    EXPECT_TRUE(_obj_inst->_pg_map.size() == 1);
     {
-        auto lg = std::shared_lock(ho->_pg_lock);
-        for (auto& [_, pg] : ho->_pg_map) {
+        auto lg = std::shared_lock(_obj_inst->_pg_lock);
+        for (auto& [_, pg] : _obj_inst->_pg_map) {
             auto hs_pg = static_cast< HSHomeObject::HS_PG* >(pg.get());
             EXPECT_EQ(hs_pg->durable_entities_.blob_sequence_num, 12345);
         }