From cbe008fcf065484eb54adcd5cf33ae9fcbca09ac Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Thu, 18 Apr 2024 17:43:27 -0700 Subject: [PATCH 01/10] flatbuffer for baseline resync --- conanfile.py | 4 +-- src/lib/homestore_backend/CMakeLists.txt | 9 +++++++ .../replication_state_machine.hpp | 6 +++++ .../homestore_backend/resync_blob_data.fbs | 14 +++++++++++ src/lib/homestore_backend/resync_pg_shard.fbs | 25 +++++++++++++++++++ 5 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/lib/homestore_backend/resync_blob_data.fbs create mode 100644 src/lib/homestore_backend/resync_pg_shard.fbs diff --git a/conanfile.py b/conanfile.py index 0a6e58b2..16deef24 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "1.0.16" + version = "1.2.1" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" topics = ("ebay") @@ -40,7 +40,7 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[>=6.2, include_prerelease=True]@oss/master") + self.requires("homestore/[>=6.3, include_prerelease=True]@oss/master") self.requires("sisl/[>=12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) diff --git a/src/lib/homestore_backend/CMakeLists.txt b/src/lib/homestore_backend/CMakeLists.txt index c8a546a8..5e7cade7 100644 --- a/src/lib/homestore_backend/CMakeLists.txt +++ b/src/lib/homestore_backend/CMakeLists.txt @@ -4,6 +4,13 @@ if(NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) endif() +#list(APPEND SCHEMA_FLAGS "--scoped-enums" "--gen-name-strings" "--cpp-std=c++17" "--cpp-static-reflection" "--reflect-names") +#flatbuffers_generate_headers( +# TARGET ho_replication_fb +# SCHEMAS resync_pg_shard.fbs resync_blob_data.fbs +# FLAGS ${SCHEMA_FLAGS} +#) + add_library ("${PROJECT_NAME}_homestore") target_sources("${PROJECT_NAME}_homestore" PRIVATE hs_homeobject.cpp @@ -28,6 +35,8 @@ settings_gen_cpp( ${CMAKE_CURRENT_BINARY_DIR}/generated/ "${PROJECT_NAME}_homestore" hs_backend_config.fbs + resync_pg_shard.fbs + resync_blob_data.fbs ) if(BUILD_TESTING) diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index 0c966f0c..5ed24d17 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -166,6 +166,12 @@ class ReplicationStateMachine : public homestore::ReplDevListener { /// @brief Called when the replica set is being stopped void on_replica_stop() override; + /// @brief Called when the snapshot is being created. + /// @param snapshot - The snapshot object which contains the lsn and term; + void create_snapshot(homestore::repl_snapshot&) override { + // To be implemented; + } + private: HSHomeObject* home_object_{nullptr}; }; diff --git a/src/lib/homestore_backend/resync_blob_data.fbs b/src/lib/homestore_backend/resync_blob_data.fbs new file mode 100644 index 00000000..1f775e25 --- /dev/null +++ b/src/lib/homestore_backend/resync_blob_data.fbs @@ -0,0 +1,14 @@ +namespace homeobject; + +table BlobData { + shard_id : uint64; + key : [ubyte]; + size : uint32; + data : [ubyte]; +} + +table ResyncBlobDataBatch { + data_array : [BlobData]; +} + +root_type ResyncBlobDataBatch; diff --git a/src/lib/homestore_backend/resync_pg_shard.fbs b/src/lib/homestore_backend/resync_pg_shard.fbs new file mode 100644 index 00000000..3bf38031 --- /dev/null +++ b/src/lib/homestore_backend/resync_pg_shard.fbs @@ -0,0 +1,25 @@ +namespace homeobject; + +table Member { + uuid : [ubyte]; + name : [ubyte]; +} + +table PGInfoEntry { + pg_id : uint32; // only low 16 bit is used for pg_id; + replica_set_uuid : [ubyte]; // uuid of replica set + members : [Member]; // peers; +} + +table ShardInfoEntry { + pg_id : uint32; // pg id which this shard belongs to; + shard_id : uint64; // shard id to be created with; + shard_size : uint64; // shard size; +} + +table ResyncPGShardInfo { + pg : PGInfoEntry; + shards : [ShardInfoEntry]; +} + +root_type ResyncPGShardInfo; From 945febf153c72bf9ffb0181ea6b05dece48d4bbc Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 11:50:24 -0700 Subject: [PATCH 02/10] address comment --- src/lib/homestore_backend/resync_blob_data.fbs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/homestore_backend/resync_blob_data.fbs b/src/lib/homestore_backend/resync_blob_data.fbs index 1f775e25..57a39605 100644 --- a/src/lib/homestore_backend/resync_blob_data.fbs +++ b/src/lib/homestore_backend/resync_blob_data.fbs @@ -2,8 +2,10 @@ namespace homeobject; table BlobData { shard_id : uint64; - key : [ubyte]; - size : uint32; + user_key_size : uint32; + user_key : [ubyte]; + blob_id : uint64; + data_size : uint32; data : [ubyte]; } From 8836d7338aaad36c1ccc1df398ae75c5ee4a11c2 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 12:23:49 -0700 Subject: [PATCH 03/10] fix return type --- src/lib/homestore_backend/replication_state_machine.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index 5ed24d17..591b8533 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -168,8 +168,9 @@ class ReplicationStateMachine : public homestore::ReplDevListener { /// @brief Called when the snapshot is being created. /// @param snapshot - The snapshot object which contains the lsn and term; - void create_snapshot(homestore::repl_snapshot&) override { + AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { // To be implemented; + return make_async_success<>(); } private: From f50af4f2c47b278f482b21cc9d3eecaaf3399c85 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 13:31:34 -0700 Subject: [PATCH 04/10] fix build --- src/lib/homestore_backend/replication_state_machine.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index 591b8533..a10670d6 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -168,7 +168,7 @@ class ReplicationStateMachine : public homestore::ReplDevListener { /// @brief Called when the snapshot is being created. /// @param snapshot - The snapshot object which contains the lsn and term; - AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { + homestore::AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { // To be implemented; return make_async_success<>(); } From b4a02da294edb0ab3d642232d9c0258fc74ba29e Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 13:51:23 -0700 Subject: [PATCH 05/10] remove unused code --- src/lib/homestore_backend/CMakeLists.txt | 7 ------- src/lib/homestore_backend/replication_state_machine.hpp | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/lib/homestore_backend/CMakeLists.txt b/src/lib/homestore_backend/CMakeLists.txt index 5e7cade7..836ce393 100644 --- a/src/lib/homestore_backend/CMakeLists.txt +++ b/src/lib/homestore_backend/CMakeLists.txt @@ -4,13 +4,6 @@ if(NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) endif() -#list(APPEND SCHEMA_FLAGS "--scoped-enums" "--gen-name-strings" "--cpp-std=c++17" "--cpp-static-reflection" "--reflect-names") -#flatbuffers_generate_headers( -# TARGET ho_replication_fb -# SCHEMAS resync_pg_shard.fbs resync_blob_data.fbs -# FLAGS ${SCHEMA_FLAGS} -#) - add_library ("${PROJECT_NAME}_homestore") target_sources("${PROJECT_NAME}_homestore" PRIVATE hs_homeobject.cpp diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index a10670d6..6ce4d810 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -170,7 +170,7 @@ class ReplicationStateMachine : public homestore::ReplDevListener { /// @param snapshot - The snapshot object which contains the lsn and term; homestore::AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { // To be implemented; - return make_async_success<>(); + return homestore::make_async_success<>(); } private: From 1f44046832e718800a515ec18e0d59a6f354e1a2 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 14:13:25 -0700 Subject: [PATCH 06/10] return value fix --- conanfile.py | 3 ++- src/lib/homestore_backend/replication_state_machine.hpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index 9a2d5dc6..aa587aaa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -41,7 +41,8 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[~6.3, include_prerelease=True]@oss/master") + #self.requires("homestore/[~6.3, include_prerelease=True]@oss/master") + self.requires("homestore/6.3.1@yakuang/master") self.requires("sisl/[~12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index 6ce4d810..202d9a35 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -170,7 +170,7 @@ class ReplicationStateMachine : public homestore::ReplDevListener { /// @param snapshot - The snapshot object which contains the lsn and term; homestore::AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { // To be implemented; - return homestore::make_async_success<>(); + return folly::makeSemiFuture< homestore::ReplResult< folly::Unit > >(folly::Unit{}); } private: From 5eda8dbbc7e6976f371de7d3ca39d33c6591522d Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Fri, 19 Apr 2024 14:15:32 -0700 Subject: [PATCH 07/10] fix homestore ver --- conanfile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index aa587aaa..9a2d5dc6 100644 --- a/conanfile.py +++ b/conanfile.py @@ -41,8 +41,7 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - #self.requires("homestore/[~6.3, include_prerelease=True]@oss/master") - self.requires("homestore/6.3.1@yakuang/master") + self.requires("homestore/[~6.3, include_prerelease=True]@oss/master") self.requires("sisl/[~12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) From 2746743322f2608acbe0bdb310a6cce7964a56f2 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Wed, 24 Apr 2024 12:48:35 -0700 Subject: [PATCH 08/10] address comments --- src/lib/homestore_backend/resync_blob_data.fbs | 4 ++-- src/lib/homestore_backend/resync_pg_shard.fbs | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lib/homestore_backend/resync_blob_data.fbs b/src/lib/homestore_backend/resync_blob_data.fbs index 57a39605..ab1759f5 100644 --- a/src/lib/homestore_backend/resync_blob_data.fbs +++ b/src/lib/homestore_backend/resync_blob_data.fbs @@ -2,10 +2,10 @@ namespace homeobject; table BlobData { shard_id : uint64; - user_key_size : uint32; - user_key : [ubyte]; blob_id : uint64; + user_key_size : uint32; data_size : uint32; + user_key : [ubyte]; data : [ubyte]; } diff --git a/src/lib/homestore_backend/resync_pg_shard.fbs b/src/lib/homestore_backend/resync_pg_shard.fbs index 3bf38031..7623832f 100644 --- a/src/lib/homestore_backend/resync_pg_shard.fbs +++ b/src/lib/homestore_backend/resync_pg_shard.fbs @@ -7,14 +7,18 @@ table Member { table PGInfoEntry { pg_id : uint32; // only low 16 bit is used for pg_id; + priority : int32; // pg priority; replica_set_uuid : [ubyte]; // uuid of replica set members : [Member]; // peers; } table ShardInfoEntry { + state : ubyte; // shard state; pg_id : uint32; // pg id which this shard belongs to; shard_id : uint64; // shard id to be created with; shard_size : uint64; // shard size; + created_time : uint64; // shard creation time + last_modified_time : uint64; // shard last modify time } table ResyncPGShardInfo { From b9ea29e4db7b102dfe17543c6cdbc944b6bad291 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Tue, 20 Aug 2024 14:38:48 -0700 Subject: [PATCH 09/10] address comment --- src/lib/homestore_backend/replication_state_machine.hpp | 7 ------- src/lib/homestore_backend/resync_blob_data.fbs | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/lib/homestore_backend/replication_state_machine.hpp b/src/lib/homestore_backend/replication_state_machine.hpp index f1f21042..dad9ee7f 100644 --- a/src/lib/homestore_backend/replication_state_machine.hpp +++ b/src/lib/homestore_backend/replication_state_machine.hpp @@ -182,13 +182,6 @@ class ReplicationStateMachine : public homestore::ReplDevListener { std::shared_ptr< homestore::snapshot_data > snp_data) override; virtual void free_user_snp_ctx(void*& user_snp_ctx) override; - /// @brief Called when the snapshot is being created. - /// @param snapshot - The snapshot object which contains the lsn and term; - homestore::AsyncReplResult<> create_snapshot(homestore::repl_snapshot&) override { - // To be implemented; - return folly::makeSemiFuture< homestore::ReplResult< folly::Unit > >(folly::Unit{}); - } - private: HSHomeObject* home_object_{nullptr}; }; diff --git a/src/lib/homestore_backend/resync_blob_data.fbs b/src/lib/homestore_backend/resync_blob_data.fbs index ab1759f5..ff1010dc 100644 --- a/src/lib/homestore_backend/resync_blob_data.fbs +++ b/src/lib/homestore_backend/resync_blob_data.fbs @@ -11,6 +11,7 @@ table BlobData { table ResyncBlobDataBatch { data_array : [BlobData]; + end_of_batch: ubyte; } root_type ResyncBlobDataBatch; From 255e3a5857a16177d4b99ac2aa0f4376ed2e8a85 Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Thu, 22 Aug 2024 14:19:48 -0700 Subject: [PATCH 10/10] comments --- src/lib/homestore_backend/resync_blob_data.fbs | 2 ++ src/lib/homestore_backend/resync_pg_shard.fbs | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/lib/homestore_backend/resync_blob_data.fbs b/src/lib/homestore_backend/resync_blob_data.fbs index ff1010dc..49695629 100644 --- a/src/lib/homestore_backend/resync_blob_data.fbs +++ b/src/lib/homestore_backend/resync_blob_data.fbs @@ -1,3 +1,5 @@ +native_include "sisl/utility/non_null_ptr.hpp"; + namespace homeobject; table BlobData { diff --git a/src/lib/homestore_backend/resync_pg_shard.fbs b/src/lib/homestore_backend/resync_pg_shard.fbs index 7623832f..00a76dec 100644 --- a/src/lib/homestore_backend/resync_pg_shard.fbs +++ b/src/lib/homestore_backend/resync_pg_shard.fbs @@ -1,3 +1,5 @@ +native_include "sisl/utility/non_null_ptr.hpp"; + namespace homeobject; table Member { @@ -19,6 +21,7 @@ table ShardInfoEntry { shard_size : uint64; // shard size; created_time : uint64; // shard creation time last_modified_time : uint64; // shard last modify time + created_lsn : int64; // lsn on shard creation; } table ResyncPGShardInfo {