Merge branch 'master' into yk_meta

.github/workflows/build_commit.yml

@@ -20,10 +20,22 @@ on:
         type: string
 
 jobs:
-  Sisl10Deps:
-    uses: eBay/sisl/.github/workflows/build_dependencies.yml@stable/v10.x
+  SislDeps:
+    uses: eBay/sisl/.github/workflows/build_dependencies.yml@master
     with:
-      branch: stable/v10.x
+      branch: master
+      platform: ${{ inputs.platform }}
+      build-type: ${{ inputs.build-type }}
+      malloc-impl: ${{ inputs.malloc-impl }}
+      prerelease: ${{ inputs.prerelease }}
+      tooling: None
+    if: ${{ github.event_name != 'pull_request' }}
+
+  NuraftMesgDeps:
+    needs: SislDeps
+    uses: eBay/nuraft_mesg/.github/workflows/build_dependencies.yml@main
+    with:
+      branch: main
       platform: ${{ inputs.platform }}
       build-type: ${{ inputs.build-type }}
       malloc-impl: ${{ inputs.malloc-impl }}
@@ -32,7 +44,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' }}
 
   IOMgrDeps:
-    needs: Sisl10Deps
+    needs: SislDeps
     uses: eBay/iomanager/.github/workflows/build_dependencies.yml@master
     with:
       branch: master
@@ -44,7 +56,7 @@ jobs:
     if: ${{ github.event_name != 'pull_request' }}
 
   HomestoreDeps:
-    needs: IOMgrDeps
+    needs: [IOMgrDeps, NuraftMesgDeps]
     uses: ./.github/workflows/build_dependencies.yml
     with:
       branch: ${{ github.ref }}

.github/workflows/build_dependencies.yml

@@ -108,7 +108,7 @@ jobs:
       uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
       with:
         load_any: 'True'
-        key_prefix: Sisl10Deps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        key_prefix: SislDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
 
     - name: Retrieve Dependencies
@@ -119,6 +119,14 @@ jobs:
         ref: master
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Retrieve Dependencies
+      uses: actions/checkout@v3
+      with:
+        repository: eBay/nuraft_mesg
+        path: import/nuraft_mesg
+        ref: main
+      if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Load IOMgr Cache
       uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
       with:
@@ -128,6 +136,15 @@ jobs:
         fail_on_cache_miss: true
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Load NuraftMesg Cache
+      uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
+      with:
+        testing: 'False'
+        path: import/nuraft_mesg
+        key_prefix: NuMesgDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        fail_on_cache_miss: true
+      if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Setup Conan
       uses: eBay/sisl/.github/actions/setup_conan@stable/v8.x
       with:
@@ -139,6 +156,7 @@ jobs:
         sudo apt-get install -y python3-pyelftools libaio-dev
         python -m pip install pyelftools
         conan export import/iomgr oss/master
+        conan export import/nuraft_mesg oss/main
         cached_pkgs=$(ls -1d ~/.conan/data/*/*/*/*/package | sed 's,.*data/,,' | cut -d'/' -f1,2 | paste -sd',' - -)
         echo "::info:: Pre-cached: ${cached_pkgs}"
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
@@ -150,6 +168,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o testing=off \
             -s build_type=${{ inputs.build-type }} \
             --build missing \
@@ -166,7 +185,7 @@ jobs:
       uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
       with:
         load_any: 'True'
-        key_prefix: Sisl10Deps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        key_prefix: SislDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
       if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }}
 
     - name: Reload IOMgr Cache
@@ -178,6 +197,15 @@ jobs:
         fail_on_cache_miss: true
       if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Reload NuraftMesg Cache
+      uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
+      with:
+        testing: 'False'
+        path: import/nuraft_mesg
+        key_prefix: NuMesgDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        fail_on_cache_miss: true
+      if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Create and Test Package
       run: |
         sanitize=$([[ "${{ inputs.tooling }}" == "Sanitize" ]] && echo "True" || echo "False")
@@ -186,6 +214,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o homestore:sanitize=${sanitize} \
             -s build_type=${{ inputs.build-type }} \
             --build missing \
@@ -199,6 +228,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o coverage=True \
             -s build_type=${{ inputs.build-type }} \
             --build missing \

.gitignore

@@ -105,3 +105,4 @@ cmake-*/**
 # Visual Studio
 CMakeSettings.json
 .vs/**
+.cache

conanfile.py

@@ -5,8 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "4.9.4"
-
+    version = "5.0.2"
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
     topics = ("ebay", "nublox")
@@ -55,11 +54,13 @@ def build_requirements(self):
         self.build_requires("gtest/1.14.0")
 
     def requirements(self):
-        self.requires("iomgr/[~=10, include_prerelease=True]@oss/master")
-        self.requires("sisl/[>=10.3]")
+        self.requires("iomgr/[~=11, include_prerelease=True]@oss/master")
+        self.requires("sisl/[~=11, include_prerelease=True]@oss/master")
+        self.requires("nuraft_mesg/[~=2, include_prerelease=True]@oss/main")
 
         self.requires("farmhash/cci.20190513@")
-        self.requires("isa-l/2.30.0")
+        if self.settings.arch in ['x86', 'x86_64']:
+            self.requires("isa-l/2.30.0")
         self.requires("spdk/21.07.y")
 
     def build(self):

src/CMakeLists.txt

@@ -3,25 +3,34 @@ cmake_minimum_required(VERSION 3.13)
 set(CMAKE_THREAD_PREFER_PTHREAD TRUE)
 
 find_package(Threads)
-find_library(LIB_AIO aio REQUIRED)
-find_package(isa-l REQUIRED)
-find_package(iomgr REQUIRED)
-find_package(farmhash REQUIRED)
-find_package(GTest REQUIRED)
-
-set (COMMON_DEPS
-        iomgr::iomgr
-        farmhash::farmhash
-        isa-l::isa-l
-        sisl::sisl
-    )
-
-set(COMMON_TEST_DEPS
-        ${COMMON_DEPS}
-        ${spdk_LIBRARY_LIST}
-        ${dpdk_LIBRARY_LIST}
-        GTest::gmock
-    )
+find_library(LIB_AIO aio QUIET REQUIRED)
+find_package(isa-l QUIET)
+find_package(iomgr QUIET REQUIRED)
+find_package(farmhash QUIET REQUIRED)
+find_package(GTest QUIET REQUIRED)
+find_package(NuraftMesg QUIET REQUIRED)
+
+list(APPEND COMMON_DEPS
+    iomgr::iomgr
+    farmhash::farmhash
+    -Wl,--whole-archive
+    NuraftMesg::proto
+    -Wl,--no-whole-archive
+    nuraft::nuraft
+    sisl::sisl
+)
+if (${isa-l_FOUND})
+    list(APPEND COMMON_DEPS isa-l::isa-l)
+else ()
+    add_flags("-DNO_ISAL")
+endif()
+
+list(APPEND COMMON_TEST_DEPS
+    ${COMMON_DEPS}
+    ${spdk_LIBRARY_LIST}
+    ${dpdk_LIBRARY_LIST}
+    GTest::gmock
+)
 
 include_directories (BEFORE lib/)
 include_directories (BEFORE include/)
@@ -49,6 +58,8 @@ set(HOMESTORE_OBJECTS
     $<TARGET_OBJECTS:hs_datasvc>
     $<TARGET_OBJECTS:hs_replication>
     lib/homestore.cpp
+    lib/crc.cpp
+    lib/logging.cpp
     #$<TARGET_OBJECTS:hs_cp>
     #$<TARGET_OBJECTS:indx_mgr>
 )

src/include/homestore/blk.h

@@ -133,6 +133,7 @@ struct MultiBlkId : public BlkId {
     BlkId to_single_blkid() const;
 
     static uint32_t expected_serialized_size(uint16_t num_pieces);
+    static uint32_t max_serialized_size();
     static int compare(MultiBlkId const& one, MultiBlkId const& two);
 
     struct iterator {

src/include/homestore/blkdata_service.hpp

@@ -99,7 +99,7 @@ class BlkDataService {
      * @return A Future that will resolve to an error code indicating the result of the write operation.
      */
     folly::Future< std::error_code > async_write(const char* buf, uint32_t size, MultiBlkId const& bid,
-                                                 bool part_of_batch);
+                                                 bool part_of_batch = false);
     /**
      * @brief : asynchronous write with input block ids;
      *
@@ -171,6 +171,13 @@ class BlkDataService {
      */
     uint32_t get_blk_size() const { return m_blk_size; }
 
+    /**
+     * @brief : get the blk size of this data service;
+     *
+     * @return : blk size
+     */
+    uint32_t get_align_size() const;
+
     /**
      * @brief : get the read block tracker handle;
      *

src/include/homestore/btree/btree_kv.hpp

@@ -256,17 +256,17 @@ class BtreeLinkInfo : public BtreeValue {
 
     sisl::blob serialize() const override {
         sisl::blob b;
-        b.size = sizeof(bnode_link_info);
-        b.bytes = uintptr_cast(const_cast< bnode_link_info* >(&info));
+        b.set_size(sizeof(bnode_link_info));
+        b.set_bytes(r_cast< const uint8_t* >(&info));
         return b;
     }
     uint32_t serialized_size() const override { return sizeof(bnode_link_info); }
     static uint32_t get_fixed_size() { return sizeof(bnode_link_info); }
     std::string to_string() const override { return fmt::format("{}.{}", info.m_bnodeid, info.m_link_version); }
 
     void deserialize(const sisl::blob& b, bool copy) override {
-        DEBUG_ASSERT_EQ(b.size, sizeof(bnode_link_info), "BtreeLinkInfo deserialize received invalid blob");
-        auto other = r_cast< bnode_link_info* >(b.bytes);
+        DEBUG_ASSERT_EQ(b.size(), sizeof(bnode_link_info), "BtreeLinkInfo deserialize received invalid blob");
+        auto other = r_cast< bnode_link_info const* >(b.cbytes());
         set_bnode_id(other->m_bnodeid);
         set_link_version(other->m_link_version);
     }

src/include/homestore/btree/detail/btree_node.hpp

@@ -24,8 +24,7 @@
 #include <sisl/utility/obj_life_counter.hpp>
 #include "btree_internal.hpp"
 #include <homestore/btree/btree_kv.hpp>
-// #include <iomgr/iomgr_flip.hpp>
-#include <isa-l/crc.h>
+#include <homestore/crc.h>
 
 namespace homestore {
 ENUM(locktype_t, uint8_t, NONE, READ, WRITE)
@@ -273,9 +272,9 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     virtual BtreeLinkInfo get_edge_value() const { return BtreeLinkInfo{edge_id(), edge_link_version()}; }
 
     virtual void set_edge_value(const BtreeValue& v) {
-        const auto b = v.serialize();
-        auto l = r_cast< BtreeLinkInfo::bnode_link_info* >(b.bytes);
-        DEBUG_ASSERT_EQ(b.size, sizeof(BtreeLinkInfo::bnode_link_info));
+        auto const b = v.serialize();
+        auto const l = r_cast< BtreeLinkInfo::bnode_link_info const* >(b.cbytes());
+        DEBUG_ASSERT_EQ(b.size(), sizeof(BtreeLinkInfo::bnode_link_info));
         set_edge_info(*l);
     }
 

src/include/homestore/btree/detail/prefix_node.hpp

@@ -79,21 +79,20 @@ class FixedPrefixNode : public VariantNode< K, V > {
                 sisl::blob const kblob = s_cast< K const& >(key).serialize_prefix();
                 sisl::blob const vblob = s_cast< V const& >(val).serialize_prefix();
 
-                DEBUG_ASSERT_EQ(kblob.size, key_size(), "Prefix key size mismatch with serialized prefix size");
-                DEBUG_ASSERT_EQ(vblob.size, value_size(), "Prefix value size mismatch with serialized prefix size");
+                DEBUG_ASSERT_EQ(kblob.size(), key_size(), "Prefix key size mismatch with serialized prefix size");
+                DEBUG_ASSERT_EQ(vblob.size(), value_size(), "Prefix value size mismatch with serialized prefix size");
 
                 uint8_t* cur_ptr = uintptr_cast(this) + sizeof(prefix_entry);
-                std::memcpy(cur_ptr, kblob.bytes, kblob.size);
-                cur_ptr += kblob.size;
-                std::memcpy(cur_ptr, vblob.bytes, vblob.size);
+                std::memcpy(cur_ptr, kblob.cbytes(), kblob.size());
+                cur_ptr += kblob.size();
+                std::memcpy(cur_ptr, vblob.cbytes(), vblob.size());
             }
         }
 
         sisl::blob key_buf() const {
-            return sisl::blob{const_cast< uint8_t* >(r_cast< uint8_t const* >(this) + sizeof(prefix_entry)),
-                              key_size()};
+            return sisl::blob{r_cast< uint8_t const* >(this) + sizeof(prefix_entry), key_size()};
         }
-        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes + key_buf().size, value_size()}; }
+        sisl::blob val_buf() const { return sisl::blob{key_buf().cbytes() + key_buf().size(), value_size()}; }
     };
 
     struct suffix_entry {
@@ -131,19 +130,19 @@ class FixedPrefixNode : public VariantNode< K, V > {
                 kblob = key.serialize();
                 vblob = val.serialize();
             }
-            DEBUG_ASSERT_EQ(kblob.size, key_size(), "Suffix key size mismatch with serialized suffix size");
-            DEBUG_ASSERT_EQ(vblob.size, value_size(), "Suffix value size mismatch with serialized suffix size");
+            DEBUG_ASSERT_EQ(kblob.size(), key_size(), "Suffix key size mismatch with serialized suffix size");
+            DEBUG_ASSERT_EQ(vblob.size(), value_size(), "Suffix value size mismatch with serialized suffix size");
 
-            std::memcpy(cur_ptr, kblob.bytes, kblob.size);
-            cur_ptr += kblob.size;
-            std::memcpy(cur_ptr, vblob.bytes, vblob.size);
+            std::memcpy(cur_ptr, kblob.cbytes(), kblob.size());
+            cur_ptr += kblob.size();
+            std::memcpy(cur_ptr, vblob.cbytes(), vblob.size());
         }
 
         sisl::blob key_buf() const {
             return sisl::blob{const_cast< uint8_t* >(r_cast< uint8_t const* >(this) + sizeof(suffix_entry)),
                               key_size()};
         }
-        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes + key_buf().size, value_size()}; }
+        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes() + key_buf().size(), value_size()}; }
     };
 #pragma pack()
 
@@ -778,7 +777,7 @@ class FixedPrefixNode : public VariantNode< K, V > {
         K prevKey;
         while (i < this->total_entries()) {
             K key = BtreeNode::get_nth_key< K >(i, false);
-            uint64_t kp = *(uint64_t*)key.serialize().bytes;
+            uint64_t kp = *(uint64_t*)key.serialize().bytes();
             if (i > 0 && prevKey.compare(key) > 0) {
                 DEBUG_ASSERT(false, "Found non sorted entry: {} -> {}", kp, to_string());
             }