Merge pull request #267 from eBay/repl

merge repl branch into master branch

.github/workflows/build_dependencies.yml

@@ -119,6 +119,14 @@ jobs:
         ref: master
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Retrieve Dependencies
+      uses: actions/checkout@v3
+      with:
+        repository: eBay/nuraft_mesg
+        path: import/nuraft_mesg
+        ref: main
+      if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Load IOMgr Cache
       uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
       with:
@@ -128,6 +136,15 @@ jobs:
         fail_on_cache_miss: true
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Load NuraftMesg Cache
+      uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
+      with:
+        testing: 'False'
+        path: import/nuraft_mesg
+        key_prefix: NuMesgDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        fail_on_cache_miss: true
+      if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Setup Conan
       uses: eBay/sisl/.github/actions/setup_conan@stable/v8.x
       with:
@@ -139,6 +156,7 @@ jobs:
         sudo apt-get install -y python3-pyelftools libaio-dev
         python -m pip install pyelftools
         conan export import/iomgr oss/master
+        conan export import/nuraft_mesg oss/main
         cached_pkgs=$(ls -1d ~/.conan/data/*/*/*/*/package | sed 's,.*data/,,' | cut -d'/' -f1,2 | paste -sd',' - -)
         echo "::info:: Pre-cached: ${cached_pkgs}"
       if: ${{ inputs.testing == 'True' || steps.restore-cache.outputs.cache-hit != 'true' }}
@@ -150,6 +168,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o testing=off \
             -s build_type=${{ inputs.build-type }} \
             --build missing \
@@ -178,6 +197,15 @@ jobs:
         fail_on_cache_miss: true
       if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }}
 
+    - name: Reload NuraftMesg Cache
+      uses: eBay/sisl/.github/actions/load_conan@stable/v8.x
+      with:
+        testing: 'False'
+        path: import/nuraft_mesg
+        key_prefix: NuMesgDeps-${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.malloc-impl }}-${{ inputs.prerelease }}
+        fail_on_cache_miss: true
+      if: ${{ inputs.testing == 'True' && github.event_name != 'pull_request' && steps.restore-cache.outputs.cache-hit != 'true' }}
+
     - name: Create and Test Package
       run: |
         sanitize=$([[ "${{ inputs.tooling }}" == "Sanitize" ]] && echo "True" || echo "False")
@@ -186,6 +214,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o homestore:sanitize=${sanitize} \
             -s build_type=${{ inputs.build-type }} \
             --build missing \
@@ -199,6 +228,7 @@ jobs:
             -o sisl:malloc_impl=${{ inputs.malloc-impl }} \
             -o sisl:testing=False \
             -o iomgr:testing=off \
+            -o nuraft_mesg:testing=False \
             -o coverage=True \
             -s build_type=${{ inputs.build-type }} \
             --build missing \

conanfile.py

@@ -5,8 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "4.9.4"
-
+    version = "5.0.1"
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
     topics = ("ebay", "nublox")
@@ -57,6 +56,7 @@ def build_requirements(self):
     def requirements(self):
         self.requires("iomgr/[~=11, include_prerelease=True]@oss/master")
         self.requires("sisl/[~=11, include_prerelease=True]@oss/master")
+        self.requires("nuraft_mesg/[~=2, include_prerelease=True]@oss/main")
 
         self.requires("farmhash/cci.20190513@")
         if self.settings.arch in ['x86', 'x86_64']:

src/CMakeLists.txt

@@ -8,10 +8,15 @@ find_package(isa-l QUIET)
 find_package(iomgr QUIET REQUIRED)
 find_package(farmhash QUIET REQUIRED)
 find_package(GTest QUIET REQUIRED)
+find_package(NuraftMesg QUIET REQUIRED)
 
 list(APPEND COMMON_DEPS
     iomgr::iomgr
     farmhash::farmhash
+    -Wl,--whole-archive
+    NuraftMesg::proto
+    -Wl,--no-whole-archive
+    nuraft::nuraft
     sisl::sisl
 )
 if (${isa-l_FOUND})
@@ -54,6 +59,7 @@ set(HOMESTORE_OBJECTS
     $<TARGET_OBJECTS:hs_replication>
     lib/homestore.cpp
     lib/crc.cpp
+    lib/logging.cpp
     #$<TARGET_OBJECTS:hs_cp>
     #$<TARGET_OBJECTS:indx_mgr>
 )

src/include/homestore/blk.h

@@ -133,6 +133,7 @@ struct MultiBlkId : public BlkId {
     BlkId to_single_blkid() const;
 
     static uint32_t expected_serialized_size(uint16_t num_pieces);
+    static uint32_t max_serialized_size();
     static int compare(MultiBlkId const& one, MultiBlkId const& two);
 
     struct iterator {

src/include/homestore/blkdata_service.hpp

@@ -99,7 +99,7 @@ class BlkDataService {
      * @return A Future that will resolve to an error code indicating the result of the write operation.
      */
     folly::Future< std::error_code > async_write(const char* buf, uint32_t size, MultiBlkId const& bid,
-                                                 bool part_of_batch);
+                                                 bool part_of_batch = false);
     /**
      * @brief : asynchronous write with input block ids;
      *
@@ -171,6 +171,13 @@ class BlkDataService {
      */
     uint32_t get_blk_size() const { return m_blk_size; }
 
+    /**
+     * @brief : get the blk size of this data service;
+     *
+     * @return : blk size
+     */
+    uint32_t get_align_size() const;
+
     /**
      * @brief : get the read block tracker handle;
      *

src/include/homestore/btree/btree_kv.hpp

@@ -256,17 +256,17 @@ class BtreeLinkInfo : public BtreeValue {
 
     sisl::blob serialize() const override {
         sisl::blob b;
-        b.size = sizeof(bnode_link_info);
-        b.bytes = uintptr_cast(const_cast< bnode_link_info* >(&info));
+        b.set_size(sizeof(bnode_link_info));
+        b.set_bytes(r_cast< const uint8_t* >(&info));
         return b;
     }
     uint32_t serialized_size() const override { return sizeof(bnode_link_info); }
     static uint32_t get_fixed_size() { return sizeof(bnode_link_info); }
     std::string to_string() const override { return fmt::format("{}.{}", info.m_bnodeid, info.m_link_version); }
 
     void deserialize(const sisl::blob& b, bool copy) override {
-        DEBUG_ASSERT_EQ(b.size, sizeof(bnode_link_info), "BtreeLinkInfo deserialize received invalid blob");
-        auto other = r_cast< bnode_link_info* >(b.bytes);
+        DEBUG_ASSERT_EQ(b.size(), sizeof(bnode_link_info), "BtreeLinkInfo deserialize received invalid blob");
+        auto other = r_cast< bnode_link_info const* >(b.cbytes());
         set_bnode_id(other->m_bnodeid);
         set_link_version(other->m_link_version);
     }

src/include/homestore/btree/detail/btree_node.hpp

@@ -272,9 +272,9 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     virtual BtreeLinkInfo get_edge_value() const { return BtreeLinkInfo{edge_id(), edge_link_version()}; }
 
     virtual void set_edge_value(const BtreeValue& v) {
-        const auto b = v.serialize();
-        auto l = r_cast< BtreeLinkInfo::bnode_link_info* >(b.bytes);
-        DEBUG_ASSERT_EQ(b.size, sizeof(BtreeLinkInfo::bnode_link_info));
+        auto const b = v.serialize();
+        auto const l = r_cast< BtreeLinkInfo::bnode_link_info const* >(b.cbytes());
+        DEBUG_ASSERT_EQ(b.size(), sizeof(BtreeLinkInfo::bnode_link_info));
         set_edge_info(*l);
     }
 

src/include/homestore/btree/detail/prefix_node.hpp

@@ -79,21 +79,20 @@ class FixedPrefixNode : public VariantNode< K, V > {
                 sisl::blob const kblob = s_cast< K const& >(key).serialize_prefix();
                 sisl::blob const vblob = s_cast< V const& >(val).serialize_prefix();
 
-                DEBUG_ASSERT_EQ(kblob.size, key_size(), "Prefix key size mismatch with serialized prefix size");
-                DEBUG_ASSERT_EQ(vblob.size, value_size(), "Prefix value size mismatch with serialized prefix size");
+                DEBUG_ASSERT_EQ(kblob.size(), key_size(), "Prefix key size mismatch with serialized prefix size");
+                DEBUG_ASSERT_EQ(vblob.size(), value_size(), "Prefix value size mismatch with serialized prefix size");
 
                 uint8_t* cur_ptr = uintptr_cast(this) + sizeof(prefix_entry);
-                std::memcpy(cur_ptr, kblob.bytes, kblob.size);
-                cur_ptr += kblob.size;
-                std::memcpy(cur_ptr, vblob.bytes, vblob.size);
+                std::memcpy(cur_ptr, kblob.cbytes(), kblob.size());
+                cur_ptr += kblob.size();
+                std::memcpy(cur_ptr, vblob.cbytes(), vblob.size());
             }
         }
 
         sisl::blob key_buf() const {
-            return sisl::blob{const_cast< uint8_t* >(r_cast< uint8_t const* >(this) + sizeof(prefix_entry)),
-                              key_size()};
+            return sisl::blob{r_cast< uint8_t const* >(this) + sizeof(prefix_entry), key_size()};
         }
-        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes + key_buf().size, value_size()}; }
+        sisl::blob val_buf() const { return sisl::blob{key_buf().cbytes() + key_buf().size(), value_size()}; }
     };
 
     struct suffix_entry {
@@ -131,19 +130,19 @@ class FixedPrefixNode : public VariantNode< K, V > {
                 kblob = key.serialize();
                 vblob = val.serialize();
             }
-            DEBUG_ASSERT_EQ(kblob.size, key_size(), "Suffix key size mismatch with serialized suffix size");
-            DEBUG_ASSERT_EQ(vblob.size, value_size(), "Suffix value size mismatch with serialized suffix size");
+            DEBUG_ASSERT_EQ(kblob.size(), key_size(), "Suffix key size mismatch with serialized suffix size");
+            DEBUG_ASSERT_EQ(vblob.size(), value_size(), "Suffix value size mismatch with serialized suffix size");
 
-            std::memcpy(cur_ptr, kblob.bytes, kblob.size);
-            cur_ptr += kblob.size;
-            std::memcpy(cur_ptr, vblob.bytes, vblob.size);
+            std::memcpy(cur_ptr, kblob.cbytes(), kblob.size());
+            cur_ptr += kblob.size();
+            std::memcpy(cur_ptr, vblob.cbytes(), vblob.size());
         }
 
         sisl::blob key_buf() const {
             return sisl::blob{const_cast< uint8_t* >(r_cast< uint8_t const* >(this) + sizeof(suffix_entry)),
                               key_size()};
         }
-        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes + key_buf().size, value_size()}; }
+        sisl::blob val_buf() const { return sisl::blob{key_buf().bytes() + key_buf().size(), value_size()}; }
     };
 #pragma pack()
 
@@ -778,7 +777,7 @@ class FixedPrefixNode : public VariantNode< K, V > {
         K prevKey;
         while (i < this->total_entries()) {
             K key = BtreeNode::get_nth_key< K >(i, false);
-            uint64_t kp = *(uint64_t*)key.serialize().bytes;
+            uint64_t kp = *(uint64_t*)key.serialize().bytes();
             if (i > 0 && prevKey.compare(key) > 0) {
                 DEBUG_ASSERT(false, "Found non sorted entry: {} -> {}", kp, to_string());
             }

src/include/homestore/btree/detail/simple_node.hpp

@@ -183,9 +183,7 @@ class SimpleNode : public VariantNode< K, V > {
 
     void get_nth_key_internal(uint32_t ind, BtreeKey& out_key, bool copy) const override {
         DEBUG_ASSERT_LT(ind, this->total_entries(), "node={}", to_string());
-        sisl::blob b;
-        b.bytes = (uint8_t*)(this->node_data_area_const() + (get_nth_obj_size(ind) * ind));
-        b.size = get_nth_key_size(ind);
+        sisl::blob b{this->node_data_area_const() + (get_nth_obj_size(ind) * ind), get_nth_key_size(ind)};
         out_key.deserialize(b, copy);
     }
 
@@ -322,11 +320,11 @@ class SimpleNode : public VariantNode< K, V > {
             set_nth_value(ind, v);
         } else {
             uint8_t* entry = this->node_data_area() + (get_nth_obj_size(ind) * ind);
-            sisl::blob key_blob = k.serialize();
-            memcpy((void*)entry, key_blob.bytes, key_blob.size);
+            sisl::blob const key_blob = k.serialize();
+            memcpy((void*)entry, key_blob.cbytes(), key_blob.size());
 
-            sisl::blob val_blob = v.serialize();
-            memcpy((void*)(entry + key_blob.size), val_blob.bytes, val_blob.size);
+            sisl::blob const val_blob = v.serialize();
+            memcpy((void*)(entry + key_blob.size()), val_blob.cbytes(), val_blob.size());
         }
     }
 
@@ -343,20 +341,20 @@ class SimpleNode : public VariantNode< K, V > {
 
     void set_nth_key(uint32_t ind, BtreeKey* key) {
         uint8_t* entry = this->node_data_area() + (get_nth_obj_size(ind) * ind);
-        sisl::blob b = key->serialize();
-        memcpy(entry, b.bytes, b.size);
+        sisl::blob const b = key->serialize();
+        memcpy(entry, b.cbytes(), b.size());
     }
 
     void set_nth_value(uint32_t ind, const BtreeValue& v) {
         sisl::blob b = v.serialize();
         if (ind >= this->total_entries()) {
             RELEASE_ASSERT_EQ(this->is_leaf(), false, "setting value outside bounds on leaf node");
-            DEBUG_ASSERT_EQ(b.size, sizeof(BtreeLinkInfo::bnode_link_info),
+            DEBUG_ASSERT_EQ(b.size(), sizeof(BtreeLinkInfo::bnode_link_info),
                             "Invalid value size being set for non-leaf node");
-            this->set_edge_info(*r_cast< BtreeLinkInfo::bnode_link_info* >(b.bytes));
+            this->set_edge_info(*r_cast< BtreeLinkInfo::bnode_link_info const* >(b.cbytes()));
         } else {
             uint8_t* entry = this->node_data_area() + (get_nth_obj_size(ind) * ind) + get_nth_key_size(ind);
-            std::memcpy(entry, b.bytes, b.size);
+            std::memcpy(entry, b.cbytes(), b.size());
         }
     }
 };