Merge branch 'master' into master

.jenkins/Dockerfile

@@ -15,8 +15,12 @@ RUN set -eux; \
 COPY test_index_btree /usr/local/bin/test_index_btree
 COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr
 COPY test_log_store /usr/local/bin/test_log_store
-COPY btree_test.py /usr/local/bin/scripts/btree_test.py
+COPY test_raft_repl_dev /usr/local/bin/test_raft_repl_dev
+COPY test_solo_repl_dev /usr/local/bin/test_solo_repl_dev
+COPY index_test.py /usr/local/bin/scripts/index_test.py
 COPY log_meta_test.py /usr/local/bin/scripts/log_meta_test.py
+COPY long_running.py /usr/local/bin/scripts/long_running.py
+
 
 EXPOSE 5000
 # ##########   #######   ############

.jenkins/jenkinsfile_nightly

@@ -44,8 +44,11 @@ pipeline {
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_log_store' -exec cp {} .jenkins/test_log_store \\;"
-                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/btree_test.py' -exec install -Dm755 {} .jenkins/btree_test.py \\; "
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_raft_repl_dev' -exec cp {} .jenkins/test_raft_repl_dev \\;"
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_solo_repl_dev' -exec cp {} .jenkins/test_solo_repl_dev \\;"
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/index_test.py' -exec install -Dm755 {} .jenkins/index_test.py \\; "
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; "
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/long_running.py' -exec install -Dm755 {} .jenkins/long_running.py \\; "
                     }
                     post {
                         failure {

conanfile.py

@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.4.9"
+    version = "6.4.11"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/lib/meta/meta_blk_service.cpp

@@ -196,9 +196,14 @@ void MetaBlkService::format_ssb() {
 // m_meta_lock should be while calling this function;
 void MetaBlkService::write_ssb() {
     // write current ovf blk to disk;
-    try {
-        m_sb_vdev->sync_write((const char*)m_ssb, block_size(), m_ssb->bid);
-    } catch (std::exception& e) { HS_REL_ASSERT(false, "exception happen during write {}", e.what()); }
+    auto error = m_sb_vdev->sync_write((const char*)m_ssb, block_size(), m_ssb->bid);
+    if (error.value()) {
+        // the offset and buffer length is printed in the error messages of iomgr.
+        // buf address here is to show whether the buffer is aligned or not.
+        // TODO: hanle this error properly
+        HS_REL_ASSERT(false, "error happens happen during write ssb: {}, buf address: {}", error.value(),
+                      (const char*)m_ssb);
+    }
 
     LOGINFO("Successfully write m_ssb to disk: {}", m_ssb->to_string());
 
@@ -436,9 +441,14 @@ void MetaBlkService::write_ovf_blk_to_disk(meta_blk_ovf_hdr* ovf_hdr, const uint
     HS_DBG_ASSERT_LE(ovf_hdr->h.context_sz + offset, sz);
 
     // write current ovf blk to disk;
-    try {
-        m_sb_vdev->sync_write((const char*)ovf_hdr, block_size(), ovf_hdr->h.bid);
-    } catch (std::exception& e) { HS_REL_ASSERT(false, "exception happen during write {}", e.what()); }
+    auto error = m_sb_vdev->sync_write((const char*)ovf_hdr, block_size(), ovf_hdr->h.bid);
+    if (error.value()) {
+        // the offset and buffer length is printed in the error messages of iomgr.
+        // buf address here is to show whether the buffer is aligned or not.
+        // TODO: hanle this error properly
+        HS_REL_ASSERT(false, "error happens happen during write: {}, buf address: {}", error.value(),
+                      (const char*)ovf_hdr);
+    }
 
     // NOTE: The start write pointer which is context data pointer plus offset must be dma boundary aligned
     // TO DO: Might need to differentiate based on data or fast type
@@ -488,9 +498,14 @@ void MetaBlkService::write_ovf_blk_to_disk(meta_blk_ovf_hdr* ovf_hdr, const uint
             size_written += (ovf_hdr->h.context_sz - size_written);
         }
 
-        try {
-            m_sb_vdev->sync_write(r_cast< const char* >(cur_ptr), cur_size, data_bid[i]);
-        } catch (std::exception& e) { HS_REL_ASSERT(false, "exception happen during write {}", e.what()); }
+        auto error = m_sb_vdev->sync_write(r_cast< const char* >(cur_ptr), cur_size, data_bid[i]);
+        if (error.value()) {
+            // the offset and buffer length is printed in the error messages of iomgr.
+            // buf address here is to show whether the buffer is aligned or not.
+            // TODO: hanle this error properly
+            HS_REL_ASSERT(false, "error happens happen during write: {}, buf address: {}", error.value(),
+                          r_cast< const char* >(cur_ptr));
+        }
     }
 
     if (data_buf) { hs_utils::iobuf_free(data_buf, sisl::buftag::metablk); }
@@ -501,9 +516,14 @@ void MetaBlkService::write_ovf_blk_to_disk(meta_blk_ovf_hdr* ovf_hdr, const uint
 
 void MetaBlkService::write_meta_blk_to_disk(meta_blk* mblk) {
     // write current ovf blk to disk;
-    try {
-        m_sb_vdev->sync_write((const char*)mblk, block_size(), mblk->hdr.h.bid);
-    } catch (std::exception& e) { HS_REL_ASSERT(false, "exception happen during write {}", e.what()); }
+    auto error = m_sb_vdev->sync_write((const char*)mblk, block_size(), mblk->hdr.h.bid);
+    if (error.value()) {
+        // the offset and buffer length is printed in the error messages of iomgr.
+        // buf address here is to show whether the buffer is aligned or not.
+        // TODO: hanle this error properly
+        HS_REL_ASSERT(false, "error happens happen during write_meta_blk_to_disk: {}, buf address: {}", error.value(),
+                      (const char*)mblk);
+    }
 }
 
 //
@@ -1122,11 +1142,8 @@ void MetaBlkService::recover_meta_block(meta_blk* mblk) {
         // if subsystem registered crc protection, verify crc before sending to subsystem;
         if (itr->second.do_crc) {
             const auto crc = crc32_ieee(init_crc32, buf->cbytes(), mblk->hdr.h.context_sz);
-
-            HS_REL_ASSERT_EQ(crc, uint32_cast(mblk->hdr.h.crc),
-                             "[type={}], CRC mismatch: {}/{}, on mblk bid: {}, context_sz: {}", mblk->hdr.h.type, crc,
-                             uint32_cast(mblk->hdr.h.crc), mblk->hdr.h.bid.to_string(),
-                             uint64_cast(mblk->hdr.h.context_sz));
+            HS_REL_ASSERT_EQ(crc, uint32_cast(mblk->hdr.h.crc), "CRC mismatch: {}/{}, meta_blk details: {}", crc,
+                             uint32_cast(mblk->hdr.h.crc), mblk->hdr.h.to_string());
         } else {
             HS_LOG(DEBUG, metablk, "[type={}] meta blk found with bypassing crc.", mblk->hdr.h.type);
         }

src/lib/meta/meta_sb.hpp

@@ -134,10 +134,11 @@ struct meta_blk_hdr_s {
     uint8_t pad[7];
 
     std::string to_string() const {
-        return fmt::format("type: {}, version: {}, magic: {}, crc: {}, next_bid: {}, prev_bid: {}, ovf_bid: {}, "
-                           "self_bid: {}, compressed: {}",
-                           type, version, magic, crc, next_bid.to_string(), prev_bid.to_string(), ovf_bid.to_string(),
-                           bid.to_string(), compressed);
+        return fmt::format(
+            "magic: {}, type: {}, version: {}, gen_cnt: {}, crc: {}, next_bid: {}, prev_bid: {}, "
+            "ovf_bid: {}, self_bid: {}, context_sz: {}, compressed_sz: {}, src_context_sz : {}, compressed: {} ",
+            magic, type, version, gen_cnt, crc, next_bid.to_string(), prev_bid.to_string(), ovf_bid.to_string(),
+            bid.to_string(), context_sz, compressed_sz, src_context_sz, compressed);
     }
 };
 #pragma pack()

src/tests/test_scripts/CMakeLists.txt

@@ -1,8 +1,9 @@
 file(COPY vol_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
 file(COPY home_blk_flip.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
 file(COPY home_blk_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
-file(COPY btree_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
+file(COPY index_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
 file(COPY log_meta_test.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
+file(COPY long_running.py DESTINATION ${CMAKE_BINARY_DIR}/bin/scripts)
 #add_test(NAME TestVolRecovery COMMAND ${CMAKE_BINARY_DIR}/bin/scripts/vol_test.py --test_suits=recovery --dirpath=${CMAKE_BINARY_DIR}/bin/)
 #SET_TESTS_PROPERTIES(TestVolRecovery PROPERTIES DEPENDS TestVol)