Merge branch 'master' into yk_repl_fetch_remote

.jenkins/Dockerfile

@@ -13,7 +13,10 @@ RUN set -eux; \
     rm -rf /var/lib/apt/lists/*;
 
 COPY test_index_btree /usr/local/bin/test_index_btree
+COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr
+COPY test_log_store /usr/local/bin/test_log_store
 COPY btree_test.py /usr/local/bin/scripts/btree_test.py
+COPY log_meta_test.py /usr/local/bin/scripts/log_meta_test.py
 
 EXPOSE 5000
-# ##########   #######   ############
+# ##########   #######   ############

.jenkins/jenkinsfile_nightly

@@ -42,7 +42,10 @@ pipeline {
                     steps {
                         sh "conan create --build missing -o sisl:prerelease=True -o homestore:sanitize=True -o homestore:skip_testing=True -pr debug . ${PROJECT}/${VER}@"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;"
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;"
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/test_log_store' -exec cp {} .jenkins/test_log_store \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/btree_test.py' -exec install -Dm755 {} .jenkins/btree_test.py \\; "
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*bin/scripts/log_meta_test.py' -exec install -Dm755 {} .jenkins/log_meta_test.py \\; "
                     }
                     post {
                         failure {

conanfile.py

@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "5.0.1"
+    version = "5.0.4"
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"
     topics = ("ebay", "nublox")

src/include/homestore/checkpoint/cp_mgr.hpp

@@ -154,6 +154,7 @@ class CPManager {
     superblk< cp_mgr_super_block > m_sb;
     std::vector< iomgr::io_fiber_t > m_cp_io_fibers;
     iomgr::timer_handle_t m_cp_timer_hdl;
+    std::atomic< bool > m_cp_shutdown_initiated{false};
 
 public:
     CPManager();

src/include/homestore/index/index_table.hpp

@@ -50,9 +50,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
     }
 
     void destroy() override {
-        auto cpg = hs()->cp_mgr().cp_guard();
-        auto op_context = (void*)cpg.context(cp_consumer_t::INDEX_SVC);
-        Btree< K, V >::destroy_btree(op_context);
+        Btree< K, V >::destroy_btree(nullptr);
     }
 
     btree_status_t init() {

src/lib/blkalloc/bitmap_blk_allocator.cpp

@@ -84,7 +84,7 @@ bool BitmapBlkAllocator::is_blk_alloced_on_disk(const BlkId& b, bool use_lock) c
 
 BlkAllocStatus BitmapBlkAllocator::alloc_on_disk(BlkId const& bid) {
     if (!is_persistent()) {
-        //for non-persistent bitmap nothing is needed to do. So always return success
+        // for non-persistent bitmap nothing is needed to do. So always return success
         return BlkAllocStatus::SUCCESS;
     }
 
@@ -149,6 +149,7 @@ void BitmapBlkAllocator::free_on_disk(BlkId const& bid) {
                                         "Expected disk bits to set blk num {} num blks {}", b.blk_num(), b.blk_count());
                 }
             }
+
             m_disk_bm->reset_bits(b.blk_num(), b.blk_count());
         }
     };

src/lib/blkalloc/varsize_blk_allocator.cpp

@@ -640,6 +640,7 @@ BlkAllocStatus VarsizeBlkAllocator::mark_blk_allocated(BlkId const& bid) {
                              "Expected end bit to be smaller than portion end bit");
 #endif
         m_cache_bm->set_bits(bid.blk_num(), bid.blk_count());
+        incr_alloced_blk_count(bid.blk_count());
     }
     BLKALLOC_LOG(TRACE, "mark blk alloced directly to portion={} blkid={} set_bits_count={}",
                      blknum_to_portion_num(bid.blk_num()), bid.to_string(), get_alloced_blk_count());

src/lib/checkpoint/cp_mgr.cpp

@@ -74,10 +74,16 @@ void CPManager::shutdown() {
     LOGINFO("Stopping cp timer");
     iomanager.cancel_timer(m_cp_timer_hdl, true);
     m_cp_timer_hdl = iomgr::null_timer_handle;
+    m_cp_shutdown_initiated = true;
 
-    auto cp = get_cur_cp();
-    delete (cp);
+    LOGINFO("Trigger cp flush");
+    auto success = trigger_cp_flush(true /* force */).get();
+    HS_REL_ASSERT_EQ(success, true, "CP Flush failed");
+    LOGINFO("Trigger cp done");
+
+    delete (m_cur_cp);
     rcu_xchg_pointer(&m_cur_cp, nullptr);
+
     m_metrics.reset();
     if (m_wd_cp) {
         m_wd_cp->stop();
@@ -220,12 +226,24 @@ void CPManager::on_cp_flush_done(CP* cp) {
         m_sb.write();
 
         cleanup_cp(cp);
-        cp->m_comp_promise.setValue(true);
 
-        m_in_flush_phase = false;
+        // Setting promise will cause the CP manager destructor to cleanup
+        // before getting a chance to do the checking if shutdown has been
+        // initiated or not.
+        auto shutdown_initiated = m_cp_shutdown_initiated.load();
+        auto promise = std::move(cp->m_comp_promise);
+
         m_wd_cp->reset_cp();
         delete cp;
 
+        promise.setValue(true);
+        if (shutdown_initiated) {
+            // If shutdown initiated, dont trigger another CP.
+            // Dont access any cp state after this.
+            return;
+        }
+        m_in_flush_phase = false;
+
         // Trigger CP in case there is one back to back CP
         {
             auto cur_cp = cp_guard();
@@ -257,7 +275,7 @@ void CPManager::start_cp_thread() {
     auto ctx = std::make_shared< Context >();
 
     // Start a reactor with 9 fibers (8 for sync io)
-    iomanager.create_reactor("cp_io", iomgr::INTERRUPT_LOOP, 8u, [this, &ctx](bool is_started) {
+    iomanager.create_reactor("cp_io", iomgr::INTERRUPT_LOOP, 8u, [this, ctx](bool is_started) {
         if (is_started) {
             {
                 std::unique_lock< std::mutex > lk{ctx->mtx};

src/lib/device/virtual_dev.cpp

@@ -150,9 +150,10 @@ BlkAllocStatus VirtualDev::commit_blk(BlkId const& blkid) {
     HS_LOG(DEBUG, device, "commit_blk: bid {}", blkid.to_string());
     auto const recovering = homestore::hs()->is_initializing();
     if (!recovering) {
-        HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode", blkid.to_string());
+        HS_DBG_ASSERT(is_blk_alloced(blkid), "commiting blkid {} is not allocated in non-recovery mode",
+                      blkid.to_string());
     } else {
-       chunk->blk_allocator_mutable()->mark_blk_allocated(blkid);
+        chunk->blk_allocator_mutable()->mark_blk_allocated(blkid);
     }
     return chunk->blk_allocator_mutable()->alloc_on_disk(blkid);
 }
@@ -169,8 +170,13 @@ BlkAllocStatus VirtualDev::alloc_contiguous_blks(blk_count_t nblks, blk_alloc_hi
         } else {
             ret = alloc_blks(nblks, hints, mbid);
         }
-        HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!");
-        out_blkid = mbid.to_single_blkid();
+
+        if (ret == BlkAllocStatus::SUCCESS || (ret == BlkAllocStatus::PARTIAL && hints.partial_alloc_ok)) {
+            HS_REL_ASSERT_EQ(mbid.num_pieces(), 1, "out blkid more than 1 entries will lead to blk leak!");
+            out_blkid = mbid.to_single_blkid();
+        }
+
+        // for failure case, fall through and return the status to caller;
     } catch (const std::exception& e) {
         ret = BlkAllocStatus::FAILED;
         HS_DBG_ASSERT(0, "{}", e.what());
@@ -235,6 +241,13 @@ BlkAllocStatus VirtualDev::alloc_blks(blk_count_t nblks, blk_alloc_hints const&
 
         auto nblks_this_iter = out_bid.blk_count();
         nblks_remain = (nblks_remain < nblks_this_iter) ? 0 : (nblks_remain - nblks_this_iter);
+
+        if (status != BlkAllocStatus::SUCCESS && status != BlkAllocStatus::PARTIAL) {
+            out_blkids.pop_back();
+            // all chunks has been tried, but still failed to allocate;
+            // break out and return status to caller;
+            break;
+        }
     } while (nblks_remain);
 
     return status;

src/lib/homestore.cpp

@@ -230,9 +230,17 @@ void HomeStore::shutdown() {
 
     LOGINFO("Homestore shutdown is started");
 
+    m_cp_mgr->shutdown();
+    m_cp_mgr.reset();
+
+    if (has_repl_data_service()) {
+        s_cast< GenericReplService* >(m_repl_service.get())->stop();
+        m_repl_service.reset();
+    }
+
     if (has_index_service()) {
         m_index_service->stop();
-        //        m_index_service.reset();
+        // m_index_service.reset();
     }
     if (has_repl_data_service()) {
         s_cast< GenericReplService* >(m_repl_service.get())->stop();
@@ -253,8 +261,10 @@ void HomeStore::shutdown() {
 
     m_dev_mgr->close_devices();
     m_dev_mgr.reset();
-    m_cp_mgr->shutdown();
-    m_cp_mgr.reset();
+
+#ifdef _PRERELEASE
+    flip::Flip::instance().stop_rpc_server();
+#endif
 
     HomeStore::reset_instance();
     LOGINFO("Homestore is completed its shutdown");

src/lib/index/index_service.cpp

@@ -78,11 +78,6 @@ void IndexService::start() {
 
 void IndexService::stop() {
     std::unique_lock lg(m_index_map_mtx);
-    auto fut = homestore::hs()->cp_mgr().trigger_cp_flush(true /* force */);
-    auto success = std::move(fut).get();
-    HS_REL_ASSERT_EQ(success, true, "CP Flush failed");
-    LOGINFO("CP Flush completed");
-
     for (auto [id, tbl] : m_index_map) {
         tbl->destroy();
     }

src/lib/meta/meta_blk_service.cpp

@@ -930,6 +930,8 @@ void MetaBlkService::free_ovf_blk_chain(const BlkId& obid) {
         // free on-disk data bid
         auto* data_bid = ovf_hdr->get_data_bid();
         for (decltype(ovf_hdr->h.nbids) i{0}; i < ovf_hdr->h.nbids; ++i) {
+            HS_LOG(DEBUG, metablk, "before freeing data bid: {}, mstore used size: {}", data_bid[i].to_string(),
+                   m_sb_vdev->used_size());
             m_sb_vdev->free_blk(data_bid[i]);
             total_nblks_freed += data_bid[i].blk_count();
 

src/lib/replication/repl_dev/solo_repl_dev.cpp

@@ -138,7 +138,6 @@ void SoloReplDev::cp_flush(CP*) {
     m_rd_sb.write();
 }
 
-void SoloReplDev::cp_cleanup(CP*) { m_data_journal->truncate(m_rd_sb->checkpoint_lsn); }
+void SoloReplDev::cp_cleanup(CP*) { /* m_data_journal->truncate(m_rd_sb->checkpoint_lsn); */ }
 
 } // namespace homestore
-

src/tests/CMakeLists.txt

@@ -37,7 +37,7 @@ if (${build_nonio_tests})
     add_executable(test_mem_btree ${TEST_MEMBTREE_SOURCE_FILES})
     target_link_libraries(test_mem_btree ${COMMON_TEST_DEPS} GTest::gtest)
     add_test(NAME MemBtree COMMAND test_mem_btree)
-    set_tests_properties(MemBtree PROPERTIES TIMEOUT 180)
+    set_tests_properties(MemBtree PROPERTIES TIMEOUT 600)
 
     add_executable(test_blk_read_tracker)
     target_sources(test_blk_read_tracker PRIVATE test_blk_read_tracker.cpp ../lib/blkdata_svc/blk_read_tracker.cpp ../lib/blkalloc/blk.cpp)
@@ -74,6 +74,7 @@ if (${io_tests})
     target_link_libraries(test_index_btree homestore ${COMMON_TEST_DEPS} GTest::gtest)
     add_test(NAME IndexBtree COMMAND test_index_btree)
     set_property(TEST IndexBtree PROPERTY ENVIRONMENT "ASAN_OPTIONS=detect_stack_use_after_return=true")
+    set_tests_properties(IndexBtree PROPERTIES TIMEOUT 600)
 
     add_executable(test_data_service)
     target_sources(test_data_service PRIVATE test_data_service.cpp)
@@ -110,8 +111,8 @@ if (${io_tests})
         add_test(NAME MetaBlkMgr-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_meta_blk_mgr)
         add_test(NAME DataService-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_data_service)
         add_test(NAME SoloReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_solo_repl_dev)
-        add_test(NAME HomeRaftLogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_home_raft_logstore)
-        add_test(NAME RaftReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_raft_repl_dev)
+        # add_test(NAME HomeRaftLogStore-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_home_raft_logstore)
+        # add_test(NAME RaftReplDev-Epoll COMMAND ${CMAKE_BINARY_DIR}/bin/test_raft_repl_dev)
     endif()
 
     can_build_spdk_io_tests(spdk_tests)