From 898bf8f748e57cf40345641134f2bc25905b0cfa Mon Sep 17 00:00:00 2001 From: Yaming Kuang Date: Mon, 13 Nov 2023 17:37:08 -0700 Subject: [PATCH] issue: 226 Expose CP for external HomeStore client --- conanfile.py | 2 +- .../homestore}/checkpoint/cp.hpp | 33 +++++--------- src/include/homestore/checkpoint/cp_mgr.hpp | 11 +---- src/lib/blkalloc/append_blk_allocator.cpp | 2 +- src/lib/blkdata_svc/data_svc_cp.hpp | 2 +- src/lib/checkpoint/cp_internel.hpp | 45 +++++++++++++++++++ src/lib/checkpoint/cp_mgr.cpp | 2 +- src/lib/index/index_cp.hpp | 7 ++- src/lib/index/index_service.cpp | 1 + src/tests/test_cp_mgr.cpp | 2 +- 10 files changed, 67 insertions(+), 40 deletions(-) rename src/{lib => include/homestore}/checkpoint/cp.hpp (89%) create mode 100644 src/lib/checkpoint/cp_internel.hpp diff --git a/conanfile.py b/conanfile.py index b4477e155..e5d0a0ff9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,7 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "4.8.2" + version = "4.9.0" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" diff --git a/src/lib/checkpoint/cp.hpp b/src/include/homestore/checkpoint/cp.hpp similarity index 89% rename from src/lib/checkpoint/cp.hpp rename to src/include/homestore/checkpoint/cp.hpp index 05bda5be7..843db24d9 100644 --- a/src/lib/checkpoint/cp.hpp +++ b/src/include/homestore/checkpoint/cp.hpp @@ -20,9 +20,8 @@ #include #include -#include +#include #include -#include "common/homestore_assert.hpp" /* * These are the design requirements of this class. If we don't follow these requirements then there can be serious @@ -50,6 +49,7 @@ SISL_LOGGING_DECL(cp, replay) HS_PERIODIC_DETAILED_LOG(level, cp, "cp_id", cp_id, , , msg, ##__VA_ARGS__) #define CP_LOG(level, cp_id, msg, ...) HS_SUBMOD_LOG(level, cp, , "cp_id", cp_id, msg, ##__VA_ARGS__) +typedef int64_t cp_id_t; ENUM(cp_status_t, uint8_t, cp_unknown, // It is not inited yet. @@ -66,6 +66,16 @@ ENUM(cp_status_t, uint8_t, cp_cleaning); class CPContext; +class CPManager; + +VENUM(cp_consumer_t, uint8_t, + HS_CLIENT = 0, // Client of the homestore module + INDEX_SVC = 1, // Index service module + BLK_DATA_SVC = 2, // Block data service module + REPLICATION_SVC = 3, // Replication service module + SENTINEL = 4 // Should always be the last in this list +); + struct CP { std::atomic< cp_status_t > m_cp_status{cp_status_t::cp_unknown}; sisl::atomic_counter< int64_t > m_enter_cnt; @@ -89,23 +99,4 @@ struct CP { return fmt::format("CP={}: status={}, enter_count={}", m_cp_id, enum_name(get_status()), m_enter_cnt.get()); } }; - -class CPManager; -class CPWatchdog { -public: - CPWatchdog(CPManager* cp_mgr); - void set_cp(CP* cp); - void reset_cp(); - void cp_watchdog_timer(); - void stop(); - -private: - CP* m_cp; - CPManager* m_cp_mgr; - std::shared_mutex m_cp_mtx; - Clock::time_point m_last_state_ch_time; - uint64_t m_timer_sec{0}; - iomgr::timer_handle_t m_timer_hdl; - uint32_t m_progress_pct{0}; -}; } // namespace homestore diff --git a/src/include/homestore/checkpoint/cp_mgr.hpp b/src/include/homestore/checkpoint/cp_mgr.hpp index 15f0d67e7..507475c6e 100644 --- a/src/include/homestore/checkpoint/cp_mgr.hpp +++ b/src/include/homestore/checkpoint/cp_mgr.hpp @@ -25,10 +25,10 @@ #include #include +#include namespace homestore { static constexpr size_t MAX_CP_COUNT{2}; -typedef int64_t cp_id_t; class CPMgrMetrics : public sisl::MetricsGroup { public: @@ -46,15 +46,6 @@ class CPMgrMetrics : public sisl::MetricsGroup { ~CPMgrMetrics() { deregister_me_from_farm(); } }; -VENUM(cp_consumer_t, uint8_t, - HS_CLIENT = 0, // Client of the homestore module - INDEX_SVC = 1, // Index service module - BLK_DATA_SVC = 2, // Block data service module - REPLICATION_SVC = 3, // Replication service module - SENTINEL = 4 // Should always be the last in this list -); - -struct CP; class CPContext { protected: CP* m_cp; diff --git a/src/lib/blkalloc/append_blk_allocator.cpp b/src/lib/blkalloc/append_blk_allocator.cpp index 5852682d9..f149f9d8e 100644 --- a/src/lib/blkalloc/append_blk_allocator.cpp +++ b/src/lib/blkalloc/append_blk_allocator.cpp @@ -14,10 +14,10 @@ * * * *********************************************************************************/ #include +#include #include #include "append_blk_allocator.h" -#include "checkpoint/cp.hpp" namespace homestore { diff --git a/src/lib/blkdata_svc/data_svc_cp.hpp b/src/lib/blkdata_svc/data_svc_cp.hpp index a72e9dda1..eb516d170 100644 --- a/src/lib/blkdata_svc/data_svc_cp.hpp +++ b/src/lib/blkdata_svc/data_svc_cp.hpp @@ -15,8 +15,8 @@ *********************************************************************************/ #pragma once #include +#include #include -#include "checkpoint/cp.hpp" namespace homestore { diff --git a/src/lib/checkpoint/cp_internel.hpp b/src/lib/checkpoint/cp_internel.hpp new file mode 100644 index 000000000..a757b57aa --- /dev/null +++ b/src/lib/checkpoint/cp_internel.hpp @@ -0,0 +1,45 @@ +/********************************************************************************* + * Modifications Copyright 2017-2019 eBay Inc. + * + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed + * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + *********************************************************************************/ +#pragma once +#include +#include +#include +#include + +#include +#include +#include + +namespace homestore { +class CPManager; +class CPWatchdog { +public: + CPWatchdog(CPManager* cp_mgr); + void set_cp(CP* cp); + void reset_cp(); + void cp_watchdog_timer(); + void stop(); + +private: + CP* m_cp; + CPManager* m_cp_mgr; + std::shared_mutex m_cp_mtx; + Clock::time_point m_last_state_ch_time; + uint64_t m_timer_sec{0}; + iomgr::timer_handle_t m_timer_hdl; + uint32_t m_progress_pct{0}; +}; +} // namespace homestore diff --git a/src/lib/checkpoint/cp_mgr.cpp b/src/lib/checkpoint/cp_mgr.cpp index d26424248..9ac5754e0 100644 --- a/src/lib/checkpoint/cp_mgr.cpp +++ b/src/lib/checkpoint/cp_mgr.cpp @@ -22,7 +22,7 @@ #include "common/homestore_assert.hpp" #include "common/homestore_config.hpp" #include "common/resource_mgr.hpp" -#include "cp.hpp" +#include "cp_internel.hpp" namespace homestore { thread_local std::stack< CP* > CPGuard::t_cp_stack; diff --git a/src/lib/index/index_cp.hpp b/src/lib/index/index_cp.hpp index d7aad88dc..15a55184c 100644 --- a/src/lib/index/index_cp.hpp +++ b/src/lib/index/index_cp.hpp @@ -20,8 +20,8 @@ #include #include #include +#include -#include "checkpoint/cp.hpp" #include "device/virtual_dev.hpp" SISL_LOGGING_DECL(wbcache) @@ -37,7 +37,6 @@ struct IndexCPContext : public VDevCPContext { sisl::ConcurrentInsertVector< IndexBufferPtr >::iterator m_dirty_buf_it; public: - IndexCPContext(CP* cp) : VDevCPContext(cp) {} virtual ~IndexCPContext() = default; @@ -97,7 +96,8 @@ struct IndexCPContext : public VDevCPContext { // Use dfs to find if the graph is cycle auto it = m_dirty_buf_list.begin(); while (it != m_dirty_buf_list.end()) { - IndexBufferPtr buf = *it;; + IndexBufferPtr buf = *it; + ; std::set< IndexBuffer* > visited; check_cycle_recurse(buf, visited); ++it; @@ -153,7 +153,6 @@ struct IndexCPContext : public VDevCPContext { } }; - class IndexWBCache; class IndexCPCallbacks : public CPCallbacks { public: diff --git a/src/lib/index/index_service.cpp b/src/lib/index/index_service.cpp index bfa96f8bc..bfce92f09 100644 --- a/src/lib/index/index_service.cpp +++ b/src/lib/index/index_service.cpp @@ -19,6 +19,7 @@ #include "index/wb_cache.hpp" #include "index/index_cp.hpp" #include "common/homestore_utils.hpp" +#include "common/homestore_assert.hpp" #include "device/virtual_dev.hpp" #include "device/physical_dev.hpp" #include "device/chunk.h" diff --git a/src/tests/test_cp_mgr.cpp b/src/tests/test_cp_mgr.cpp index ad0e8a60d..33a0c77d4 100644 --- a/src/tests/test_cp_mgr.cpp +++ b/src/tests/test_cp_mgr.cpp @@ -22,7 +22,7 @@ #include #include #include -#include "checkpoint/cp.hpp" +#include #include "test_common/homestore_test_common.hpp" using namespace homestore;