Allowing consumer of HS specified device type for each service (#355)

* Add dev_type in format_params. Setting to auto will decide based on whether we have FAST and the service characteristic. Also chaning pct_to_size to take the size of that type. Signed-off-by: Xiaoxi Chen <[email protected]>
eBay · Apr 5, 2024 · 60808d0 · 60808d0
1 parent 485e656
commit 60808d0
Show file tree

Hide file tree

Showing 11 changed files with 72 additions and 38 deletions.
diff --git a/conanfile.py b/conanfile.py
@@ -5,7 +5,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.0.1"
+    version = "6.2.1"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

diff --git a/src/include/homestore/blkdata_service.hpp b/src/include/homestore/blkdata_service.hpp
@@ -65,7 +65,7 @@ class BlkDataService {
      * @param chunk_sel_type The type of chunk selector to use for the virtual device.
      * @param num_chunks The number of chunks to use for the virtual device.
      */
-    void create_vdev(uint64_t size, uint32_t blk_size, blk_allocator_type_t alloc_type,
+    void create_vdev(uint64_t size, HSDevType devType, uint32_t blk_size, blk_allocator_type_t alloc_type,
                      chunk_selector_type_t chunk_sel_type, uint32_t num_chunks);
 
     /**

diff --git a/src/include/homestore/homestore_decl.hpp b/src/include/homestore/homestore_decl.hpp
@@ -148,7 +148,8 @@ static std::string in_bytes(uint64_t sz) {
 }
 
 struct hs_format_params {
-    float size_pct;
+    HSDevType dev_type{HSDevType::Data};
+    float size_pct; // size pct to that type
     uint32_t num_chunks{1};
     uint64_t chunk_size{0};
     uint32_t block_size{0};
@@ -176,6 +177,7 @@ struct hs_input_params {
     nlohmann::json to_json() const;
     std::string to_string() const { return to_json().dump(4); }
     uint64_t io_mem_size() const { return (hugepage_size != 0) ? hugepage_size : app_mem_size; }
+    bool has_fast_dev() const { return std::any_of(devices.begin(), devices.end(), [](const dev_info& d) { return d.dev_type == HSDevType::Fast; }); }
 };
 
 struct hs_engine_config {

diff --git a/src/include/homestore/index_service.hpp b/src/include/homestore/index_service.hpp
@@ -51,7 +51,7 @@ class IndexService {
     IndexService(std::unique_ptr< IndexServiceCallbacks > cbs);
 
     // Creates the vdev that is needed to initialize the device
-    void create_vdev(uint64_t size, uint32_t num_chunks);
+    void create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks);
 
     // Open the existing vdev which is represnted by the vdev_info_block
     shared< VirtualDev > open_vdev(const vdev_info& vb, bool load_existing);

diff --git a/src/include/homestore/logstore_service.hpp b/src/include/homestore/logstore_service.hpp
@@ -144,7 +144,7 @@ class LogStoreService {
      */
     void device_truncate(const device_truncate_cb_t& cb = nullptr, bool wait_till_done = false, bool dry_run = false);
 
-    folly::Future< std::error_code > create_vdev(uint64_t size, uint32_t chunk_size);
+    folly::Future< std::error_code > create_vdev(uint64_t size, HSDevType devType, uint32_t chunk_size);
     std::shared_ptr< VirtualDev > open_vdev(const vdev_info& vinfo, bool load_existing);
     std::shared_ptr< JournalVirtualDev > get_vdev() const { return m_logdev_vdev; }
     std::vector< std::shared_ptr< LogDev > > get_all_logdevs();

diff --git a/src/include/homestore/meta_service.hpp b/src/include/homestore/meta_service.hpp
@@ -103,7 +103,7 @@ class MetaBlkService {
     ~MetaBlkService() = default;
 
     // Creates the vdev that is needed to initialize the device
-    void create_vdev(uint64_t size, uint32_t num_chunks);
+    void create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks);
 
     // Open the existing vdev which is represented by the vdev_info
     shared< VirtualDev > open_vdev(const vdev_info& vinfo, bool load_existing);

diff --git a/src/lib/blkdata_svc/blkdata_service.cpp b/src/lib/blkdata_svc/blkdata_service.cpp
@@ -37,18 +37,18 @@ BlkDataService::BlkDataService(shared< ChunkSelector > chunk_selector) :
 BlkDataService::~BlkDataService() = default;
 
 // first-time boot path
-void BlkDataService::create_vdev(uint64_t size, uint32_t blk_size, blk_allocator_type_t alloc_type,
+void BlkDataService::create_vdev(uint64_t size, HSDevType devType, uint32_t blk_size, blk_allocator_type_t alloc_type,
                                  chunk_selector_type_t chunk_sel_type, uint32_t num_chunks) {
     hs_vdev_context vdev_ctx;
     vdev_ctx.type = hs_vdev_type_t::DATA_VDEV;
 
-    if (blk_size == 0) { blk_size = hs()->device_mgr()->optimal_page_size(HSDevType::Data); }
+    if (blk_size == 0) { blk_size = hs()->device_mgr()->optimal_page_size(devType); }
     m_vdev =
         hs()->device_mgr()->create_vdev(vdev_parameters{.vdev_name = "blkdata",
                                                         .vdev_size = size,
                                                         .num_chunks = num_chunks,
                                                         .blk_size = blk_size,
-                                                        .dev_type = HSDevType::Data,
+                                                        .dev_type = devType,
                                                         .alloc_type = alloc_type,
                                                         .chunk_sel_type = chunk_sel_type,
                                                         .multi_pdev_opts = vdev_multi_pdev_opts_t::ALL_PDEV_STRIPED,

diff --git a/src/lib/homestore.cpp b/src/lib/homestore.cpp
@@ -138,7 +138,11 @@ bool HomeStore::start(const hs_input_params& input, hs_before_services_starting_
 
     if (!m_dev_mgr->is_first_time_boot()) {
         m_dev_mgr->load_devices();
-        hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
+        if (input.has_fast_dev()) {
+            hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
+        } else {
+            hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Data}));
+        }
         do_start();
         return false;
     } else {
@@ -147,37 +151,66 @@ bool HomeStore::start(const hs_input_params& input, hs_before_services_starting_
 }
 
 void HomeStore::format_and_start(std::map< uint32_t, hs_format_params >&& format_opts) {
-    auto total_pct_sum = 0.0f;
+
+    std::map< HSDevType, float > total_pct_by_type = {{HSDevType::Fast, 0.0f}, {HSDevType::Data, 0.0f}};
+    // Accumulate total percentage of services on each device type
     for (const auto& [svc_type, fparams] : format_opts) {
-        total_pct_sum += fparams.size_pct;
+        total_pct_by_type[fparams.dev_type] += fparams.size_pct;
+    }
+
+    // Sanity check, each type accumulated pct <=100%
+    auto all_pct = 0;
+    for (const auto& [DevType, total_pct] : total_pct_by_type) {
+        all_pct += total_pct;
+        if (total_pct > 100.0f) {
+            LOGERROR("Total percentage of services on Device type {} is greater than 100.0f, total_pct_sum={}", DevType,
+                     total_pct);
+            throw std::invalid_argument("total percentage of on Device type {} services is greater than 100.0f");
+        }
+    }
+    // Sanity check , at least one service should be placed on some device type
+    if (all_pct == 0) {
+        LOGERROR("No services are configured to be placed on any device type");
+        throw std::invalid_argument("No services are configured to be placed on any device type");
     }
 
-    if (total_pct_sum > 100.0f) {
-        LOGERROR("Total percentage of all services is greater than 100.0f, total_pct_sum={}", total_pct_sum);
-        throw std::invalid_argument("total percentage of all services is greater than 100.0f");
+    // Sanity check, should have fast if fast pct >0
+    if (total_pct_by_type[HSDevType::Fast] > 0 && !HomeStoreStaticConfig::instance().input.has_fast_dev()) {
+        LOGERROR("Fast device is not configured but services are configured to be placed on fast device");
+        throw std::invalid_argument(
+            "Fast device is not configured but services are configured to be placed on fast device");
     }
 
     m_dev_mgr->format_devices();
-    hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
+    if (HomeStoreStaticConfig::instance().input.has_fast_dev()) {
+        hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Fast}));
+    } else {
+        hs_utils::set_btree_mempool_size(m_dev_mgr->atomic_page_size({HSDevType::Data}));
+    }
+
 
     std::vector< folly::Future< std::error_code > > futs;
     for (const auto& [svc_type, fparams] : format_opts) {
         if (fparams.size_pct == 0) { continue; }
 
         if ((svc_type & HS_SERVICE::META) && has_meta_service()) {
-            m_meta_service->create_vdev(pct_to_size(fparams.size_pct, HSDevType::Fast), fparams.num_chunks);
+            m_meta_service->create_vdev(pct_to_size(fparams.size_pct, fparams.dev_type), fparams.dev_type,
+                                        fparams.num_chunks);
 
         } else if ((svc_type & HS_SERVICE::LOG) && has_log_service()) {
-            futs.emplace_back(
-                m_log_service->create_vdev(pct_to_size(fparams.size_pct, HSDevType::Fast), fparams.chunk_size));
-        } else if ((svc_type & HS_SERVICE::DATA) && has_data_service()) {
-            m_data_service->create_vdev(pct_to_size(fparams.size_pct, HSDevType::Data), fparams.block_size,
-                                        fparams.alloc_type, fparams.chunk_sel_type, fparams.num_chunks);
+            futs.emplace_back(m_log_service->create_vdev(pct_to_size(fparams.size_pct, fparams.dev_type),
+                                                         fparams.dev_type, fparams.chunk_size));
         } else if ((svc_type & HS_SERVICE::INDEX) && has_index_service()) {
-            m_index_service->create_vdev(pct_to_size(fparams.size_pct, HSDevType::Fast), fparams.num_chunks);
+            m_index_service->create_vdev(pct_to_size(fparams.size_pct, fparams.dev_type), fparams.dev_type,
+                                         fparams.num_chunks);
+        } else if ((svc_type & HS_SERVICE::DATA) && has_data_service()) {
+            m_data_service->create_vdev(pct_to_size(fparams.size_pct, fparams.dev_type), fparams.dev_type,
+                                        fparams.block_size, fparams.alloc_type, fparams.chunk_sel_type,
+                                        fparams.num_chunks);
         } else if ((svc_type & HS_SERVICE::REPLICATION) && has_repl_data_service()) {
-            m_data_service->create_vdev(pct_to_size(fparams.size_pct, HSDevType::Data), fparams.block_size,
-                                        fparams.alloc_type, fparams.chunk_sel_type, fparams.num_chunks);
+            m_data_service->create_vdev(pct_to_size(fparams.size_pct, fparams.dev_type), fparams.dev_type,
+                                        fparams.block_size, fparams.alloc_type, fparams.chunk_sel_type,
+                                        fparams.num_chunks);
         }
     }
 
@@ -365,7 +398,7 @@ shared< VirtualDev > HomeStore::create_vdev_cb(const vdev_info& vinfo, bool load
 }
 
 uint64_t HomeStore::pct_to_size(float pct, HSDevType dev_type) const {
-    uint64_t sz = uint64_cast((pct * static_cast< double >(m_dev_mgr->total_capacity())) / 100);
+    uint64_t sz = uint64_cast((pct * static_cast< double >(m_dev_mgr->total_capacity(dev_type))) / 100);
     return sisl::round_up(sz, m_dev_mgr->optimal_page_size(dev_type));
 }
 

diff --git a/src/lib/index/index_service.cpp b/src/lib/index/index_service.cpp
@@ -36,16 +36,16 @@ IndexService::IndexService(std::unique_ptr< IndexServiceCallbacks > cbs) : m_svc
         nullptr);
 }
 
-void IndexService::create_vdev(uint64_t size, uint32_t num_chunks) {
-    auto const atomic_page_size = hs()->device_mgr()->atomic_page_size(HSDevType::Fast);
+void IndexService::create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks) {
+    auto const atomic_page_size = hs()->device_mgr()->atomic_page_size(devType);
     hs_vdev_context vdev_ctx;
     vdev_ctx.type = hs_vdev_type_t::INDEX_VDEV;
 
     hs()->device_mgr()->create_vdev(vdev_parameters{.vdev_name = "index",
                                                     .vdev_size = size,
                                                     .num_chunks = num_chunks,
                                                     .blk_size = atomic_page_size,
-                                                    .dev_type = HSDevType::Fast,
+                                                    .dev_type = devType,
                                                     .alloc_type = blk_allocator_type_t::fixed,
                                                     .chunk_sel_type = chunk_selector_type_t::ROUND_ROBIN,
                                                     .multi_pdev_opts = vdev_multi_pdev_opts_t::ALL_PDEV_STRIPED,
@@ -68,8 +68,7 @@ void IndexService::meta_blk_found(const sisl::byte_view& buf, void* meta_cookie)
 
 void IndexService::start() {
     // Start Writeback cache
-    m_wb_cache = std::make_unique< IndexWBCache >(m_vdev, hs()->evictor(),
-                                                  hs()->device_mgr()->atomic_page_size(HSDevType::Fast));
+    m_wb_cache = std::make_unique< IndexWBCache >(m_vdev, hs()->evictor(), m_vdev->atomic_page_size());
 
     // Register to CP for flush dirty buffers
     hs()->cp_mgr().register_consumer(cp_consumer_t::INDEX_SVC,
@@ -94,7 +93,7 @@ void IndexService::remove_index_table(const std::shared_ptr< IndexTableBase >& t
     m_index_map.erase(tbl->uuid());
 }
 
-uint32_t IndexService::node_size() const { return hs()->device_mgr()->atomic_page_size(HSDevType::Fast); }
+uint32_t IndexService::node_size() const { return m_vdev->atomic_page_size(); }
 
 uint64_t IndexService::used_size() const {
     auto size{0};

diff --git a/src/lib/logstore/log_store_service.cpp b/src/lib/logstore/log_store_service.cpp
@@ -54,8 +54,8 @@ LogStoreService::LogStoreService() {
         nullptr);
 }
 
-folly::Future< std::error_code > LogStoreService::create_vdev(uint64_t size, uint32_t chunk_size) {
-    const auto atomic_page_size = hs()->device_mgr()->atomic_page_size(HSDevType::Fast);
+folly::Future< std::error_code > LogStoreService::create_vdev(uint64_t size, HSDevType devType, uint32_t chunk_size) {
+    const auto atomic_page_size = hs()->device_mgr()->atomic_page_size(devType);
 
     hs_vdev_context hs_ctx;
     hs_ctx.type = hs_vdev_type_t::LOGDEV_VDEV;
@@ -78,7 +78,7 @@ folly::Future< std::error_code > LogStoreService::create_vdev(uint64_t size, uin
                                                         .num_chunks = 0,
                                                         .blk_size = atomic_page_size,
                                                         .chunk_size = chunk_size,
-                                                        .dev_type = HSDevType::Fast,
+                                                        .dev_type = devType,
                                                         .alloc_type = blk_allocator_type_t::none,
                                                         .chunk_sel_type = chunk_selector_type_t::ROUND_ROBIN,
                                                         .multi_pdev_opts = vdev_multi_pdev_opts_t::ALL_PDEV_STRIPED,

diff --git a/src/lib/meta/meta_blk_service.cpp b/src/lib/meta/meta_blk_service.cpp
@@ -46,8 +46,8 @@ MetaBlkService& meta_service() { return hs()->meta_service(); }
 
 MetaBlkService::MetaBlkService(const char* name) : m_metrics{name} { m_last_mblk_id = std::make_unique< BlkId >(); }
 
-void MetaBlkService::create_vdev(uint64_t size, uint32_t num_chunks) {
-    const auto phys_page_size = hs()->device_mgr()->optimal_page_size(HSDevType::Fast);
+void MetaBlkService::create_vdev(uint64_t size, HSDevType devType, uint32_t num_chunks) {
+    const auto phys_page_size = hs()->device_mgr()->optimal_page_size(devType);
 
     meta_vdev_context meta_ctx;
     meta_ctx.type = hs_vdev_type_t::META_VDEV;
@@ -56,7 +56,7 @@ void MetaBlkService::create_vdev(uint64_t size, uint32_t num_chunks) {
                                                     .vdev_size = size,
                                                     .num_chunks = num_chunks,
                                                     .blk_size = phys_page_size,
-                                                    .dev_type = HSDevType::Fast,
+                                                    .dev_type = devType,
                                                     .alloc_type = blk_allocator_type_t::varsize,
                                                     .chunk_sel_type = chunk_selector_type_t::ROUND_ROBIN,
                                                     .multi_pdev_opts = vdev_multi_pdev_opts_t::ALL_PDEV_STRIPED,