diff --git a/conanfile.py b/conanfile.py index 0afc2ba77..79ec4b8d4 100644 --- a/conanfile.py +++ b/conanfile.py @@ -5,8 +5,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "6.4.15" - + version = "6.4.16" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine" topics = ("ebay", "nublox") diff --git a/src/include/homestore/btree/detail/simple_node.hpp b/src/include/homestore/btree/detail/simple_node.hpp index af3ff6a50..d909c202f 100644 --- a/src/include/homestore/btree/detail/simple_node.hpp +++ b/src/include/homestore/btree/detail/simple_node.hpp @@ -225,63 +225,67 @@ class SimpleNode : public VariantNode< K, V > { } std::string to_string_keys(bool print_friendly = false) const override { - std::string delimiter = print_friendly ? "\n" : "\t"; - std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode()); - auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ", - print_friendly ? "------------------------------------------------------------\n" : "", - this->node_id(), this->link_version(), this->level(), this->total_entries(), - (this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen()); - if (!this->is_leaf() && (this->has_valid_edge())) { - fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid, - this->edge_info().m_link_version); - } - if (this->total_entries() == 0) { - fmt::format_to(std::back_inserter(str), " [EMPTY] "); - return str; - } - if (!this->is_leaf()) { - fmt::format_to(std::back_inserter(str), " ["); - for (uint32_t i{0}; i < this->total_entries(); ++i) { - uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key(); - BtreeLinkInfo child_info; - get_nth_value(i, &child_info, false /* copy */); - fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(), - i == this->total_entries() - 1 ? "" : ", "); + // FIXME: Implement this, key may not be a unit32_t + return ""; +#if 0 + std::string delimiter = print_friendly ? "\n" : "\t"; + std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode()); + auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ", + print_friendly ? "------------------------------------------------------------\n" : "", + this->node_id(), this->link_version(), this->level(), this->total_entries(), + (this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen()); + if (!this->is_leaf() && (this->has_valid_edge())) { + fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid, + this->edge_info().m_link_version); } - fmt::format_to(std::back_inserter(str), "]"); - return str; - } - uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key(); - uint32_t cur_key = prev_key; - uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key(); - if (last_key - prev_key == this->total_entries() - 1) { - if (this->total_entries() == 1) - fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key); - else - fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key); - return str; - } - fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key); - uint32_t start_interval_key = prev_key; - for (uint32_t i{1}; i < this->total_entries(); ++i) { - cur_key = BtreeNode::get_nth_key< K >(i, false).key(); - if (cur_key != prev_key + 1) { - if (start_interval_key == prev_key) { - fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key); - } else { - fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key); + if (this->total_entries() == 0) { + fmt::format_to(std::back_inserter(str), " [EMPTY] "); + return str; + } + if (!this->is_leaf()) { + fmt::format_to(std::back_inserter(str), " ["); + for (uint32_t i{0}; i < this->total_entries(); ++i) { + uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key(); + BtreeLinkInfo child_info; + get_nth_value(i, &child_info, false /* copy */); + fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(), + i == this->total_entries() - 1 ? "" : ", "); } - start_interval_key = cur_key; + fmt::format_to(std::back_inserter(str), "]"); + return str; + } + uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key(); + uint32_t cur_key = prev_key; + uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key(); + if (last_key - prev_key == this->total_entries() - 1) { + if (this->total_entries() == 1) + fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key); + else + fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key); + return str; + } + fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key); + uint32_t start_interval_key = prev_key; + for (uint32_t i{1}; i < this->total_entries(); ++i) { + cur_key = BtreeNode::get_nth_key< K >(i, false).key(); + if (cur_key != prev_key + 1) { + if (start_interval_key == prev_key) { + fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key); + } else { + fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key); + } + start_interval_key = cur_key; + } + prev_key = cur_key; } - prev_key = cur_key; - } - if (start_interval_key == prev_key) { - fmt::format_to(std::back_inserter(str), "]"); - } else { - fmt::format_to(std::back_inserter(str), "-{}]", cur_key); - } - return str; + if (start_interval_key == prev_key) { + fmt::format_to(std::back_inserter(str), "]"); + } else { + fmt::format_to(std::back_inserter(str), "-{}]", cur_key); + } + return str; +#endif } #ifndef NDEBUG diff --git a/src/include/homestore/index/index_table.hpp b/src/include/homestore/index/index_table.hpp index 27ff2e4e5..3461fb357 100644 --- a/src/include/homestore/index/index_table.hpp +++ b/src/include/homestore/index/index_table.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include SISL_LOGGING_DECL(wbcache) @@ -127,12 +128,12 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { auto prev_state = idx_node->m_idx_buf->m_state.exchange(index_buf_state_t::DIRTY); if (prev_state == index_buf_state_t::CLEAN) { // It was clean before, dirtying it first time, add it to the wb_cache list to flush - HS_DBG_ASSERT_EQ(idx_node->m_idx_buf->m_dirtied_cp_id, cp_ctx->id(), + BT_DBG_ASSERT_EQ(idx_node->m_idx_buf->m_dirtied_cp_id, cp_ctx->id(), "Writing a node which was not acquired by this cp"); wb_cache().write_buf(node, idx_node->m_idx_buf, cp_ctx); LOGTRACEMOD(wbcache, "add to dirty list cp {} {}", cp_ctx->id(), idx_node->m_idx_buf->to_string()); } else { - HS_DBG_ASSERT_NE( + BT_DBG_ASSERT_NE( (int)prev_state, (int)index_buf_state_t::FLUSHING, "Writing on a node buffer which was currently in flushing state on cur_cp={} buffer_cp_id={}", cp_ctx->id(), idx_node->m_idx_buf->m_dirtied_cp_id) @@ -211,7 +212,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { auto const last_parent_key = parent_node->get_last_key< K >(); auto const is_parent_edge_node = parent_node->has_valid_edge(); if ((parent_node->total_entries() == 0) && !is_parent_edge_node) { - HS_LOG_ASSERT(false, "Parent node={} is empty and not an edge node but was asked to repair", + BT_LOG_ASSERT(false, "Parent node={} is empty and not an edge node but was asked to repair", parent_node->node_id()); return btree_status_t::not_found; } @@ -222,7 +223,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { auto ret = this->get_child_and_lock_node(parent_node, 0, child_info, child_node, locktype_t::READ, locktype_t::READ, cp_ctx); if (ret != btree_status_t::success) { - HS_LOG_ASSERT(false, "Parent node={} repair failed, because first child_node get has failed with ret={}", + BT_LOG_ASSERT(false, "Parent node={} repair failed, because first child_node get has failed with ret={}", parent_node->node_id(), enum_name(ret)); return ret; } @@ -239,7 +240,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { BtreeNodeList new_parent_nodes; do { if (child_node->has_valid_edge()) { - HS_DBG_ASSERT(!is_parent_edge_node, + BT_DBG_ASSERT(!is_parent_edge_node, "Child node={} is an edge node but parent_node={} is not an edge node", child_node->node_id(), parent_node->node_id()); cur_parent->set_edge_value(BtreeLinkInfo{child_node->node_id(), child_node->link_version()}); @@ -277,7 +278,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { // Move to the next child node auto const next_node_id = child_node->next_bnode(); if (next_node_id == empty_bnodeid) { - HS_LOG_ASSERT( + BT_LOG_ASSERT( false, "Child node={} next_node_id is empty, while its not a edge node, parent_node={} repair is partial", child_node->node_id(), parent_node->node_id()); @@ -287,7 +288,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > { ret = this->read_and_lock_node(next_node_id, child_node, locktype_t::READ, locktype_t::READ, cp_ctx); if (ret != btree_status_t::success) { - HS_LOG_ASSERT(false, "Parent node={} repair is partial, because child_node get has failed with ret={}", + BT_LOG_ASSERT(false, "Parent node={} repair is partial, because child_node get has failed with ret={}", parent_node->node_id(), enum_name(ret)); break; } diff --git a/src/lib/device/device_manager.cpp b/src/lib/device/device_manager.cpp index 8a6fe5a92..3fe811de5 100644 --- a/src/lib/device/device_manager.cpp +++ b/src/lib/device/device_manager.cpp @@ -308,8 +308,10 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) { // according to their capacity // the total size of all pdevs of a certain type - auto total_type_size = - std::accumulate(pdevs.begin(), pdevs.end(), 0u, [](int r, const PhysicalDev* a) { return r + a->data_size(); }); + uint64_t total_type_size = std::accumulate(pdevs.begin(), pdevs.end(), 0ull, + [](uint64_t r, const PhysicalDev* a) { return r + a->data_size(); }); + + LOGINFO("total size of type {} in this homestore is {}", vparam.dev_type, total_type_size) uint32_t total_created_chunks{0}; @@ -321,8 +323,12 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) { auto total_chunk_num_in_pdev = static_cast< uint32_t >(vparam.num_chunks * (pdev->data_size() / static_cast< float >(total_type_size))); - LOGINFO("{} chunks is created on pdev {} for vdev {}", total_chunk_num_in_pdev, pdev->get_devname(), - vparam.vdev_name) + RELEASE_ASSERT(vparam.num_chunks >= total_chunk_num_in_pdev, + "chunks in pdev {} is {}, larger than total chunks {} , which is expected to be created ", + pdev->get_devname(), total_chunk_num_in_pdev, vparam.num_chunks); + + LOGINFO("{} chunks is created on pdev {} for vdev {}, pdev data size is {}", total_chunk_num_in_pdev, + pdev->get_devname(), vparam.vdev_name, pdev->data_size()); // Create chunk ids for all chunks in each of these pdevs for (uint32_t c{0}; c < total_chunk_num_in_pdev; ++c) { @@ -343,7 +349,6 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) { } LOGINFO("{} chunks is created for vdev {}, expected {}", total_created_chunks, vparam.vdev_name, vparam.num_chunks); - // Handle any initialization needed. vdev->init(); diff --git a/src/tests/test_common/homestore_test_common.hpp b/src/tests/test_common/homestore_test_common.hpp index eeec3201c..cc7d59720 100644 --- a/src/tests/test_common/homestore_test_common.hpp +++ b/src/tests/test_common/homestore_test_common.hpp @@ -237,8 +237,11 @@ class HSTestHelper { if (token.devs_.empty()) { auto const devs = SISL_OPTIONS["device_list"].as< std::vector< std::string > >(); for (const auto& name : devs) { - // iomgr::DriveInterface::emulate_drive_type(name, iomgr::drive_type::block_hdd); - token.devs_.emplace_back(name, homestore::HSDevType::Data); + // TODO:: Add support for fast and data devices in device_list + token.devs_.emplace_back(name, + token.devs_.empty() + ? homestore::HSDevType::Fast + : homestore::HSDevType::Data); // First device is fast device } } else { LOGINFO("Taking input dev_list: {}", @@ -246,9 +249,8 @@ class HSTestHelper { [](const std::string& s, const homestore::dev_info& dinfo) { return s.empty() ? dinfo.dev_name : s + "," + dinfo.dev_name; })); - - if (init_device && !fake_restart) { init_raw_devices(token.devs_); } } + if (init_device && !fake_restart) init_raw_devices(token.devs_); } else { /* create files */ LOGINFO("creating {} device files with each of size {} ", ndevices, homestore::in_bytes(dev_size)); @@ -265,11 +267,12 @@ class HSTestHelper { } } - if (!fake_restart) - HS_REL_ASSERT_EQ(token.devs_.size() > 2, true, - "if not fake restart, we need at least 3 device to run the ut of simulating restart with " - "missing drive. current device num is {}", - token.devs_.size()); + if (!fake_restart && token.name_ == "test_data_service") + HS_REL_ASSERT_EQ( + token.devs_.size() > 2, true, + "if not fake restart, we need at least 3 device to run the data_service ut of simulating restart with " + "missing drive. current device num is {}", + token.devs_.size()); if (is_spdk) { LOGINFO("Spdk with more than 2 threads will cause overburden test systems, changing nthreads to 2"); @@ -333,7 +336,8 @@ class HSTestHelper { .chunk_sel_type = svc_params[HS_SERVICE::DATA].custom_chunk_selector ? chunk_selector_type_t::CUSTOM : chunk_selector_type_t::ROUND_ROBIN}}, - {HS_SERVICE::INDEX, {.size_pct = svc_params[HS_SERVICE::INDEX].size_pct}}, + {HS_SERVICE::INDEX, + {.dev_type = homestore::HSDevType::Fast, .size_pct = svc_params[HS_SERVICE::INDEX].size_pct}}, {HS_SERVICE::REPLICATION, {.size_pct = svc_params[HS_SERVICE::REPLICATION].size_pct, .alloc_type = svc_params[HS_SERVICE::REPLICATION].blkalloc_type,