Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

using BT instead of HS in index_table.hpp #442

Merged
merged 3 commits into from
Jun 13, 2024
Merged

using BT instead of HS in index_table.hpp #442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8e21034
change log module
JacksonYao287 Jun 12, 2024
bc9d538
fix bug
JacksonYao287 Jun 13, 2024
b10f962
make index using fast device
JacksonYao287 Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions conanfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@

class HomestoreConan(ConanFile):
name = "homestore"
version = "6.4.15"

version = "6.4.16"
homepage = "https://github.com/eBay/Homestore"
description = "HomeStore Storage Engine"
topics = ("ebay", "nublox")
Expand Down
110 changes: 57 additions & 53 deletions src/include/homestore/btree/detail/simple_node.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,63 +225,67 @@ class SimpleNode : public VariantNode< K, V > {
}

std::string to_string_keys(bool print_friendly = false) const override {
std::string delimiter = print_friendly ? "\n" : "\t";
JacksonYao287 marked this conversation as resolved.
Show resolved Hide resolved
std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ",
print_friendly ? "------------------------------------------------------------\n" : "",
this->node_id(), this->link_version(), this->level(), this->total_entries(),
(this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen());
if (!this->is_leaf() && (this->has_valid_edge())) {
fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid,
this->edge_info().m_link_version);
}
if (this->total_entries() == 0) {
fmt::format_to(std::back_inserter(str), " [EMPTY] ");
return str;
}
if (!this->is_leaf()) {
fmt::format_to(std::back_inserter(str), " [");
for (uint32_t i{0}; i < this->total_entries(); ++i) {
uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key();
BtreeLinkInfo child_info;
get_nth_value(i, &child_info, false /* copy */);
fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(),
i == this->total_entries() - 1 ? "" : ", ");
// FIXME: Implement this, key may not be a unit32_t
return "";
#if 0
std::string delimiter = print_friendly ? "\n" : "\t";
std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ",
print_friendly ? "------------------------------------------------------------\n" : "",
this->node_id(), this->link_version(), this->level(), this->total_entries(),
(this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen());
if (!this->is_leaf() && (this->has_valid_edge())) {
fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid,
this->edge_info().m_link_version);
}
fmt::format_to(std::back_inserter(str), "]");
return str;
}
uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key();
uint32_t cur_key = prev_key;
uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key();
if (last_key - prev_key == this->total_entries() - 1) {
if (this->total_entries() == 1)
fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key);
else
fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key);
return str;
}
fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key);
uint32_t start_interval_key = prev_key;
for (uint32_t i{1}; i < this->total_entries(); ++i) {
cur_key = BtreeNode::get_nth_key< K >(i, false).key();
if (cur_key != prev_key + 1) {
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key);
} else {
fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key);
if (this->total_entries() == 0) {
fmt::format_to(std::back_inserter(str), " [EMPTY] ");
return str;
}
if (!this->is_leaf()) {
fmt::format_to(std::back_inserter(str), " [");
for (uint32_t i{0}; i < this->total_entries(); ++i) {
uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key();
JacksonYao287 marked this conversation as resolved.
Show resolved Hide resolved
BtreeLinkInfo child_info;
get_nth_value(i, &child_info, false /* copy */);
fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(),
i == this->total_entries() - 1 ? "" : ", ");
}
start_interval_key = cur_key;
fmt::format_to(std::back_inserter(str), "]");
return str;
}
uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key();
uint32_t cur_key = prev_key;
uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key();
if (last_key - prev_key == this->total_entries() - 1) {
if (this->total_entries() == 1)
fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key);
else
fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key);
return str;
}
fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key);
uint32_t start_interval_key = prev_key;
for (uint32_t i{1}; i < this->total_entries(); ++i) {
cur_key = BtreeNode::get_nth_key< K >(i, false).key();
if (cur_key != prev_key + 1) {
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key);
} else {
fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key);
}
start_interval_key = cur_key;
}
prev_key = cur_key;
}
prev_key = cur_key;
}

if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "]");
} else {
fmt::format_to(std::back_inserter(str), "-{}]", cur_key);
}
return str;
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "]");
} else {
fmt::format_to(std::back_inserter(str), "-{}]", cur_key);
}
return str;
#endif
}

#ifndef NDEBUG
Expand Down
15 changes: 8 additions & 7 deletions src/include/homestore/index/index_table.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <homestore/index_service.hpp>
#include <homestore/checkpoint/cp_mgr.hpp>
#include <homestore/index/wb_cache_base.hpp>
#include <homestore/btree/detail/btree_internal.hpp>
#include <iomgr/iomgr_flip.hpp>

SISL_LOGGING_DECL(wbcache)
Expand Down Expand Up @@ -127,12 +128,12 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
auto prev_state = idx_node->m_idx_buf->m_state.exchange(index_buf_state_t::DIRTY);
if (prev_state == index_buf_state_t::CLEAN) {
// It was clean before, dirtying it first time, add it to the wb_cache list to flush
HS_DBG_ASSERT_EQ(idx_node->m_idx_buf->m_dirtied_cp_id, cp_ctx->id(),
BT_DBG_ASSERT_EQ(idx_node->m_idx_buf->m_dirtied_cp_id, cp_ctx->id(),
"Writing a node which was not acquired by this cp");
wb_cache().write_buf(node, idx_node->m_idx_buf, cp_ctx);
LOGTRACEMOD(wbcache, "add to dirty list cp {} {}", cp_ctx->id(), idx_node->m_idx_buf->to_string());
} else {
HS_DBG_ASSERT_NE(
BT_DBG_ASSERT_NE(
(int)prev_state, (int)index_buf_state_t::FLUSHING,
"Writing on a node buffer which was currently in flushing state on cur_cp={} buffer_cp_id={}",
cp_ctx->id(), idx_node->m_idx_buf->m_dirtied_cp_id)
Expand Down Expand Up @@ -211,7 +212,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
auto const last_parent_key = parent_node->get_last_key< K >();
auto const is_parent_edge_node = parent_node->has_valid_edge();
if ((parent_node->total_entries() == 0) && !is_parent_edge_node) {
HS_LOG_ASSERT(false, "Parent node={} is empty and not an edge node but was asked to repair",
BT_LOG_ASSERT(false, "Parent node={} is empty and not an edge node but was asked to repair",
parent_node->node_id());
return btree_status_t::not_found;
}
Expand All @@ -222,7 +223,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
auto ret = this->get_child_and_lock_node(parent_node, 0, child_info, child_node, locktype_t::READ,
locktype_t::READ, cp_ctx);
if (ret != btree_status_t::success) {
HS_LOG_ASSERT(false, "Parent node={} repair failed, because first child_node get has failed with ret={}",
BT_LOG_ASSERT(false, "Parent node={} repair failed, because first child_node get has failed with ret={}",
parent_node->node_id(), enum_name(ret));
return ret;
}
Expand All @@ -239,7 +240,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
BtreeNodeList new_parent_nodes;
do {
if (child_node->has_valid_edge()) {
HS_DBG_ASSERT(!is_parent_edge_node,
BT_DBG_ASSERT(!is_parent_edge_node,
"Child node={} is an edge node but parent_node={} is not an edge node",
child_node->node_id(), parent_node->node_id());
cur_parent->set_edge_value(BtreeLinkInfo{child_node->node_id(), child_node->link_version()});
Expand Down Expand Up @@ -277,7 +278,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {
// Move to the next child node
auto const next_node_id = child_node->next_bnode();
if (next_node_id == empty_bnodeid) {
HS_LOG_ASSERT(
BT_LOG_ASSERT(
false,
"Child node={} next_node_id is empty, while its not a edge node, parent_node={} repair is partial",
child_node->node_id(), parent_node->node_id());
Expand All @@ -287,7 +288,7 @@ class IndexTable : public IndexTableBase, public Btree< K, V > {

ret = this->read_and_lock_node(next_node_id, child_node, locktype_t::READ, locktype_t::READ, cp_ctx);
if (ret != btree_status_t::success) {
HS_LOG_ASSERT(false, "Parent node={} repair is partial, because child_node get has failed with ret={}",
BT_LOG_ASSERT(false, "Parent node={} repair is partial, because child_node get has failed with ret={}",
parent_node->node_id(), enum_name(ret));
break;
}
Expand Down
15 changes: 10 additions & 5 deletions src/lib/device/device_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,10 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) {
// according to their capacity

// the total size of all pdevs of a certain type
auto total_type_size =
std::accumulate(pdevs.begin(), pdevs.end(), 0u, [](int r, const PhysicalDev* a) { return r + a->data_size(); });
uint64_t total_type_size = std::accumulate(pdevs.begin(), pdevs.end(), 0ull,
[](uint64_t r, const PhysicalDev* a) { return r + a->data_size(); });

LOGINFO("total size of type {} in this homestore is {}", vparam.dev_type, total_type_size)

uint32_t total_created_chunks{0};

Expand All @@ -321,8 +323,12 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) {
auto total_chunk_num_in_pdev =
static_cast< uint32_t >(vparam.num_chunks * (pdev->data_size() / static_cast< float >(total_type_size)));

LOGINFO("{} chunks is created on pdev {} for vdev {}", total_chunk_num_in_pdev, pdev->get_devname(),
vparam.vdev_name)
RELEASE_ASSERT(vparam.num_chunks >= total_chunk_num_in_pdev,
"chunks in pdev {} is {}, larger than total chunks {} , which is expected to be created ",
pdev->get_devname(), total_chunk_num_in_pdev, vparam.num_chunks);

LOGINFO("{} chunks is created on pdev {} for vdev {}, pdev data size is {}", total_chunk_num_in_pdev,
JacksonYao287 marked this conversation as resolved.
Show resolved Hide resolved
pdev->get_devname(), vparam.vdev_name, pdev->data_size());

// Create chunk ids for all chunks in each of these pdevs
for (uint32_t c{0}; c < total_chunk_num_in_pdev; ++c) {
Expand All @@ -343,7 +349,6 @@ shared< VirtualDev > DeviceManager::create_vdev(vdev_parameters&& vparam) {
}

LOGINFO("{} chunks is created for vdev {}, expected {}", total_created_chunks, vparam.vdev_name, vparam.num_chunks);

// Handle any initialization needed.
vdev->init();

Expand Down
24 changes: 14 additions & 10 deletions src/tests/test_common/homestore_test_common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,18 +237,20 @@ class HSTestHelper {
if (token.devs_.empty()) {
auto const devs = SISL_OPTIONS["device_list"].as< std::vector< std::string > >();
for (const auto& name : devs) {
// iomgr::DriveInterface::emulate_drive_type(name, iomgr::drive_type::block_hdd);
token.devs_.emplace_back(name, homestore::HSDevType::Data);
// TODO:: Add support for fast and data devices in device_list
token.devs_.emplace_back(name,
token.devs_.empty()
? homestore::HSDevType::Fast
: homestore::HSDevType::Data); // First device is fast device
}
} else {
LOGINFO("Taking input dev_list: {}",
std::accumulate(token.devs_.begin(), token.devs_.end(), std::string(""),
[](const std::string& s, const homestore::dev_info& dinfo) {
return s.empty() ? dinfo.dev_name : s + "," + dinfo.dev_name;
}));

if (init_device && !fake_restart) { init_raw_devices(token.devs_); }
}
if (init_device && !fake_restart) init_raw_devices(token.devs_);
} else {
/* create files */
LOGINFO("creating {} device files with each of size {} ", ndevices, homestore::in_bytes(dev_size));
Expand All @@ -265,11 +267,12 @@ class HSTestHelper {
}
}

if (!fake_restart)
HS_REL_ASSERT_EQ(token.devs_.size() > 2, true,
"if not fake restart, we need at least 3 device to run the ut of simulating restart with "
"missing drive. current device num is {}",
token.devs_.size());
if (!fake_restart && token.name_ == "test_data_service")
HS_REL_ASSERT_EQ(
token.devs_.size() > 2, true,
"if not fake restart, we need at least 3 device to run the data_service ut of simulating restart with "
"missing drive. current device num is {}",
token.devs_.size());

if (is_spdk) {
LOGINFO("Spdk with more than 2 threads will cause overburden test systems, changing nthreads to 2");
Expand Down Expand Up @@ -333,7 +336,8 @@ class HSTestHelper {
.chunk_sel_type = svc_params[HS_SERVICE::DATA].custom_chunk_selector
? chunk_selector_type_t::CUSTOM
: chunk_selector_type_t::ROUND_ROBIN}},
{HS_SERVICE::INDEX, {.size_pct = svc_params[HS_SERVICE::INDEX].size_pct}},
{HS_SERVICE::INDEX,
{.dev_type = homestore::HSDevType::Fast, .size_pct = svc_params[HS_SERVICE::INDEX].size_pct}},
{HS_SERVICE::REPLICATION,
{.size_pct = svc_params[HS_SERVICE::REPLICATION].size_pct,
.alloc_type = svc_params[HS_SERVICE::REPLICATION].blkalloc_type,
Expand Down
Loading