Skip to content

Commit

Permalink
Changes to support PrefixNode and consolidation of tree (#211)
Browse files Browse the repository at this point in the history
* Changes to support PrefixNode and consolidation of tree

Following are the major changes in btree library:

1. A new BtreeKey and BtreeValue type called IntervalKey and IntervalValue which allows 
a particular key to be part of an larger interval (say 1-3 will result in key1, key2, key3 and 
each key is an BtreeIntervalKey).

2. Implemented a new tree node called PrefixNode which can store interval key and value effectively stores them with prefix and suffix to store the intervals compactly.

3. Removed the Extent code which was not used and replaced it with the IntervalKey/Value 
as mentioned above.

4. Added filtering for read/write/remove operations, so that consumer can change the way 
which key/value to be custom modify. This change effectively provides compare_and_set or 
compare_and_remove operations atomically. This change also removed the previous 
on_read_cb etc, which was passed on every callback

5. Removed the requirement for BtreeKey and Value to implement some static methods like get_key_size() etc, by creating a dummyKey, dummyValue. Now the requirement of the 
implementor of btree key and value only needs to ensure it is default constructible 
(which is already the case today)

6. Created new internal btree node APIs such as multi_put, multi_get instead of each btree operations repeat similar code to read or remove operations in leaf node. As a result, a 
method called match_range() is introduced in the btree node, which consolidates and 
searches for all range operation requirements. Moved most of the common operations into
variant_node class and now variant_node is derived from BtreeNode. PrefixNode overrides 
variant_node and implements its own version of multi_put etc. range operations.

7. Removed obsolete btree return status.

8. Consolidated the mem_btree test, mem_btree concurrent test and index_btree test into 
one btree_test_helper library. Modified the way range_scheduler picks the existing and 
working keys from boost icl to sis::Bitset as it simplifies them and also provides some 
essential feature which is always pick some keys to schedule instead of failing.
  • Loading branch information
hkadayam authored Nov 10, 2023
1 parent 8556e48 commit c979057
Show file tree
Hide file tree
Showing 27 changed files with 2,588 additions and 2,171 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ endif()

if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
message(STATUS "Debug build")
add_flags("-DDEBUG_RCU")
add_flags("-DDEBUG_RCU -D_DEBUG")
else()
message(STATUS "Release build")
if((${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") OR (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang"))
Expand Down
2 changes: 1 addition & 1 deletion conanfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class HomestoreConan(ConanFile):
name = "homestore"
version = "4.7.1"
version = "4.8.1"

homepage = "https://github.com/eBay/Homestore"
description = "HomeStore Storage Engine"
Expand Down
22 changes: 5 additions & 17 deletions src/include/homestore/btree/btree.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,6 @@ SISL_LOGGING_DECL(btree)

namespace homestore {

typedef std::function< bool(const BtreeKey&, const BtreeValue&, const BtreeRequest&) > on_kv_read_t;
typedef std::function< bool(const BtreeKey&, const BtreeValue&, const BtreeRequest&) > on_kv_remove_t;
typedef std::function< bool(const BtreeKey&, const BtreeKey&, const BtreeValue&, const BtreeRequest&) > on_kv_update_t;

using BtreeNodePtr = boost::intrusive_ptr< BtreeNode >;

struct BtreeThreadVariables {
Expand All @@ -57,11 +53,6 @@ class Btree {
std::atomic< uint64_t > m_req_id{0};
#endif

// Optional callback on various read or kv operations
on_kv_read_t m_on_read_cb{nullptr};
on_kv_update_t m_on_update_cb{nullptr};
on_kv_remove_t m_on_remove_cb{nullptr};

// This workaround of BtreeThreadVariables is needed instead of directly declaring statics
// to overcome the gcc bug, pointer here: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66944
static BtreeThreadVariables* bt_thread_vars() {
Expand All @@ -72,13 +63,14 @@ class Btree {
return fiber_map[this_id].get();
}

static bool is_repair_needed(const BtreeNodePtr& child_node, const BtreeLinkInfo& child_info);

protected:
BtreeConfig m_bt_cfg;

public:
/////////////////////////////////////// All External APIs /////////////////////////////
Btree(const BtreeConfig& cfg, on_kv_read_t&& read_cb = nullptr, on_kv_update_t&& update_cb = nullptr,
on_kv_remove_t&& remove_cb = nullptr);
Btree(const BtreeConfig& cfg);
virtual ~Btree();
virtual btree_status_t init(void* op_context);

Expand Down Expand Up @@ -174,10 +166,6 @@ class Btree {
void validate_sanity_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
void validate_sanity_next_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
void print_node(const bnodeid_t& bnodeid) const;
bool call_on_read_kv_cb(const BtreeNodePtr& node, uint32_t idx, const BtreeRequest& req) const;
bool call_on_remove_kv_cb(const BtreeNodePtr& node, uint32_t idx, const BtreeRequest& req) const;
bool call_on_update_kv_cb(const BtreeNodePtr& node, uint32_t idx, const BtreeKey& new_key,
const BtreeRequest& req) const;
void append_route_trace(BtreeRequest& req, const BtreeNodePtr& node, btree_event_t event, uint32_t start_idx = 0,
uint32_t end_idx = 0) const;

Expand All @@ -194,10 +182,10 @@ class Btree {
btree_status_t check_split_root(ReqT& req);

template < typename ReqT >
bool is_split_needed(const BtreeNodePtr& node, const BtreeConfig& cfg, ReqT& req) const;
bool is_split_needed(const BtreeNodePtr& node, ReqT& req) const;

btree_status_t split_node(const BtreeNodePtr& parent_node, const BtreeNodePtr& child_node, uint32_t parent_ind,
BtreeKey* out_split_key, void* context);
K* out_split_key, void* context);
btree_status_t mutate_extents_in_leaf(const BtreeNodePtr& my_node, BtreeRangePutRequest< K >& rpreq);
btree_status_t repair_split(const BtreeNodePtr& parent_node, const BtreeNodePtr& child_node1,
uint32_t parent_split_idx, void* context);
Expand Down
28 changes: 13 additions & 15 deletions src/include/homestore/btree/btree.ipp
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,8 @@

namespace homestore {
template < typename K, typename V >
Btree< K, V >::Btree(const BtreeConfig& cfg, on_kv_read_t&& read_cb, on_kv_update_t&& update_cb,
on_kv_remove_t&& remove_cb) :
m_metrics{cfg.name().c_str()},
m_node_size{cfg.node_size()},
m_on_read_cb{std::move(read_cb)},
m_on_update_cb{std::move(update_cb)},
m_on_remove_cb{std::move(remove_cb)},
m_bt_cfg{cfg} {
Btree< K, V >::Btree(const BtreeConfig& cfg) :
m_metrics{cfg.name().c_str()}, m_node_size{cfg.node_size()}, m_bt_cfg{cfg} {
m_bt_cfg.set_node_data_size(cfg.node_size() - sizeof(persistent_hdr_t));
}

Expand Down Expand Up @@ -105,7 +99,7 @@ retry:
if (ret != btree_status_t::success) { goto out; }
is_leaf = root->is_leaf();

if (is_split_needed(root, m_bt_cfg, put_req)) {
if (is_split_needed(root, put_req)) {
// Time to do the split of root.
unlock_node(root, acq_lock);
m_btree_lock.unlock_shared();
Expand Down Expand Up @@ -143,8 +137,7 @@ out:
#ifndef NDEBUG
check_lock_debug();
#endif
if (ret != btree_status_t::success && ret != btree_status_t::fast_path_not_possible &&
ret != btree_status_t::cp_mismatch) {
if (ret != btree_status_t::success && ret != btree_status_t::cp_mismatch) {
BT_LOG(ERROR, "btree put failed {}", ret);
COUNTER_INCREMENT(m_metrics, write_err_cnt, 1);
}
Expand Down Expand Up @@ -267,9 +260,9 @@ btree_status_t Btree< K, V >::query(BtreeQueryRequest< K >& qreq, std::vector< s
if ((qreq.query_type() == BtreeQueryType::SWEEP_NON_INTRUSIVE_PAGINATION_QUERY ||
qreq.query_type() == BtreeQueryType::TREE_TRAVERSAL_QUERY)) {
if (out_values.size()) {
K& out_last_key = out_values.back().first;
qreq.set_cursor_key(out_last_key);
K out_last_key = out_values.back().first;
if (out_last_key.compare(qreq.input_range().end_key()) >= 0) { ret = btree_status_t::success; }
qreq.shift_working_range(std::move(out_last_key), false /* non inclusive*/);
} else {
DEBUG_ASSERT_NE(ret, btree_status_t::has_more, "Query returned has_more, but no values added")
}
Expand All @@ -280,8 +273,7 @@ out:
#ifndef NDEBUG
check_lock_debug();
#endif
if (ret != btree_status_t::success && ret != btree_status_t::has_more &&
ret != btree_status_t::fast_path_not_possible) {
if ((ret != btree_status_t::success) && (ret != btree_status_t::has_more)) {
BT_LOG(ERROR, "btree query failed {}", ret);
COUNTER_INCREMENT(m_metrics, query_err_cnt, 1);
}
Expand Down Expand Up @@ -354,11 +346,17 @@ template < typename K, typename V >
bnodeid_t Btree< K, V >::root_node_id() const {
return m_root_node_info.bnode_id();
}

template < typename K, typename V >
uint64_t Btree< K, V >::root_link_version() const {
return m_root_node_info.link_version();
}

template < typename K, typename V >
bool Btree< K, V >::is_repair_needed(const BtreeNodePtr& child_node, const BtreeLinkInfo& child_info) {
return child_info.link_version() != child_node->link_version();
}

// TODO: Commenting out flip till we figure out how to move flip dependency inside sisl package.
#if 0
#ifdef _PRERELEASE
Expand Down
Loading

0 comments on commit c979057

Please sign in to comment.