Skip to content

Commit

Permalink
Fixed split crash recovery link_buf issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@hkadayam
hkadayam committed Aug 2, 2024
1 parent 5c83762 commit ed6bd0d
Show file tree
Hide file tree
Showing 17 changed files with 370 additions and 321 deletions.
2 changes: 1 addition & 1 deletion conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class HomestoreConan(ConanFile):
name = "homestore"
version = "6.4.33"
version = "6.4.34"

homepage = "https://github.com/eBay/Homestore"
description = "HomeStore Storage Engine"
Expand Down
9 changes: 4 additions & 5 deletions src/include/homestore/btree/btree.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace homestore {
using BtreeNodePtr = boost::intrusive_ptr< BtreeNode >;
using BtreeNodeList = folly::small_vector< BtreeNodePtr, 3 >;

struct BtreeVisualizeVariables{
struct BtreeVisualizeVariables {
uint64_t parent;
uint64_t midPoint;
uint64_t index;
Expand Down Expand Up @@ -120,8 +120,8 @@ class Btree {
virtual std::pair< btree_status_t, uint64_t > destroy_btree(void* context);
nlohmann::json get_status(int log_level) const;

void print_tree(const std::string& file = "") const;
void print_tree_keys() const;
void dump_tree_to_file(const std::string& file = "") const;
std::string to_string_keys() const;
std::string visualize_tree_keys(const std::string& file) const;
uint64_t count_keys(bnodeid_t bnodeid) const;

Expand Down Expand Up @@ -202,8 +202,7 @@ class Btree {
uint64_t get_child_node_cnt(bnodeid_t bnodeid) const;
void to_string(bnodeid_t bnodeid, std::string& buf) const;
void to_string_keys(bnodeid_t bnodeid, std::string& buf) const;
void to_dot_keys(bnodeid_t bnodeid, std::string& buf,
std::map< uint32_t, std::vector< uint64_t > >& l_map,
void to_dot_keys(bnodeid_t bnodeid, std::string& buf, std::map< uint32_t, std::vector< uint64_t > >& l_map,
std::map< uint64_t, BtreeVisualizeVariables >& info_map) const;
void validate_sanity_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
void validate_sanity_next_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
Expand Down
8 changes: 4 additions & 4 deletions src/include/homestore/btree/btree.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,13 +308,13 @@ nlohmann::json Btree< K, V >::get_status(int log_level) const {
}

template < typename K, typename V >
void Btree< K, V >::print_tree(const std::string& file) const {
void Btree< K, V >::dump_tree_to_file(const std::string& file) const {
std::string buf;
m_btree_lock.lock_shared();
to_string(m_root_node_info.bnode_id(), buf);
m_btree_lock.unlock_shared();

LOGINFO( "Pre order traversal of tree:\n<{}>", buf);
BT_LOG(INFO, "Pre order traversal of tree:\n<{}>", buf);
if (!file.empty()) {
std::ofstream o(file);
o.write(buf.c_str(), buf.size());
Expand All @@ -323,13 +323,13 @@ void Btree< K, V >::print_tree(const std::string& file) const {
}

template < typename K, typename V >
void Btree< K, V >::print_tree_keys() const {
std::string Btree< K, V >::to_string_keys() const {
std::string buf;
m_btree_lock.lock_shared();
to_string_keys(m_root_node_info.bnode_id(), buf);
m_btree_lock.unlock_shared();

LOGINFO("Pre order traversal of tree:\n<{}>", buf);
return buf;
}

template < typename K, typename V >
Expand Down
31 changes: 17 additions & 14 deletions src/include/homestore/btree/detail/btree_mutate_impl.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,28 +299,31 @@ btree_status_t Btree< K, V >::split_node(const BtreeNodePtr& parent_node, const
return ret;
}

// template < typename K, typename V >
// template < typename ReqT >
// bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
// if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
// return node->total_entries() >= 3;
// } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
// return node->total_entries() >= 3;
// } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
// return node->total_entries() >= 3;;
// } else {
// return false;
// }
// }

template < typename K, typename V >
template < typename ReqT >
bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
return node->total_entries() >= 3;
return !node->has_room_for_put(btree_put_type::UPSERT, K::get_max_size(), BtreeLinkInfo::get_fixed_size());
} else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
return node->total_entries() >= 3;
return !node->has_room_for_put(req.m_put_type, req.first_key_size(), req.m_newval->serialized_size());
} else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
return node->total_entries() >= 3;;
return !node->has_room_for_put(req.m_put_type, req.key().serialized_size(), req.value().serialized_size());
} else {
return false;
}
}
//bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
// if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
// return !node->has_room_for_put(btree_put_type::UPSERT, K::get_max_size(), BtreeLinkInfo::get_fixed_size());
// } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
// return !node->has_room_for_put(req.m_put_type, req.first_key_size(), req.m_newval->serialized_size());
// } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
// return !node->has_room_for_put(req.m_put_type, req.key().serialized_size(), req.value().serialized_size());
// } else {
// return false;
// }
//}
} // namespace homestore
29 changes: 15 additions & 14 deletions src/include/homestore/btree/detail/btree_node.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,23 +69,24 @@ struct persistent_hdr_t {

persistent_hdr_t() : nentries{0}, leaf{0}, node_deleted{0} {}
std::string to_string() const {
auto snext = (next_node == empty_bnodeid) ? "" : "next_node="+ std::to_string(next_node);
auto sedge = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_nodeid="+ std::to_string(edge_info.m_bnodeid);
auto sedgelink = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_link_version="+ std::to_string(edge_info.m_link_version);
return fmt::format("magic={} version={} csum={} node_id={} {} nentries={} node_type={} is_leaf={} "
"node_deleted={} node_gen={} modified_cp_id={} link_version={} {}, "
"{} level={} ",
magic, version, checksum, node_id, snext, nentries, node_type, leaf, node_deleted,
node_gen, modified_cp_id, link_version, sedge, sedgelink,
level);
auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
? ""
: fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
return fmt::format("magic={} version={} csum={} node_id={}{} nentries={} node_type={} is_leaf={} "
"node_deleted={} node_gen={} modified_cp_id={} link_version={}{} level={} ",
magic, version, checksum, node_id, snext, nentries, node_type, leaf, node_deleted, node_gen,
modified_cp_id, link_version, sedge, level);
}

std::string to_compact_string() const {
auto snext = (next_node == empty_bnodeid) ? "" : "next="+ std::to_string(next_node);
auto sedge = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_nodeid="+ std::to_string(edge_info.m_bnodeid);
auto sleaf = leaf?"LEAF": "INTERIOR";
return fmt::format(" id={} {}{} {} nentries={} {} level={} modified_cp_id={}", node_id, snext,sedge, sleaf, nentries,
(node_deleted == 0x1) ? "Deleted" : "", level, modified_cp_id);
auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
? ""
: fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
return fmt::format("id={}{}{} {} level={} nentries={}{} mod_cp={}", node_id, snext, sedge,
leaf ? "LEAF" : "INTERIOR", level, nentries, (node_deleted == 0x1) ? " Deleted" : "",
modified_cp_id);
}
};
#pragma pack()
Expand Down
128 changes: 67 additions & 61 deletions src/include/homestore/btree/detail/simple_node.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ class SimpleNode : public VariantNode< K, V > {
}

bool has_room_for_put(btree_put_type put_type, uint32_t key_size, uint32_t value_size) const override {
#ifdef _PRERELEASE
// return (this->total_entries() <= 3);
#endif
return ((put_type == btree_put_type::UPSERT) || (put_type == btree_put_type::INSERT))
? (get_available_entries() > 0)
: true;
Expand All @@ -214,7 +217,7 @@ class SimpleNode : public VariantNode< K, V > {
(this->is_leaf() ? "LEAF" : "INTERIOR"), snext);
if (!this->is_leaf() && (this->has_valid_edge())) {
auto sedge = this->has_valid_edge() ? "edge:" + std::to_string(this->edge_info().m_bnodeid) + "." +
std::to_string(this->edge_info().m_link_version)
std::to_string(this->edge_info().m_link_version)
: "";
fmt::format_to(std::back_inserter(str), "{}", sedge);
}
Expand All @@ -229,71 +232,71 @@ class SimpleNode : public VariantNode< K, V > {

std::string to_string_keys(bool print_friendly = false) const override {
// FIXME: Implement this, key may not be a unit32_t
// return "";
//#if 0
std::string delimiter = print_friendly ? "\n" : "\t";
std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ",
print_friendly ? "------------------------------------------------------------\n" : "",
this->node_id(), this->link_version(), this->level(), this->total_entries(),
(this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen());
if (!this->is_leaf() && (this->has_valid_edge())) {
fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid,
this->edge_info().m_link_version);
}
if (this->total_entries() == 0) {
fmt::format_to(std::back_inserter(str), " [EMPTY] ");
return str;
}
if (!this->is_leaf()) {
fmt::format_to(std::back_inserter(str), " [");
for (uint32_t i{0}; i < this->total_entries(); ++i) {
uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key();
BtreeLinkInfo child_info;
get_nth_value(i, &child_info, false /* copy */);
fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(),
i == this->total_entries() - 1 ? "" : ", ");
}
fmt::format_to(std::back_inserter(str), "]");
return str;
}
uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key();
uint32_t cur_key = prev_key;
uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key();
if (last_key - prev_key == this->total_entries() - 1) {
if (this->total_entries() == 1)
fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key);
else
fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key);
return str;
// return "";
// #if 0
std::string delimiter = print_friendly ? "\n" : "\t";
std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ",
print_friendly ? "------------------------------------------------------------\n" : "",
this->node_id(), this->link_version(), this->level(), this->total_entries(),
(this->is_leaf() ? "LEAF" : "INTERIOR"), snext, this->node_gen());
if (!this->is_leaf() && (this->has_valid_edge())) {
fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid,
this->edge_info().m_link_version);
}
if (this->total_entries() == 0) {
fmt::format_to(std::back_inserter(str), " [EMPTY] ");
return str;
}
if (!this->is_leaf()) {
fmt::format_to(std::back_inserter(str), " [");
for (uint32_t i{0}; i < this->total_entries(); ++i) {
uint32_t cur_key = BtreeNode::get_nth_key< K >(i, false).key();
BtreeLinkInfo child_info;
get_nth_value(i, &child_info, false /* copy */);
fmt::format_to(std::back_inserter(str), "{}.{} {}", cur_key, child_info.link_version(),
i == this->total_entries() - 1 ? "" : ", ");
}
fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key);
uint32_t start_interval_key = prev_key;
for (uint32_t i{1}; i < this->total_entries(); ++i) {
cur_key = BtreeNode::get_nth_key< K >(i, false).key();
if (cur_key != prev_key + 1) {
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key);
} else {
fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key);
}
start_interval_key = cur_key;
fmt::format_to(std::back_inserter(str), "]");
return str;
}
uint32_t prev_key = BtreeNode::get_nth_key< K >(0, false).key();
uint32_t cur_key = prev_key;
uint32_t last_key = BtreeNode::get_nth_key< K >(this->total_entries() - 1, false).key();
if (last_key - prev_key == this->total_entries() - 1) {
if (this->total_entries() == 1)
fmt::format_to(std::back_inserter(str), "{}[{}]", delimiter, prev_key);
else
fmt::format_to(std::back_inserter(str), "{}[{}-{}]", delimiter, prev_key, last_key);
return str;
}
fmt::format_to(std::back_inserter(str), "{}0 - [{}", delimiter, prev_key);
uint32_t start_interval_key = prev_key;
for (uint32_t i{1}; i < this->total_entries(); ++i) {
cur_key = BtreeNode::get_nth_key< K >(i, false).key();
if (cur_key != prev_key + 1) {
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "-{}]{}{}- [{}", prev_key, delimiter, i, cur_key);
} else {
fmt::format_to(std::back_inserter(str), "]{}{}- [{}", delimiter, i, cur_key);
}
prev_key = cur_key;
start_interval_key = cur_key;
}
prev_key = cur_key;
}

if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "]");
} else {
fmt::format_to(std::back_inserter(str), "-{}]", cur_key);
}
return str;
//#endif
if (start_interval_key == prev_key) {
fmt::format_to(std::back_inserter(str), "]");
} else {
fmt::format_to(std::back_inserter(str), "-{}]", cur_key);
}
return str;
// #endif
}

std::string to_dot_keys() const override {
std::string str;
std::string snext = this->next_bnode() == empty_bnodeid? "": fmt::format("next_node={}", this->next_bnode());
std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
str += fmt::format(R"("{}" [
shape = none,
labelloc="c",
Expand All @@ -317,11 +320,14 @@ class SimpleNode : public VariantNode< K, V > {
<td port="connector{}"></td><td port="key{}">{}.{}</td>)",
i, i, cur_key, child_info.link_version());
}
std::string sedge = this->has_valid_edge() ? "edge:" + std::to_string( this->edge_info().m_bnodeid) + "." + std::to_string( this->edge_info().m_link_version):"";
std::string sedge = this->has_valid_edge() ? "edge:" + std::to_string(this->edge_info().m_bnodeid) + "." +
std::to_string(this->edge_info().m_link_version)
: "";
str += fmt::format(R"(
<td port="connector{}"></td>
<td>{}.{}<br/> gen={}<br/>{} {} </td></tr></table>>];)",this->total_entries(),this->node_id(), this->link_version(), this->node_gen(), snext, sedge );

<td>{}.{}<br/> gen={}<br/>{} {} </td></tr></table>>];)",
this->total_entries(), this->node_id(), this->link_version(), this->node_gen(), snext,
sedge);

} else {
std::string keys_buf = "";
Expand Down
2 changes: 1 addition & 1 deletion src/include/homestore/index/index_internal.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,11 @@ class IndexTableBase {
public:
virtual ~IndexTableBase() = default;
virtual uuid_t uuid() const = 0;
virtual void recovery_completed() = 0;
virtual uint32_t ordinal() const = 0;
virtual uint64_t used_size() const = 0;
virtual void destroy() = 0;
virtual void repair_node(IndexBufferPtr const& buf) = 0;
virtual void repair_root(IndexBufferPtr const& buf) = 0;
};

enum class index_buf_state_t : uint8_t {
Expand Down
Loading

0 comments on commit ed6bd0d

Please sign in to comment.