Index crash recovery test cases and fixes (#458)

* Fixed the case where if cp is not flushed and homestore crashed, restart would not init certain key parameters. This is fixed by taking cp everytime we do formatting and then we persist this information to homestore first_blk. * Fixed the crash simulator issue where restart doesn't initialize existing HSTestHelper instances * Several fixes in IndexTable to make sure we can restart and reapply the logs * Add visualize, change diff and some print * Fixed non-release build duplication of down_buf_count * Removed key specific print into btree and create custom overrider for that --------- Co-authored-by: shosseinimotlagh <[email protected]>
eBay · Aug 2, 2024 · 104e54e · 104e54e
1 parent e940f13
commit 104e54e
Show file tree

Hide file tree

Showing 42 changed files with 1,133 additions and 350 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/conanfile.py b/conanfile.py
@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.4.33"
+    version = "6.4.34"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

diff --git a/src/include/homestore/btree/btree.hpp b/src/include/homestore/btree/btree.hpp
@@ -34,6 +34,12 @@ namespace homestore {
 using BtreeNodePtr = boost::intrusive_ptr< BtreeNode >;
 using BtreeNodeList = folly::small_vector< BtreeNodePtr, 3 >;
 
+struct BtreeVisualizeVariables {
+    uint64_t parent;
+    uint64_t midPoint;
+    uint64_t index;
+};
+
 struct BtreeThreadVariables {
     std::vector< btree_locked_node_info > wr_locked_nodes;
     std::vector< btree_locked_node_info > rd_locked_nodes;
@@ -114,8 +120,9 @@ class Btree {
     virtual std::pair< btree_status_t, uint64_t > destroy_btree(void* context);
     nlohmann::json get_status(int log_level) const;
 
-    void print_tree(const std::string& file = "") const;
-    void print_tree_keys() const;
+    void dump_tree_to_file(const std::string& file = "") const;
+    std::string to_custom_string(to_string_cb_t< K, V > const& cb) const;
+    std::string visualize_tree_keys(const std::string& file) const;
     uint64_t count_keys(bnodeid_t bnodeid) const;
 
     nlohmann::json get_metrics_in_json(bool updated = true);
@@ -194,7 +201,9 @@ class Btree {
     uint64_t get_btree_node_cnt() const;
     uint64_t get_child_node_cnt(bnodeid_t bnodeid) const;
     void to_string(bnodeid_t bnodeid, std::string& buf) const;
-    void to_string_keys(bnodeid_t bnodeid, std::string& buf) const;
+    void to_custom_string_internal(bnodeid_t bnodeid, std::string& buf, to_string_cb_t< K, V > const& cb) const;
+    void to_dot_keys(bnodeid_t bnodeid, std::string& buf, std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                     std::map< uint64_t, BtreeVisualizeVariables >& info_map) const;
     void validate_sanity_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void validate_sanity_next_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void print_node(const bnodeid_t& bnodeid) const;

diff --git a/src/include/homestore/btree/btree.ipp b/src/include/homestore/btree/btree.ipp
@@ -308,7 +308,7 @@ nlohmann::json Btree< K, V >::get_status(int log_level) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::print_tree(const std::string& file) const {
+void Btree< K, V >::dump_tree_to_file(const std::string& file) const {
     std::string buf;
     m_btree_lock.lock_shared();
     to_string(m_root_node_info.bnode_id(), buf);
@@ -323,13 +323,53 @@ void Btree< K, V >::print_tree(const std::string& file) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::print_tree_keys() const {
+std::string Btree< K, V >::to_custom_string(to_string_cb_t< K, V > const& cb) const {
     std::string buf;
     m_btree_lock.lock_shared();
-    to_string_keys(m_root_node_info.bnode_id(), buf);
+    to_custom_string_internal(m_root_node_info.bnode_id(), buf, cb);
     m_btree_lock.unlock_shared();
 
-    LOGINFO("Pre order traversal of tree:\n<{}>", buf);
+    return buf;
+}
+
+template < typename K, typename V >
+std::string Btree< K, V >::visualize_tree_keys(const std::string& file) const {
+    std::map< uint32_t, std::vector< uint64_t > > level_map;
+    std::map< uint64_t, BtreeVisualizeVariables > info_map;
+    std::string buf = "digraph G\n"
+                      "{ \n"
+                      "ranksep = 3.0;\n"
+                      R"(graph [splines="polyline"];
+)";
+
+    m_btree_lock.lock_shared();
+    to_dot_keys(m_root_node_info.bnode_id(), buf, level_map, info_map);
+    m_btree_lock.unlock_shared();
+    for (const auto& [child, info] : info_map) {
+        if (info.parent) {
+            buf += fmt::format(R"(
+            "{}":connector{} -> "{}":"key{}" [splines=false];)",
+                               info.parent, info.index, child, info.midPoint);
+        }
+    }
+
+    std::string result;
+    for (const auto& [key, values] : level_map) {
+        result += "{rank=same; ";
+        std::vector< std::string > quotedValues;
+        std::transform(values.begin(), values.end(), std::back_inserter(quotedValues),
+                       [](uint64_t value) { return fmt::format("\"{}\"", value); });
+
+        result += fmt::to_string(fmt::join(quotedValues, " ")) + "}\n";
+    }
+
+    buf += "\n" + result + " }\n";
+    if (!file.empty()) {
+        std::ofstream o(file);
+        o.write(buf.c_str(), buf.size());
+        o.flush();
+    }
+    return buf;
 }
 
 template < typename K, typename V >

diff --git a/src/include/homestore/btree/detail/btree_common.ipp b/src/include/homestore/btree/detail/btree_common.ipp
@@ -147,23 +147,54 @@ void Btree< K, V >::to_string(bnodeid_t bnodeid, std::string& buf) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::to_string_keys(bnodeid_t bnodeid, std::string& buf) const {
+void Btree< K, V >::to_custom_string_internal(bnodeid_t bnodeid, std::string& buf,
+                                              to_string_cb_t< K, V > const& cb) const {
     BtreeNodePtr node;
 
     locktype_t acq_lock = locktype_t::READ;
 
     if (read_and_lock_node(bnodeid, node, acq_lock, acq_lock, nullptr) != btree_status_t::success) { return; }
-    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_string_keys());
+    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_custom_string(cb));
 
     if (!node->is_leaf()) {
         uint32_t i = 0;
         while (i < node->total_entries()) {
             BtreeLinkInfo p;
             node->get_nth_value(i, &p, false);
-            to_string_keys(p.bnode_id(), buf);
+            to_custom_string_internal(p.bnode_id(), buf, cb);
             ++i;
         }
-        if (node->has_valid_edge()) { to_string_keys(node->edge_id(), buf); }
+        if (node->has_valid_edge()) { to_custom_string_internal(node->edge_id(), buf, cb); }
+    }
+    unlock_node(node, acq_lock);
+}
+
+template < typename K, typename V >
+void Btree< K, V >::to_dot_keys(bnodeid_t bnodeid, std::string& buf,
+                                std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                                std::map< uint64_t, BtreeVisualizeVariables >& info_map) const {
+    BtreeNodePtr node;
+    locktype_t acq_lock = locktype_t::READ;
+
+    if (read_and_lock_node(bnodeid, node, acq_lock, acq_lock, nullptr) != btree_status_t::success) { return; }
+    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_dot_keys());
+    l_map[node->level()].push_back(node->node_id());
+    info_map[node->node_id()].midPoint = node->is_leaf() ? 0 : node->total_entries() / 2;
+    if (!node->is_leaf()) {
+        uint32_t i = 0;
+        while (i < node->total_entries()) {
+            BtreeLinkInfo p;
+            node->get_nth_value(i, &p, false);
+            to_dot_keys(p.bnode_id(), buf, l_map, info_map);
+            info_map[p.bnode_id()].parent = node->node_id();
+            info_map[p.bnode_id()].index = i;
+            ++i;
+        }
+        if (node->has_valid_edge()) {
+            to_dot_keys(node->edge_id(), buf, l_map, info_map);
+            info_map[node->edge_id()].parent = node->node_id();
+            info_map[node->edge_id()].index = node->total_entries();
+        }
     }
     unlock_node(node, acq_lock);
 }

diff --git a/src/include/homestore/btree/detail/btree_internal.hpp b/src/include/homestore/btree/detail/btree_internal.hpp
@@ -215,6 +215,9 @@ class BtreeNode;
 void intrusive_ptr_add_ref(BtreeNode* node);
 void intrusive_ptr_release(BtreeNode* node);
 
+template < typename K, typename V >
+using to_string_cb_t = std::function< std::string(std::vector< std::pair< K, V > > const&) >;
+
 ENUM(btree_event_t, uint8_t, READ, MUTATE, REMOVE, SPLIT, REPAIR, MERGE);
 
 struct trace_route_entry {

diff --git a/src/include/homestore/btree/detail/btree_mutate_impl.ipp b/src/include/homestore/btree/detail/btree_mutate_impl.ipp
@@ -299,6 +299,20 @@ btree_status_t Btree< K, V >::split_node(const BtreeNodePtr& parent_node, const
     return ret;
 }
 
+// template < typename K, typename V >
+// template < typename ReqT >
+// bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
+//     if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
+//         return node->total_entries() >= 3;
+//     } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
+//         return node->total_entries() >= 3;
+//     } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
+//         return node->total_entries() >= 3;;
+//     } else {
+//         return false;
+//     }
+// }
+
 template < typename K, typename V >
 template < typename ReqT >
 bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {

diff --git a/src/include/homestore/btree/detail/btree_node.hpp b/src/include/homestore/btree/detail/btree_node.hpp
@@ -69,17 +69,24 @@ struct persistent_hdr_t {
 
     persistent_hdr_t() : nentries{0}, leaf{0}, node_deleted{0} {}
     std::string to_string() const {
-        return fmt::format("magic={} version={} csum={} node_id={} next_node={} nentries={} node_type={} is_leaf={} "
-                           "node_deleted={} node_gen={} modified_cp_id={} link_version={} edge_nodeid={}, "
-                           "edge_link_version={} level={} ",
-                           magic, version, checksum, node_id, next_node, nentries, node_type, leaf, node_deleted,
-                           node_gen, modified_cp_id, link_version, edge_info.m_bnodeid, edge_info.m_link_version,
-                           level);
+        auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
+            ? ""
+            : fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
+        return fmt::format("magic={} version={} csum={} node_id={}{} nentries={} node_type={} is_leaf={} "
+                           "node_deleted={} node_gen={} modified_cp_id={} link_version={}{} level={} ",
+                           magic, version, checksum, node_id, snext, nentries, node_type, leaf, node_deleted, node_gen,
+                           modified_cp_id, link_version, sedge, level);
     }
 
     std::string to_compact_string() const {
-        return fmt::format("{} id={} next={} nentries={} {} level={}", (void*)this, node_id, next_node, nentries,
-                           (node_deleted == 0x1) ? "Deleted" : "", level);
+        auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
+            ? ""
+            : fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
+        return fmt::format("id={}{}{} {} level={} nentries={}{} mod_cp={}", node_id, snext, sedge,
+                           leaf ? "LEAF" : "INTERIOR", level, nentries, (node_deleted == 0x1) ? "  Deleted" : "",
+                           modified_cp_id);
     }
 };
 #pragma pack()
@@ -111,10 +118,10 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
 
     // Identify if a node is a leaf node or not, from raw buffer, by just reading persistent_hdr_t
     static bool identify_leaf_node(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->leaf; }
+    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_compact_string(); }
     static BtreeLinkInfo::bnode_link_info identify_edge_info(uint8_t* buf) {
         return (r_cast< persistent_hdr_t* >(buf))->edge_info;
     }
-    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_string(); }
 
     static bool is_valid_node(sisl::blob const& buf) {
         auto phdr = r_cast< persistent_hdr_t const* >(buf.cbytes());
@@ -347,6 +354,41 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     void lock_acknowledge() { m_trans_hdr.upgraders.decrement(1); }
     bool any_upgrade_waiters() const { return (!m_trans_hdr.upgraders.testz()); }
 
+    template < typename K, typename V >
+    std::string to_custom_string(to_string_cb_t< K, V > const& cb) const {
+        std::string snext =
+            (this->next_bnode() == empty_bnodeid) ? "" : fmt::format(" next_node={}", this->next_bnode());
+        auto str = fmt::format("id={}.{} level={} nEntries={} {}{} node_gen={} ", this->node_id(), this->link_version(),
+                               this->level(), this->total_entries(), (this->is_leaf() ? "LEAF" : "INTERIOR"), snext,
+                               this->node_gen());
+        if (this->has_valid_edge()) {
+            fmt::format_to(std::back_inserter(str), " edge={}.{}", this->edge_info().m_bnodeid,
+                           this->edge_info().m_link_version);
+        }
+
+        if (this->total_entries() == 0) {
+            fmt::format_to(std::back_inserter(str), " [EMPTY] ");
+            return str;
+        } else if (this->is_leaf()) {
+            std::vector< std::pair< K, V > > entries;
+            for (uint32_t i{0}; i < this->total_entries(); ++i) {
+                V v;
+                get_nth_value(i, &v, false);
+                entries.emplace_back(std::make_pair(get_nth_key< K >(i, false), v));
+            }
+            fmt::format_to(std::back_inserter(str), " Keys=[{}]", cb(entries));
+            return str;
+        } else {
+            fmt::format_to(std::back_inserter(str), " Keys=[");
+            for (uint32_t i{0}; i < this->total_entries(); ++i) {
+                fmt::format_to(std::back_inserter(str), "{}{}", get_nth_key< K >(i, false).to_string(),
+                               (i == this->total_entries() - 1) ? "" : ", ");
+            }
+            fmt::format_to(std::back_inserter(str), "]");
+        }
+        return str;
+    }
+
 public:
     // Public method which needs to be implemented by variants
     virtual btree_status_t insert(uint32_t ind, const BtreeKey& key, const BtreeValue& val) = 0;
@@ -384,7 +426,7 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     }
 
     virtual std::string to_string(bool print_friendly = false) const = 0;
-    virtual std::string to_string_keys(bool print_friendly = false) const = 0;
+    virtual std::string to_dot_keys() const = 0;
 
 protected:
     node_find_result_t bsearch_node(const BtreeKey& key) const {

diff --git a/src/include/homestore/btree/detail/btree_remove_impl.ipp b/src/include/homestore/btree/detail/btree_remove_impl.ipp
@@ -119,7 +119,7 @@ retry:
                     goto retry;
                 } else if (ret == btree_status_t::merge_not_required) {
                     BT_NODE_LOG(DEBUG, my_node, "merge is not required for child = {} keys: {}", curr_idx,
-                                child_node->to_string_keys());
+                                child_node->to_string());
                 }
             }
         }

diff --git a/src/include/homestore/btree/detail/prefix_node.hpp b/src/include/homestore/btree/detail/prefix_node.hpp
@@ -644,7 +644,7 @@ class FixedPrefixNode : public VariantNode< K, V > {
         return str;
     }
 
-    std::string to_string_keys(bool print_friendly = false) const override { return "NOT Supported"; }
+    std::string to_dot_keys() const override { return "NOT Supported"; }
 
 private:
     uint16_t add_prefix(BtreeKey const& key, BtreeValue const& val) {