Index crash recovery test cases and fixes

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.4.33"
+    version = "6.4.34"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/include/homestore/btree/btree.hpp

@@ -34,6 +34,12 @@ namespace homestore {
 using BtreeNodePtr = boost::intrusive_ptr< BtreeNode >;
 using BtreeNodeList = folly::small_vector< BtreeNodePtr, 3 >;
 
+struct BtreeVisualizeVariables {
+    uint64_t parent;
+    uint64_t midPoint;
+    uint64_t index;
+};
+
 struct BtreeThreadVariables {
     std::vector< btree_locked_node_info > wr_locked_nodes;
     std::vector< btree_locked_node_info > rd_locked_nodes;
@@ -114,8 +120,9 @@ class Btree {
     virtual std::pair< btree_status_t, uint64_t > destroy_btree(void* context);
     nlohmann::json get_status(int log_level) const;
 
-    void print_tree(const std::string& file = "") const;
-    void print_tree_keys() const;
+    void dump_tree_to_file(const std::string& file = "") const;
+    std::string to_custom_string(to_string_cb_t< K, V > const& cb) const;
+    std::string visualize_tree_keys(const std::string& file) const;
     uint64_t count_keys(bnodeid_t bnodeid) const;
 
     nlohmann::json get_metrics_in_json(bool updated = true);
@@ -194,7 +201,9 @@ class Btree {
     uint64_t get_btree_node_cnt() const;
     uint64_t get_child_node_cnt(bnodeid_t bnodeid) const;
     void to_string(bnodeid_t bnodeid, std::string& buf) const;
-    void to_string_keys(bnodeid_t bnodeid, std::string& buf) const;
+    void to_custom_string_internal(bnodeid_t bnodeid, std::string& buf, to_string_cb_t< K, V > const& cb) const;
+    void to_dot_keys(bnodeid_t bnodeid, std::string& buf, std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                     std::map< uint64_t, BtreeVisualizeVariables >& info_map) const;
     void validate_sanity_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void validate_sanity_next_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void print_node(const bnodeid_t& bnodeid) const;

src/include/homestore/btree/btree.ipp

@@ -308,7 +308,7 @@ nlohmann::json Btree< K, V >::get_status(int log_level) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::print_tree(const std::string& file) const {
+void Btree< K, V >::dump_tree_to_file(const std::string& file) const {
     std::string buf;
     m_btree_lock.lock_shared();
     to_string(m_root_node_info.bnode_id(), buf);
@@ -323,13 +323,53 @@ void Btree< K, V >::print_tree(const std::string& file) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::print_tree_keys() const {
+std::string Btree< K, V >::to_custom_string(to_string_cb_t< K, V > const& cb) const {
     std::string buf;
     m_btree_lock.lock_shared();
-    to_string_keys(m_root_node_info.bnode_id(), buf);
+    to_custom_string_internal(m_root_node_info.bnode_id(), buf, cb);
     m_btree_lock.unlock_shared();
 
-    LOGINFO("Pre order traversal of tree:\n<{}>", buf);
+    return buf;
+}
+
+template < typename K, typename V >
+std::string Btree< K, V >::visualize_tree_keys(const std::string& file) const {
+    std::map< uint32_t, std::vector< uint64_t > > level_map;
+    std::map< uint64_t, BtreeVisualizeVariables > info_map;
+    std::string buf = "digraph G\n"
+                      "{ \n"
+                      "ranksep = 3.0;\n"
+                      R"(graph [splines="polyline"];
+)";
+
+    m_btree_lock.lock_shared();
+    to_dot_keys(m_root_node_info.bnode_id(), buf, level_map, info_map);
+    m_btree_lock.unlock_shared();
+    for (const auto& [child, info] : info_map) {
+        if (info.parent) {
+            buf += fmt::format(R"(
+            "{}":connector{} -> "{}":"key{}" [splines=false];)",
+                               info.parent, info.index, child, info.midPoint);
+        }
+    }
+
+    std::string result;
+    for (const auto& [key, values] : level_map) {
+        result += "{rank=same; ";
+        std::vector< std::string > quotedValues;
+        std::transform(values.begin(), values.end(), std::back_inserter(quotedValues),
+                       [](uint64_t value) { return fmt::format("\"{}\"", value); });
+
+        result += fmt::to_string(fmt::join(quotedValues, " ")) + "}\n";
+    }
+
+    buf += "\n" + result + " }\n";
+    if (!file.empty()) {
+        std::ofstream o(file);
+        o.write(buf.c_str(), buf.size());
+        o.flush();
+    }
+    return buf;
 }
 
 template < typename K, typename V >

src/include/homestore/btree/detail/btree_common.ipp

@@ -147,23 +147,54 @@ void Btree< K, V >::to_string(bnodeid_t bnodeid, std::string& buf) const {
 }
 
 template < typename K, typename V >
-void Btree< K, V >::to_string_keys(bnodeid_t bnodeid, std::string& buf) const {
+void Btree< K, V >::to_custom_string_internal(bnodeid_t bnodeid, std::string& buf,
+                                              to_string_cb_t< K, V > const& cb) const {
     BtreeNodePtr node;
 
     locktype_t acq_lock = locktype_t::READ;
 
     if (read_and_lock_node(bnodeid, node, acq_lock, acq_lock, nullptr) != btree_status_t::success) { return; }
-    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_string_keys());
+    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_custom_string(cb));
 
     if (!node->is_leaf()) {
         uint32_t i = 0;
         while (i < node->total_entries()) {
             BtreeLinkInfo p;
             node->get_nth_value(i, &p, false);
-            to_string_keys(p.bnode_id(), buf);
+            to_custom_string_internal(p.bnode_id(), buf, cb);
             ++i;
         }
-        if (node->has_valid_edge()) { to_string_keys(node->edge_id(), buf); }
+        if (node->has_valid_edge()) { to_custom_string_internal(node->edge_id(), buf, cb); }
+    }
+    unlock_node(node, acq_lock);
+}
+
+template < typename K, typename V >
+void Btree< K, V >::to_dot_keys(bnodeid_t bnodeid, std::string& buf,
+                                std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                                std::map< uint64_t, BtreeVisualizeVariables >& info_map) const {
+    BtreeNodePtr node;
+    locktype_t acq_lock = locktype_t::READ;
+
+    if (read_and_lock_node(bnodeid, node, acq_lock, acq_lock, nullptr) != btree_status_t::success) { return; }
+    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_dot_keys());
+    l_map[node->level()].push_back(node->node_id());
+    info_map[node->node_id()].midPoint = node->is_leaf() ? 0 : node->total_entries() / 2;
+    if (!node->is_leaf()) {
+        uint32_t i = 0;
+        while (i < node->total_entries()) {
+            BtreeLinkInfo p;
+            node->get_nth_value(i, &p, false);
+            to_dot_keys(p.bnode_id(), buf, l_map, info_map);
+            info_map[p.bnode_id()].parent = node->node_id();
+            info_map[p.bnode_id()].index = i;
+            ++i;
+        }
+        if (node->has_valid_edge()) {
+            to_dot_keys(node->edge_id(), buf, l_map, info_map);
+            info_map[node->edge_id()].parent = node->node_id();
+            info_map[node->edge_id()].index = node->total_entries();
+        }
     }
     unlock_node(node, acq_lock);
 }

src/include/homestore/btree/detail/btree_internal.hpp

@@ -215,6 +215,9 @@ class BtreeNode;
 void intrusive_ptr_add_ref(BtreeNode* node);
 void intrusive_ptr_release(BtreeNode* node);
 
+template < typename K, typename V >
+using to_string_cb_t = std::function< std::string(std::vector< std::pair< K, V > > const&) >;
+
 ENUM(btree_event_t, uint8_t, READ, MUTATE, REMOVE, SPLIT, REPAIR, MERGE);
 
 struct trace_route_entry {

src/include/homestore/btree/detail/btree_mutate_impl.ipp

@@ -299,6 +299,20 @@ btree_status_t Btree< K, V >::split_node(const BtreeNodePtr& parent_node, const
     return ret;
 }
 
+// template < typename K, typename V >
+// template < typename ReqT >
+// bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
+//     if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
+//         return node->total_entries() >= 3;
+//     } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
+//         return node->total_entries() >= 3;
+//     } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
+//         return node->total_entries() >= 3;;
+//     } else {
+//         return false;
+//     }
+// }
+
 template < typename K, typename V >
 template < typename ReqT >
 bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {

src/include/homestore/btree/detail/btree_node.hpp

@@ -69,17 +69,24 @@ struct persistent_hdr_t {
 
     persistent_hdr_t() : nentries{0}, leaf{0}, node_deleted{0} {}
     std::string to_string() const {
-        return fmt::format("magic={} version={} csum={} node_id={} next_node={} nentries={} node_type={} is_leaf={} "
-                           "node_deleted={} node_gen={} modified_cp_id={} link_version={} edge_nodeid={}, "
-                           "edge_link_version={} level={} ",
-                           magic, version, checksum, node_id, next_node, nentries, node_type, leaf, node_deleted,
-                           node_gen, modified_cp_id, link_version, edge_info.m_bnodeid, edge_info.m_link_version,
-                           level);
+        auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
+            ? ""
+            : fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
+        return fmt::format("magic={} version={} csum={} node_id={}{} nentries={} node_type={} is_leaf={} "
+                           "node_deleted={} node_gen={} modified_cp_id={} link_version={}{} level={} ",
+                           magic, version, checksum, node_id, snext, nentries, node_type, leaf, node_deleted, node_gen,
+                           modified_cp_id, link_version, sedge, level);
     }
 
     std::string to_compact_string() const {
-        return fmt::format("{} id={} next={} nentries={} {} level={}", (void*)this, node_id, next_node, nentries,
-                           (node_deleted == 0x1) ? "Deleted" : "", level);
+        auto snext = (next_node == empty_bnodeid) ? "" : " next=" + std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid)
+            ? ""
+            : fmt::format(" edge={}.{}", edge_info.m_bnodeid, edge_info.m_link_version);
+        return fmt::format("id={}{}{} {} level={} nentries={}{} mod_cp={}", node_id, snext, sedge,
+                           leaf ? "LEAF" : "INTERIOR", level, nentries, (node_deleted == 0x1) ? "  Deleted" : "",
+                           modified_cp_id);
     }
 };
 #pragma pack()
@@ -111,10 +118,10 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
 
     // Identify if a node is a leaf node or not, from raw buffer, by just reading persistent_hdr_t
     static bool identify_leaf_node(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->leaf; }
+    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_compact_string(); }
     static BtreeLinkInfo::bnode_link_info identify_edge_info(uint8_t* buf) {
         return (r_cast< persistent_hdr_t* >(buf))->edge_info;
     }
-    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_string(); }
 
     static bool is_valid_node(sisl::blob const& buf) {
         auto phdr = r_cast< persistent_hdr_t const* >(buf.cbytes());
@@ -347,6 +354,41 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     void lock_acknowledge() { m_trans_hdr.upgraders.decrement(1); }
     bool any_upgrade_waiters() const { return (!m_trans_hdr.upgraders.testz()); }
 
+    template < typename K, typename V >
+    std::string to_custom_string(to_string_cb_t< K, V > const& cb) const {
+        std::string snext =
+            (this->next_bnode() == empty_bnodeid) ? "" : fmt::format(" next_node={}", this->next_bnode());
+        auto str = fmt::format("id={}.{} level={} nEntries={} {}{} node_gen={} ", this->node_id(), this->link_version(),
+                               this->level(), this->total_entries(), (this->is_leaf() ? "LEAF" : "INTERIOR"), snext,
+                               this->node_gen());
+        if (this->has_valid_edge()) {
+            fmt::format_to(std::back_inserter(str), " edge={}.{}", this->edge_info().m_bnodeid,
+                           this->edge_info().m_link_version);
+        }
+
+        if (this->total_entries() == 0) {
+            fmt::format_to(std::back_inserter(str), " [EMPTY] ");
+            return str;
+        } else if (this->is_leaf()) {
+            std::vector< std::pair< K, V > > entries;
+            for (uint32_t i{0}; i < this->total_entries(); ++i) {
+                V v;
+                get_nth_value(i, &v, false);
+                entries.emplace_back(std::make_pair(get_nth_key< K >(i, false), v));
+            }
+            fmt::format_to(std::back_inserter(str), " Keys=[{}]", cb(entries));
+            return str;
+        } else {
+            fmt::format_to(std::back_inserter(str), " Keys=[");
+            for (uint32_t i{0}; i < this->total_entries(); ++i) {
+                fmt::format_to(std::back_inserter(str), "{}{}", get_nth_key< K >(i, false).to_string(),
+                               (i == this->total_entries() - 1) ? "" : ", ");
+            }
+            fmt::format_to(std::back_inserter(str), "]");
+        }
+        return str;
+    }
+
 public:
     // Public method which needs to be implemented by variants
     virtual btree_status_t insert(uint32_t ind, const BtreeKey& key, const BtreeValue& val) = 0;
@@ -384,7 +426,7 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     }
 
     virtual std::string to_string(bool print_friendly = false) const = 0;
-    virtual std::string to_string_keys(bool print_friendly = false) const = 0;
+    virtual std::string to_dot_keys() const = 0;
 
 protected:
     node_find_result_t bsearch_node(const BtreeKey& key) const {

src/include/homestore/btree/detail/btree_remove_impl.ipp

@@ -119,7 +119,7 @@ retry:
                     goto retry;
                 } else if (ret == btree_status_t::merge_not_required) {
                     BT_NODE_LOG(DEBUG, my_node, "merge is not required for child = {} keys: {}", curr_idx,
-                                child_node->to_string_keys());
+                                child_node->to_string());
                 }
             }
         }

src/include/homestore/btree/detail/prefix_node.hpp

@@ -644,7 +644,7 @@ class FixedPrefixNode : public VariantNode< K, V > {
         return str;
     }
 
-    std::string to_string_keys(bool print_friendly = false) const override { return "NOT Supported"; }
+    std::string to_dot_keys() const override { return "NOT Supported"; }
 
 private:
     uint16_t add_prefix(BtreeKey const& key, BtreeValue const& val) {