Some fixes

Add visualize
change diff and some print

src/include/homestore/btree/btree.hpp

@@ -34,6 +34,12 @@ namespace homestore {
 using BtreeNodePtr = boost::intrusive_ptr< BtreeNode >;
 using BtreeNodeList = folly::small_vector< BtreeNodePtr, 3 >;
 
+struct BtreeVisualizeVariables{
+    uint64_t parent;
+    uint64_t midPoint;
+    uint64_t index;
+};
+
 struct BtreeThreadVariables {
     std::vector< btree_locked_node_info > wr_locked_nodes;
     std::vector< btree_locked_node_info > rd_locked_nodes;
@@ -116,6 +122,7 @@ class Btree {
 
     void print_tree(const std::string& file = "") const;
     void print_tree_keys() const;
+    std::string visualize_tree_keys(const std::string& file) const;
     uint64_t count_keys(bnodeid_t bnodeid) const;
 
     nlohmann::json get_metrics_in_json(bool updated = true);
@@ -195,6 +202,9 @@ class Btree {
     uint64_t get_child_node_cnt(bnodeid_t bnodeid) const;
     void to_string(bnodeid_t bnodeid, std::string& buf) const;
     void to_string_keys(bnodeid_t bnodeid, std::string& buf) const;
+    void to_dot_keys(bnodeid_t bnodeid, std::string& buf,
+                     std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                     std::map< uint64_t, BtreeVisualizeVariables >& info_map) const;
     void validate_sanity_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void validate_sanity_next_child(const BtreeNodePtr& parent_node, uint32_t ind) const;
     void print_node(const bnodeid_t& bnodeid) const;

src/include/homestore/btree/btree.ipp

@@ -314,7 +314,7 @@ void Btree< K, V >::print_tree(const std::string& file) const {
     to_string(m_root_node_info.bnode_id(), buf);
     m_btree_lock.unlock_shared();
 
-    BT_LOG(INFO, "Pre order traversal of tree:\n<{}>", buf);
+    LOGINFO( "Pre order traversal of tree:\n<{}>", buf);
     if (!file.empty()) {
         std::ofstream o(file);
         o.write(buf.c_str(), buf.size());
@@ -332,6 +332,46 @@ void Btree< K, V >::print_tree_keys() const {
     LOGINFO("Pre order traversal of tree:\n<{}>", buf);
 }
 
+template < typename K, typename V >
+std::string Btree< K, V >::visualize_tree_keys(const std::string& file) const {
+    std::map< uint32_t, std::vector< uint64_t > > level_map;
+    std::map< uint64_t, BtreeVisualizeVariables > info_map;
+    std::string buf = "digraph G\n"
+                      "{ \n"
+                      "ranksep = 3.0;\n"
+                      R"(graph [splines="polyline"];
+)";
+
+    m_btree_lock.lock_shared();
+    to_dot_keys(m_root_node_info.bnode_id(), buf, level_map, info_map);
+    m_btree_lock.unlock_shared();
+    for (const auto& [child, info] : info_map) {
+        if (info.parent) {
+            buf += fmt::format(R"(
+            "{}":connector{} -> "{}":"key{}" [splines=false];)",
+                               info.parent, info.index, child, info.midPoint);
+        }
+    }
+
+    std::string result;
+    for (const auto& [key, values] : level_map) {
+        result += "{rank=same; ";
+        std::vector< std::string > quotedValues;
+        std::transform(values.begin(), values.end(), std::back_inserter(quotedValues),
+                       [](uint64_t value) { return fmt::format("\"{}\"", value); });
+
+        result += fmt::to_string(fmt::join(quotedValues, " ")) + "}\n";
+    }
+
+    buf += "\n" + result + " }\n";
+    if (!file.empty()) {
+        std::ofstream o(file);
+        o.write(buf.c_str(), buf.size());
+        o.flush();
+    }
+    return buf;
+}
+
 template < typename K, typename V >
 nlohmann::json Btree< K, V >::get_metrics_in_json(bool updated) {
     return m_metrics.get_result_in_json(updated);

src/include/homestore/btree/detail/btree_common.ipp

@@ -168,6 +168,36 @@ void Btree< K, V >::to_string_keys(bnodeid_t bnodeid, std::string& buf) const {
     unlock_node(node, acq_lock);
 }
 
+template < typename K, typename V >
+void Btree< K, V >::to_dot_keys(bnodeid_t bnodeid, std::string& buf,
+                                std::map< uint32_t, std::vector< uint64_t > >& l_map,
+                                std::map< uint64_t, BtreeVisualizeVariables >& info_map) const {
+    BtreeNodePtr node;
+    locktype_t acq_lock = locktype_t::READ;
+
+    if (read_and_lock_node(bnodeid, node, acq_lock, acq_lock, nullptr) != btree_status_t::success) { return; }
+    fmt::format_to(std::back_inserter(buf), "{}\n", node->to_dot_keys());
+    l_map[node->level()].push_back(node->node_id());
+    info_map[node->node_id()].midPoint = node->is_leaf() ? 0 : node->total_entries() / 2;
+    if (!node->is_leaf()) {
+        uint32_t i = 0;
+        while (i < node->total_entries()) {
+            BtreeLinkInfo p;
+            node->get_nth_value(i, &p, false);
+            to_dot_keys(p.bnode_id(), buf, l_map, info_map);
+            info_map[p.bnode_id()].parent = node->node_id();
+            info_map[p.bnode_id()].index = i;
+            ++i;
+        }
+        if (node->has_valid_edge()) {
+            to_dot_keys(node->edge_id(), buf, l_map, info_map);
+            info_map[node->edge_id()].parent = node->node_id();
+            info_map[node->edge_id()].index = node->total_entries();
+        }
+    }
+    unlock_node(node, acq_lock);
+}
+
 template < typename K, typename V >
 uint64_t Btree< K, V >::count_keys(bnodeid_t bnodeid) const {
     BtreeNodePtr node;

src/include/homestore/btree/detail/btree_mutate_impl.ipp

@@ -303,13 +303,24 @@ template < typename K, typename V >
 template < typename ReqT >
 bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
     if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
-        return !node->has_room_for_put(btree_put_type::UPSERT, K::get_max_size(), BtreeLinkInfo::get_fixed_size());
+        return node->total_entries() >= 3;
     } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
-        return !node->has_room_for_put(req.m_put_type, req.first_key_size(), req.m_newval->serialized_size());
+        return node->total_entries() >= 3;
     } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
-        return !node->has_room_for_put(req.m_put_type, req.key().serialized_size(), req.value().serialized_size());
+        return node->total_entries() >= 3;;
     } else {
         return false;
     }
 }
+//bool Btree< K, V >::is_split_needed(const BtreeNodePtr& node, ReqT& req) const {
+//    if (!node->is_leaf()) { // if internal node, size is atmost one additional entry, size of K/V
+//        return !node->has_room_for_put(btree_put_type::UPSERT, K::get_max_size(), BtreeLinkInfo::get_fixed_size());
+//    } else if constexpr (std::is_same_v< ReqT, BtreeRangePutRequest< K > >) {
+//        return !node->has_room_for_put(req.m_put_type, req.first_key_size(), req.m_newval->serialized_size());
+//    } else if constexpr (std::is_same_v< ReqT, BtreeSinglePutRequest >) {
+//        return !node->has_room_for_put(req.m_put_type, req.key().serialized_size(), req.value().serialized_size());
+//    } else {
+//        return false;
+//    }
+//}
 } // namespace homestore

src/include/homestore/btree/detail/btree_node.hpp

@@ -69,17 +69,23 @@ struct persistent_hdr_t {
 
     persistent_hdr_t() : nentries{0}, leaf{0}, node_deleted{0} {}
     std::string to_string() const {
-        return fmt::format("magic={} version={} csum={} node_id={} next_node={} nentries={} node_type={} is_leaf={} "
-                           "node_deleted={} node_gen={} modified_cp_id={} link_version={} edge_nodeid={}, "
-                           "edge_link_version={} level={} ",
-                           magic, version, checksum, node_id, next_node, nentries, node_type, leaf, node_deleted,
-                           node_gen, modified_cp_id, link_version, edge_info.m_bnodeid, edge_info.m_link_version,
+        auto snext = (next_node == empty_bnodeid) ? "" : "next_node="+ std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_nodeid="+ std::to_string(edge_info.m_bnodeid);
+        auto sedgelink = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_link_version="+ std::to_string(edge_info.m_link_version);
+        return fmt::format("magic={} version={} csum={} node_id={} {} nentries={} node_type={} is_leaf={} "
+                           "node_deleted={} node_gen={} modified_cp_id={} link_version={} {}, "
+                           "{} level={} ",
+                           magic, version, checksum, node_id, snext, nentries, node_type, leaf, node_deleted,
+                           node_gen, modified_cp_id, link_version, sedge, sedgelink,
                            level);
     }
 
     std::string to_compact_string() const {
-        return fmt::format("{} id={} next={} nentries={} {} level={}", (void*)this, node_id, next_node, nentries,
-                           (node_deleted == 0x1) ? "Deleted" : "", level);
+        auto snext = (next_node == empty_bnodeid) ? "" : "next="+ std::to_string(next_node);
+        auto sedge = (edge_info.m_bnodeid == empty_bnodeid) ? "" : "edge_nodeid="+ std::to_string(edge_info.m_bnodeid);
+        auto sleaf = leaf?"LEAF": "INTERIOR";
+        return fmt::format(" id={} {}{} {} nentries={} {} level={} modified_cp_id={}", node_id, snext,sedge, sleaf, nentries,
+                           (node_deleted == 0x1) ? "Deleted" : "", level, modified_cp_id);
     }
 };
 #pragma pack()
@@ -111,10 +117,10 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
 
     // Identify if a node is a leaf node or not, from raw buffer, by just reading persistent_hdr_t
     static bool identify_leaf_node(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->leaf; }
+    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_compact_string(); }
     static BtreeLinkInfo::bnode_link_info identify_edge_info(uint8_t* buf) {
         return (r_cast< persistent_hdr_t* >(buf))->edge_info;
     }
-    static std::string to_string_buf(uint8_t* buf) { return (r_cast< persistent_hdr_t* >(buf))->to_string(); }
 
     static bool is_valid_node(sisl::blob const& buf) {
         auto phdr = r_cast< persistent_hdr_t const* >(buf.cbytes());
@@ -385,6 +391,7 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
 
     virtual std::string to_string(bool print_friendly = false) const = 0;
     virtual std::string to_string_keys(bool print_friendly = false) const = 0;
+    virtual std::string to_dot_keys() const = 0;
 
 protected:
     node_find_result_t bsearch_node(const BtreeKey& key) const {

src/include/homestore/btree/detail/prefix_node.hpp

@@ -645,6 +645,7 @@ class FixedPrefixNode : public VariantNode< K, V > {
     }
 
     std::string to_string_keys(bool print_friendly = false) const override { return "NOT Supported"; }
+    std::string to_dot_keys() const override { return "NOT Supported"; }
 
 private:
     uint16_t add_prefix(BtreeKey const& key, BtreeValue const& val) {

src/include/homestore/btree/detail/simple_node.hpp

@@ -207,13 +207,16 @@ class SimpleNode : public VariantNode< K, V > {
     }
 
     std::string to_string(bool print_friendly = false) const override {
-        auto str = fmt::format("{}id={} level={} nEntries={} {} next_node={} ",
+        auto snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
+        auto str = fmt::format("{}id={} level={} nEntries={} {} {} ",
                                (print_friendly ? "------------------------------------------------------------\n" : ""),
                                this->node_id(), this->level(), this->total_entries(),
-                               (this->is_leaf() ? "LEAF" : "INTERIOR"), this->next_bnode());
+                               (this->is_leaf() ? "LEAF" : "INTERIOR"), snext);
         if (!this->is_leaf() && (this->has_valid_edge())) {
-            fmt::format_to(std::back_inserter(str), "edge_id={}.{}", this->edge_info().m_bnodeid,
-                           this->edge_info().m_link_version);
+            auto sedge = this->has_valid_edge() ? "edge:" + std::to_string(this->edge_info().m_bnodeid) + "." +
+                                                      std::to_string(this->edge_info().m_link_version)
+                                                : "";
+            fmt::format_to(std::back_inserter(str), "{}", sedge);
         }
 
         for (uint32_t i{0}; i < this->total_entries(); ++i) {
@@ -226,8 +229,8 @@ class SimpleNode : public VariantNode< K, V > {
 
     std::string to_string_keys(bool print_friendly = false) const override {
         // FIXME: Implement this, key may not be a unit32_t
-        return "";
-#if 0
+//        return "";
+//#if 0
             std::string delimiter = print_friendly ? "\n" : "\t";
             std::string snext = this->next_bnode() == empty_bnodeid ? "" : fmt::format("next_node={}", this->next_bnode());
             auto str = fmt::format("{}{}.{} level:{} nEntries={} {} {} node_gen={} ",
@@ -285,7 +288,93 @@ class SimpleNode : public VariantNode< K, V > {
                 fmt::format_to(std::back_inserter(str), "-{}]", cur_key);
             }
             return str;
-#endif
+//#endif
+    }
+
+    std::string to_dot_keys() const override {
+        std::string str;
+        std::string snext = this->next_bnode() == empty_bnodeid? "": fmt::format("next_node={}", this->next_bnode());
+        str += fmt::format(R"("{}" [
+                shape = none,
+                labelloc="c",
+                fontsize=25,
+                label = <<table border="1" cellborder="0" cellspacing="0">
+                <tr>)",
+                           this->node_id());
+        if (this->total_entries() == 0) {
+            return str + fmt::format(R"(
+                <td port="connector0"></td><td port="key0">E</td>
+                <td port="connector1"></td></tr></table>>])");
+        }
+
+        if (!this->is_leaf()) {
+            //            str += " <tr>";
+            for (uint32_t i{0}; i < this->total_entries(); ++i) {
+                uint32_t cur_key = get_nth_key< K >(i, false).key();
+                BtreeLinkInfo child_info;
+                get_nth_value(i, &child_info, false /* copy */);
+                str += fmt::format(R"(
+                <td port="connector{}"></td><td port="key{}">{}.{}</td>)",
+                                   i, i, cur_key, child_info.link_version());
+            }
+            std::string sedge = this->has_valid_edge() ? "edge:" + std::to_string( this->edge_info().m_bnodeid) + "." + std::to_string( this->edge_info().m_link_version):"";
+            str += fmt::format(R"(
+                <td port="connector{}"></td>
+                <td>{}.{}<br/> gen={}<br/>{} {} </td></tr></table>>];)",this->total_entries(),this->node_id(), this->link_version(), this->node_gen(), snext, sedge );
+
+
+        } else {
+            std::string keys_buf = "";
+            uint32_t prev_key = get_nth_key< K >(0, false).key();
+            uint32_t cur_key = prev_key;
+            uint32_t last_key = get_nth_key< K >(this->total_entries() - 1, false).key();
+            if (last_key - prev_key == this->total_entries() - 1) {
+                if (this->total_entries() == 1) {
+                    keys_buf += fmt::format(R"(
+                       <td port="connector{}"></td><td port="key{}">{}</td>)",
+                                            0, 0, cur_key);
+                } else {
+                    keys_buf += fmt::format(R"(
+                        <td port="connector{}"></td><td port="key{}">{}-{}</td>)",
+                                            0, 0, prev_key, last_key);
+                }
+                keys_buf += fmt::format(R"(
+                        <td port="connector{}"></td>
+                        <td>{}.{}<br/>gen={}<br/> {}</td>
+                        </tr></table>>];)",
+                                        1, this->node_id(), this->link_version(), this->node_gen(), snext);
+                return str + keys_buf;
+            }
+
+            keys_buf += fmt::format(R"(
+                        "<td port="connector{}"></td><td port="key{}">{})",
+                                    0, 0, prev_key);
+            uint32_t start_interval_key = prev_key;
+            for (uint32_t i{1}; i < this->total_entries(); ++i) {
+                cur_key = get_nth_key< K >(i, false).key();
+                if (cur_key != prev_key + 1) {
+                    if (start_interval_key == prev_key) {
+                        keys_buf += fmt::format(" {}", cur_key);
+                    } else {
+                        keys_buf += fmt::format("-{} {}", prev_key, cur_key);
+                    }
+                    start_interval_key = cur_key;
+                }
+                prev_key = cur_key;
+            }
+
+            if (start_interval_key == prev_key) {
+                keys_buf += fmt::format("</td>");
+            } else {
+                keys_buf += fmt::format(" {}</td>", cur_key);
+            }
+            keys_buf += fmt::format(R"(
+                        <td port="connector{}"></td>
+                        <td>{}.{}<br/>gen={} <br/>{}</td></tr></table>>];)",
+                                    1, this->node_id(), this->link_version(), this->node_gen(), snext);
+            return str + keys_buf;
+        }
+        return str;
     }
 
 #ifndef NDEBUG

src/include/homestore/btree/detail/varlen_node.hpp

@@ -567,6 +567,7 @@ class VariableNode : public VariantNode< K, V > {
 #endif
         return {};
     }
+    std::string to_dot_keys() const override { return "NOT Supported"; }
 
     /*int compare_nth_key_range(const BtreeKeyRange& range, uint32_t ind) const {
         return get_nth_key(ind, false).compare_range(range);

src/include/homestore/index/index_internal.hpp

@@ -122,6 +122,7 @@ struct IndexBuffer : public sisl::ObjLifeCounter< IndexBuffer > {
     bool is_meta_buf() const { return m_is_meta_buf; }
 
     std::string to_string() const;
+    std::string to_string_dot() const;
 };
 
 // This is a special buffer which is used to write to the meta block