feat: increase tree leaf log2 size to 5

src/Makefile

@@ -66,7 +66,7 @@ CXX=clang++
 AR=libtool -static -o
 INCS=
 
-BREW_PREFIX=$(shell brew --prefix)
+BREW_PREFIX=$(shell which brew)
 PORT_PREFIX=$(shell which port)
 
 ifeq ($(MACOSX_DEPLOYMENT_TARGET),)

src/access-log.h

@@ -46,10 +46,17 @@ static inline void set_word_access_data(uint64_t w, access_data &ad) {
     ad.insert(ad.end(), p, p + sizeof(w));
 }
 
-static inline uint64_t get_word_access_data(const access_data &ad) {
-    assert(ad.size() == 8);
+static inline void replace_word_access_data(uint64_t w, access_data &ad, int offset = 0) {
+    assert(ad.size() >= offset + sizeof(uint64_t));
+    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
+    auto *p = reinterpret_cast<uint8_t *>(&w);
+    memcpy(ad.data() + offset, p, sizeof(w));
+}
+
+static inline uint64_t get_word_access_data(const access_data &ad, int offset = 0) {
+    assert(ad.size() >= offset + sizeof(uint64_t));
     uint64_t w = 0;
-    memcpy(&w, ad.data(), sizeof(w));
+    memcpy(&w, ad.data() + offset, sizeof(w));
     return w;
 }
 
@@ -165,18 +172,29 @@ class access {
     /// \param root_hash Hash to be used as the root of the proof.
     /// \return The corresponding proof
     proof_type make_proof(const hash_type root_hash) const {
+        // the access can be of data smaller than the merkle tree word size
+        // however, the proof must be at least as big as the merkle tree word size
+        const int proof_log2_size = std::max(m_log2_size, machine_merkle_tree::get_log2_word_size());
+        // the proof address is the access address aligned to the merkle tree word size
+        const uint64_t proof_address = m_address & ~(machine_merkle_tree::get_word_size() - 1);
         if (!m_sibling_hashes.has_value()) {
             throw std::runtime_error("can't make proof if access doesn't have sibling hashes");
         }
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
         const auto &sibling_hashes = m_sibling_hashes.value();
-        const int log2_root_size = m_log2_size + static_cast<int>(sibling_hashes.size());
-        proof_type proof(log2_root_size, m_log2_size);
+        const int log2_root_size = proof_log2_size + static_cast<int>(sibling_hashes.size());
+        if (m_read.has_value() && m_read.value().size() != (static_cast<uint64_t>(1) << proof_log2_size)) {
+            throw std::runtime_error("access read data size is inconsistent with proof size");
+        }
+        if (m_written.has_value() && m_written.value().size() != (static_cast<uint64_t>(1) << proof_log2_size)) {
+            throw std::runtime_error("access written data size is inconsistent with proof size");
+        }
+        proof_type proof(log2_root_size, proof_log2_size);
         proof.set_root_hash(root_hash);
-        proof.set_target_address(m_address);
+        proof.set_target_address(proof_address);
         proof.set_target_hash(m_read_hash);
-        for (int log2_size = m_log2_size; log2_size < log2_root_size; log2_size++) {
-            proof.set_sibling_hash(sibling_hashes[log2_size - m_log2_size], log2_size);
+        for (int log2_size = proof_log2_size; log2_size < log2_root_size; log2_size++) {
+            proof.set_sibling_hash(sibling_hashes[log2_size - proof_log2_size], log2_size);
         }
         return proof;
     }

src/cartesi/util.lua

@@ -14,6 +14,8 @@
 -- with this program (see COPYING). If not, see <https://www.gnu.org/licenses/>.
 --
 
+local cartesi = require("cartesi")
+
 local _M = {}
 
 local function indentout(f, indent, fmt, ...) f:write(string.rep("  ", indent), string.format(fmt, ...)) end
@@ -197,8 +199,16 @@ end
 
 local function hexhash8(hash) return string.sub(hexhash(hash), 1, 8) end
 
-local function accessdatastring(data, data_hash, log2_size)
-    if log2_size == 3 then
+local function accessdatastring(data, data_hash, data_log2_size, address)
+    local data_size = 1 << data_log2_size
+    if data_log2_size == 3 then
+        if data_size < #data then
+            -- access data is  smaller than the tree leaf size
+            -- the logged data is the entire tree leaf, but we only need the data that was accessed
+            local leaf_aligned_addrss = address & ~((1 << cartesi.TREE_LOG2_WORD_SIZE) - 1)
+            local word_offset = address - leaf_aligned_addrss
+            data = data:sub(word_offset + 1, word_offset + data_size)
+        end
         data = string.unpack("<I8", data)
         return string.format("0x%x(%u)", data, data)
     else
@@ -209,7 +219,7 @@ local function accessdatastring(data, data_hash, log2_size)
             data_snippet = data_snippet
                 .. string.format("%s...%s", hexstring(data:sub(1, 3)), hexstring(data:sub(-3, -1)))
         end
-        return string.format("%s(2^%d bytes)", data_snippet, log2_size)
+        return string.format("%s(2^%d bytes)", data_snippet, data_log2_size)
     end
 end
 
@@ -237,12 +247,12 @@ function _M.dump_log(log, out)
             j = j + 1
         -- Otherwise, output access
         elseif ai then
-            local read = accessdatastring(ai.read, ai.read_hash, ai.log2_size)
+            local read = accessdatastring(ai.read, ai.read_hash, ai.log2_size, ai.address)
             if ai.type == "read" then
                 indentout(out, indent, "%d: read %s@0x%x(%u): %s\n", i, notes[i] or "", ai.address, ai.address, read)
             else
                 assert(ai.type == "write", "unknown access type")
-                local written = accessdatastring(ai.written, ai.written_hash, ai.log2_size)
+                local written = accessdatastring(ai.written, ai.written_hash, ai.log2_size, ai.address)
                 indentout(
                     out,
                     indent,

src/clua-cartesi.cpp

@@ -114,6 +114,8 @@ CM_API int luaopen_cartesi(lua_State *L) {
     luaL_setfuncs(L, cartesi_mod.data(), 1); // cluactx cartesi
 
     // Set cartesi constants
+
+    clua_setintegerfield(L, CM_TREE_LOG2_WORD_SIZE, "TREE_LOG2_WORD_SIZE", -1);
     clua_setintegerfield(L, CM_BREAK_REASON_FAILED, "BREAK_REASON_FAILED", -1);
     clua_setintegerfield(L, CM_BREAK_REASON_HALTED, "BREAK_REASON_HALTED", -1);
     clua_setintegerfield(L, CM_BREAK_REASON_YIELDED_MANUALLY, "BREAK_REASON_YIELDED_MANUALLY", -1);

src/clua-machine-util.cpp

@@ -20,6 +20,7 @@
 #include <unordered_map>
 
 #include "clua.h"
+#include "machine-c-defines.h"
 #include "riscv-constants.h"
 
 namespace cartesi {
@@ -349,6 +350,9 @@ static void check_sibling_cm_hashes(lua_State *L, int idx, size_t log2_target_si
     cm_hash_array *sibling_hashes) {
     luaL_checktype(L, idx, LUA_TTABLE);
     memset(sibling_hashes, 0, sizeof(cm_hash_array));
+    if (log2_target_size > log2_root_size) {
+        luaL_error(L, "target size cannot be greater than root size");
+    }
     const size_t sibling_hashes_count = log2_root_size - log2_target_size;
     if (sibling_hashes_count > 64) {
         luaL_error(L, "too many sibling hashes (expected max %d, got %d)", 64, static_cast<int>(sibling_hashes_count));
@@ -433,19 +437,14 @@ static void check_cm_access(lua_State *L, int tabidx, bool proofs, cm_access *a,
     a->type = check_cm_access_type_field(L, tabidx, "type");
     a->address = check_uint_field(L, tabidx, "address");
     a->log2_size = check_int_field(L, tabidx, "log2_size");
-    if (a->log2_size < CM_TREE_LOG2_WORD_SIZE || a->log2_size > CM_TREE_LOG2_ROOT_SIZE) {
-        luaL_error(L, "invalid log2_size (expected integer in {%d..%d})", CM_TREE_LOG2_WORD_SIZE,
-            CM_TREE_LOG2_ROOT_SIZE);
-    }
-
+    const int expected_data_log2_size = std::max(a->log2_size, CM_TREE_LOG2_WORD_SIZE);
     if (opt_table_field(L, tabidx, "sibling_hashes")) {
         a->sibling_hashes = new cm_hash_array{};
-        check_sibling_cm_hashes(L, -1, a->log2_size, CM_TREE_LOG2_ROOT_SIZE, a->sibling_hashes);
+        check_sibling_cm_hashes(L, -1, expected_data_log2_size, CM_TREE_LOG2_ROOT_SIZE, a->sibling_hashes);
         lua_pop(L, 1);
     } else if (proofs) {
         luaL_error(L, "missing sibling_hashes");
     }
-
     lua_getfield(L, tabidx, "read_hash");
     clua_check_cm_hash(L, -1, &a->read_hash);
     lua_pop(L, 1);
@@ -454,8 +453,8 @@ static void check_cm_access(lua_State *L, int tabidx, bool proofs, cm_access *a,
         clua_check_cm_hash(L, -1, &a->written_hash);
         lua_pop(L, 1);
     }
-    a->read_data = opt_cm_access_data_field(L, tabidx, "read", a->log2_size, &a->read_data_size);
-    a->written_data = opt_cm_access_data_field(L, tabidx, "written", a->log2_size, &a->written_data_size);
+    a->read_data = opt_cm_access_data_field(L, tabidx, "read", expected_data_log2_size, &a->read_data_size);
+    a->written_data = opt_cm_access_data_field(L, tabidx, "written", expected_data_log2_size, &a->written_data_size);
 }
 
 cm_access_log *clua_check_cm_access_log(lua_State *L, int tabidx, int ctxidx) {
@@ -662,9 +661,10 @@ void clua_push_cm_access_log(lua_State *L, const cm_access_log *log) {
         }
         if (log->log_type.proofs && a->sibling_hashes != nullptr) {
             lua_newtable(L);
-            for (size_t log2_size = a->log2_size; log2_size < CM_TREE_LOG2_ROOT_SIZE; log2_size++) {
-                clua_push_cm_hash(L, &a->sibling_hashes->entry[log2_size - a->log2_size]);
-                lua_rawseti(L, -2, static_cast<lua_Integer>(log2_size - a->log2_size) + 1);
+            const int proof_log2_size = std::max(a->log2_size, CM_TREE_LOG2_WORD_SIZE);
+            for (size_t log2_size = proof_log2_size; log2_size < CM_TREE_LOG2_ROOT_SIZE; log2_size++) {
+                clua_push_cm_hash(L, &a->sibling_hashes->entry[log2_size - proof_log2_size]);
+                lua_rawseti(L, -2, static_cast<lua_Integer>(log2_size - proof_log2_size) + 1);
             }
             lua_setfield(L, -2, "sibling_hashes");
         }

src/json-util.cpp

@@ -21,6 +21,7 @@
 #include <unordered_map>
 
 #include "base64.h"
+#include "machine-merkle-tree.h"
 
 namespace cartesi {
 
@@ -649,6 +650,9 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con
         throw std::domain_error("field \""s + new_path + "log2_size\" is out of bounds");
     }
     access.set_log2_size(static_cast<int>(log2_size));
+    // Minimum logged data size is merkle tree word size
+    const uint64_t data_log2_size =
+        std::max(log2_size, static_cast<uint64_t>(machine_merkle_tree::get_log2_word_size()));
     uint64_t address = 0;
     ju_get_field(jk, "address"s, address, new_path);
     access.set_address(address);
@@ -665,7 +669,7 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con
     std::optional<access_data> read;
     ju_get_opt_field(jk, "read"s, read, new_path);
     if (read.has_value()) {
-        if (read.value().size() != (UINT64_C(1) << log2_size)) {
+        if (read.value().size() != (UINT64_C(1) << data_log2_size)) {
             throw std::invalid_argument("field \""s + new_path + "written\" has wrong length");
         }
         access.set_read(std::move(read.value()));
@@ -674,20 +678,18 @@ void ju_get_opt_field(const nlohmann::json &j, const K &key, access &access, con
         std::optional<access_data> written;
         ju_get_opt_field(jk, "written"s, written, new_path);
         if (written.has_value()) {
-            if (written.value().size() != (UINT64_C(1) << log2_size)) {
+            if (written.value().size() != (UINT64_C(1) << data_log2_size)) {
                 throw std::invalid_argument("field \""s + new_path + "written\" has wrong length");
             }
             access.set_written(std::move(written.value()));
         }
     }
-    not_default_constructible<machine_merkle_tree::proof_type> proof;
-    ju_get_opt_field(jk, "proof"s, proof, new_path);
     if (contains(jk, "sibling_hashes")) {
         access.get_sibling_hashes().emplace();
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
         auto &sibling_hashes = access.get_sibling_hashes().value();
         ju_get_vector_like_field(jk, "sibling_hashes"s, sibling_hashes, new_path);
-        auto expected_depth = static_cast<size_t>(machine_merkle_tree::get_log2_root_size() - access.get_log2_size());
+        auto expected_depth = static_cast<size_t>(machine_merkle_tree::get_log2_root_size() - data_log2_size);
         if (sibling_hashes.size() != expected_depth) {
             throw std::invalid_argument("field \""s + new_path + "sibling_hashes\" has wrong length");
         }
@@ -1208,7 +1210,9 @@ void to_json(nlohmann::json &j, const access &a) {
     if (a.get_sibling_hashes().has_value()) {
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access)
         const auto &sibling_hashes = a.get_sibling_hashes().value();
-        auto depth = machine_merkle_tree::get_log2_root_size() - a.get_log2_size();
+        // Minimum logged data size is merkle tree word size
+        auto data_log2_size = std::max(a.get_log2_size(), machine_merkle_tree::get_log2_word_size());
+        auto depth = machine_merkle_tree::get_log2_root_size() - data_log2_size;
         nlohmann::json s = nlohmann::json::array();
         for (int i = 0; i < depth; i++) {
             s.push_back(encode_base64(sibling_hashes[i]));

src/machine-c-defines.h

@@ -27,7 +27,7 @@
 #define CM_MACHINE_F_REG_COUNT 32       // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
 #define CM_MACHINE_UARCH_X_REG_COUNT 32 // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
 
-#define CM_TREE_LOG2_WORD_SIZE 3              // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
+#define CM_TREE_LOG2_WORD_SIZE 5              // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
 #define CM_TREE_LOG2_PAGE_SIZE 12             // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
 #define CM_TREE_LOG2_ROOT_SIZE 64             // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)
 #define CM_FLASH_DRIVE_CONFIGS_MAX_SIZE 8     // NOLINT(cppcoreguidelines-macro-usage, modernize-macro-to-enum)

src/machine-merkle-tree.h

@@ -71,7 +71,7 @@ class machine_merkle_tree final {
     /// \brief LOG2_WORD_SIZE Number of bits covered by a word.
     /// I.e., log<sub>2</sub> of number of bytes subintended by the
     /// the deepest tree nodes.
-    static constexpr int LOG2_WORD_SIZE = 3;
+    static constexpr int LOG2_WORD_SIZE = 5;
     /// \brief DEPTH Depth of Merkle tree.
     static constexpr int DEPTH = LOG2_ROOT_SIZE - LOG2_WORD_SIZE;