From 7f2d3fde3ffdd5ba35b1dc68ed930be3a991c193 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 14 Nov 2024 17:54:24 +0200 Subject: [PATCH 1/6] use goroutines manager for trie commit --- trie/branchNode.go | 94 ++++++++++++++++++++++-------- trie/branchNode_test.go | 69 +++++++++------------- trie/export_test.go | 21 ------- trie/extensionNode.go | 57 +++++++++++------- trie/extensionNode_test.go | 100 +++++++++++++++----------------- trie/interface.go | 2 +- trie/leafNode.go | 28 ++++++--- trie/leafNode_test.go | 43 +++++--------- trie/node_test.go | 6 +- trie/patriciaMerkleTrie.go | 32 +++++----- trie/patriciaMerkleTrie_test.go | 86 +++++++++++++-------------- 11 files changed, 278 insertions(+), 260 deletions(-) diff --git a/trie/branchNode.go b/trie/branchNode.go index 7c4d4261bd..4d6bda9904 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -107,14 +107,14 @@ func (bn *branchNode) setHash(goRoutinesManager common.TrieGoroutinesManager) { return } - encChild, err := encodeNodeAndGetHash(bn.children[i]) + childHash, err := encodeNodeAndGetHash(bn.children[i]) if err != nil { goRoutinesManager.SetError(err) return } bn.childrenMutexes[i].Lock() - bn.EncodedChildren[i] = encChild + bn.EncodedChildren[i] = childHash bn.childrenMutexes[i].Unlock() continue } @@ -126,14 +126,14 @@ func (bn *branchNode) setHash(goRoutinesManager common.TrieGoroutinesManager) { waitGroup.Done() return } - encChild, err := encodeNodeAndGetHash(bn.children[childPos]) + childHash, err := encodeNodeAndGetHash(bn.children[childPos]) if err != nil { goRoutinesManager.SetError(err) waitGroup.Done() return } bn.childrenMutexes[childPos].Lock() - bn.EncodedChildren[childPos] = encChild + bn.EncodedChildren[childPos] = childHash bn.childrenMutexes[childPos].Unlock() waitGroup.Done() }(i) @@ -178,44 +178,90 @@ func (bn *branchNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(bn) } -func (bn *branchNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { +func (bn *branchNode) commitDirty( + level byte, + maxTrieLevelInMemory uint, + goRoutinesManager common.TrieGoroutinesManager, + originDb common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { level++ - err := bn.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } if !bn.dirty { - return nil + return } - for i := range bn.children { - if bn.children[i] == nil { + waitGroup := sync.WaitGroup{} + + for i := 0; i < nrOfChildren; i++ { + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + bn.childrenMutexes[i].RLock() + child := bn.children[i] + bn.childrenMutexes[i].RUnlock() + + if child == nil { continue } - err = bn.children[i].commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) - if err != nil { - return err + if !goRoutinesManager.CanStartGoRoutine() { + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + bn.childrenMutexes[i].Lock() + bn.EncodedChildren[i] = child.getHash() + bn.childrenMutexes[i].Unlock() + + continue } + + waitGroup.Add(1) + go func(childPos int) { + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + waitGroup.Done() + return + } + + bn.childrenMutexes[childPos].Lock() + bn.EncodedChildren[childPos] = child.getHash() + bn.childrenMutexes[childPos].Unlock() + + waitGroup.Done() + }(i) + } + + waitGroup.Wait() + bn.dirty = false - _, err = encodeNodeAndCommitToDB(bn, targetDb) + encNode, err := bn.getEncodedNode() if err != nil { - return err + goRoutinesManager.SetError(err) + return + } + hash := bn.hasher.Compute(string(encNode)) + bn.hash = hash + + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + return } + if uint(level) == maxTrieLevelInMemory { log.Trace("collapse branch node on commit") - var collapsedBn *branchNode - collapsedBn, err = bn.getCollapsedBn() - if err != nil { - return err + for i := range bn.children { + bn.childrenMutexes[i].Lock() + bn.children[i] = nil + bn.childrenMutexes[i].Unlock() } - - *bn = *collapsedBn } - return nil } func (bn *branchNode) commitSnapshot( diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index d6ed4341de..7687551a07 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "errors" - "github.com/multiversx/mx-chain-core-go/core/atomic" "testing" "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/atomic" "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -268,8 +268,9 @@ func TestBranchNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedBn) bn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, marsh, hasher) @@ -278,24 +279,6 @@ func TestBranchNode_commit(t *testing.T) { assert.Equal(t, h1, h2) } -func TestBranchNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - bn := emptyDirtyBranchNode() - - err := bn.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyBranchNode)) -} - -func TestBranchNode_commitNilNode(t *testing.T) { - t.Parallel() - - var bn *branchNode - - err := bn.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilBranchNode)) -} - func TestBranchNode_getEncodedNode(t *testing.T) { t.Parallel() @@ -337,7 +320,7 @@ func TestBranchNode_resolveIfCollapsed(t *testing.T) { childPos := byte(2) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) resolved, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) resolved.dirty = false resolved.hash = bn.EncodedChildren[childPos] @@ -426,7 +409,7 @@ func TestBranchNode_tryGetCollapsedNode(t *testing.T) { bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -538,7 +521,7 @@ func TestBranchNode_insertCollapsedNode(t *testing.T) { key := append([]byte{childPos}, []byte("dog")...) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) @@ -562,7 +545,7 @@ func TestBranchNode_insertInStoredBnOnExistingPos(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(key, db) lnHash := ln.getHash() @@ -588,7 +571,7 @@ func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) { nilChildPos := byte(11) key := append([]byte{nilChildPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) bnHash := bn.getHash() expectedHashes := [][]byte{bnHash} @@ -675,7 +658,7 @@ func TestBranchNode_deleteFromStoredBn(t *testing.T) { childPos := byte(2) lnKey := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(lnKey, db) lnHash := ln.getHash() @@ -751,7 +734,7 @@ func TestBranchNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -1101,8 +1084,9 @@ func TestBranchNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testin bn.setHash(getTestGoroutinesManager()) collapsedBn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 1, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedBn.EncodedChildren, bn.EncodedChildren) assert.Equal(t, collapsedBn.children, bn.children) @@ -1148,7 +1132,7 @@ func TestBranchNode_getDirtyHashesFromCleanNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) dirtyHashes := make(common.ModifiedHashes) err := bn.getDirtyHashes(dirtyHashes) @@ -1566,8 +1550,9 @@ func TestBranchNode_insertOnNilChild(t *testing.T) { db := testscommon.NewMemDbMock() bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() assert.True(t, len(originalHash) > 0) @@ -1671,8 +1656,9 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { Version: core.AutoBalanceEnabled, }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() assert.True(t, len(originalHash) > 0) @@ -1681,7 +1667,6 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, _ := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) - assert.Nil(t, err) newNode, modifiedHashes := bn.insertOnChild(newData, int(childPos), goRoutinesManager, db) assert.Nil(t, goRoutinesManager.GetError()) @@ -1712,8 +1697,9 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { Version: core.NotSpecified, }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -1760,8 +1746,9 @@ func TestBranchNode_insertBatch(t *testing.T) { Value: []byte("value4"), }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -1800,7 +1787,7 @@ func getNewBn() *branchNode { bn, _ := newBranchNode(marsh, hasher) bn.children = children bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + bn.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) return bn } diff --git a/trie/export_test.go b/trie/export_test.go index ea22af5917..39208dbf05 100644 --- a/trie/export_test.go +++ b/trie/export_test.go @@ -1,8 +1,6 @@ package trie import ( - "time" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/statistics" @@ -34,25 +32,6 @@ func (ts *trieSyncer) trieNodeIntercepted(hash []byte, val interface{}) { } } -// WaitForOperationToComplete - -func WaitForOperationToComplete(tsm common.StorageManager) { - for tsm.IsPruningBlocked() { - time.Sleep(10 * time.Millisecond) - } -} - -// CreateSmallTestTrieAndStorageManager - -func CreateSmallTestTrieAndStorageManager() (*patriciaMerkleTrie, *trieStorageManager) { - tr, trieStorage := newEmptyTrie() - _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Update([]byte("dogglesworth"), []byte("cat")) - - _ = tr.Commit() - - return tr, trieStorage -} - // WriteInChanNonBlocking - func WriteInChanNonBlocking(errChan chan error, err error) { select { diff --git a/trie/extensionNode.go b/trie/extensionNode.go index 8d7a4b14e2..f52aa8ef34 100644 --- a/trie/extensionNode.go +++ b/trie/extensionNode.go @@ -127,41 +127,58 @@ func (en *extensionNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(en) } -func (en *extensionNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { +func (en *extensionNode) commitDirty( + level byte, + maxTrieLevelInMemory uint, + goRoutinesManager common.TrieGoroutinesManager, + originDb common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { level++ - err := en.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } if !en.dirty { - return nil + return } - if en.child != nil { - err = en.child.commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) - if err != nil { - return err + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + en.childMutex.RLock() + child := en.child + en.childMutex.RUnlock() + + if child != nil { + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + return } + + en.EncodedChild = child.getHash() } en.dirty = false - _, err = encodeNodeAndCommitToDB(en, targetDb) + encNode, err := en.getEncodedNode() if err != nil { - return err + goRoutinesManager.SetError(err) + return } + hash := en.hasher.Compute(string(encNode)) + en.hash = hash + + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + return + } + if uint(level) == maxTrieLevelInMemory { log.Trace("collapse extension node on commit") - var collapsedEn *extensionNode - collapsedEn, err = en.getCollapsedEn() - if err != nil { - return err - } - - *en = *collapsedEn + en.childMutex.Lock() + en.child = nil + en.childMutex.Unlock() } - return nil } func (en *extensionNode) commitSnapshot( diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index 31b75e24a6..26c47e5797 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -183,8 +183,9 @@ func TestExtensionNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedEn) en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, en.marsh, en.hasher) @@ -194,24 +195,6 @@ func TestExtensionNode_commit(t *testing.T) { assert.Equal(t, h1, h2) } -func TestExtensionNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - en := &extensionNode{} - - err := en.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) -} - -func TestExtensionNode_commitNilNode(t *testing.T) { - t.Parallel() - - var en *extensionNode - - err := en.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilExtensionNode)) -} - func TestExtensionNode_commitCollapsedNode(t *testing.T) { t.Parallel() @@ -221,8 +204,9 @@ func TestExtensionNode_commitCollapsedNode(t *testing.T) { collapsedEn.setHash(getTestGoroutinesManager()) collapsedEn.dirty = true - err := collapsedEn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + collapsedEn.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, collapsedEn.marsh, collapsedEn.hasher) @@ -271,7 +255,7 @@ func TestExtensionNode_resolveCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) _, resolved := getBnAndCollapsedBn(en.marsh, en.hasher) child, err := collapsedEn.resolveIfCollapsed(db) @@ -345,7 +329,7 @@ func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -418,7 +402,7 @@ func TestExtensionNode_insertCollapsedNode(t *testing.T) { key := []byte{100, 15, 5, 6} en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) @@ -442,7 +426,7 @@ func TestExtensionNode_insertInStoredEnSameKey(t *testing.T) { key := append(enKey, []byte{11, 12}...) en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) enHash := en.getHash() bn, _, _ := en.getNext(enKey, db) bnHash := bn.getHash() @@ -469,7 +453,7 @@ func TestExtensionNode_insertInStoredEnDifferentKey(t *testing.T) { nodeKey := []byte{11, 12} en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) expectedHashes := [][]byte{en.getHash()} th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -559,7 +543,7 @@ func TestExtensionNode_deleteFromStoredEn(t *testing.T) { lnPathKey := key en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) bn, key, _ := en.getNext(key, db) ln, _, _ := bn.getNext(key, db) expectedHashes := [][]byte{ln.getHash(), bn.getHash(), en.getHash()} @@ -598,7 +582,7 @@ func TestExtensionNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -688,7 +672,7 @@ func TestExtensionNode_getChildrenCollapsedEn(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) children, err := collapsedEn.getChildren(db) assert.Nil(t, err) @@ -794,8 +778,9 @@ func TestExtensionNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *tes collapsedEn.setHash(getTestGoroutinesManager()) en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 1, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedEn.EncodedChild, en.EncodedChild) assert.Equal(t, collapsedEn.child, en.child) @@ -812,7 +797,7 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) collapsedEn.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) _ = collapsedEn.commitSnapshot(db, nil, nil, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) en.print(enWriter, 0, db) @@ -1101,8 +1086,9 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), @@ -1124,8 +1110,9 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 6, 7, 16}), "dog"), @@ -1159,8 +1146,9 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 3, 6, 7, 16}), "dog"), @@ -1192,8 +1180,9 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{2, 3, 6, 7, 16}), "dog"), @@ -1224,8 +1213,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{2, 3, 6, 7, 16}), "dog"), @@ -1247,8 +1237,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), @@ -1271,7 +1262,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { t.Parallel() en := getEn() - en.setHash(getTestGoroutinesManager()) + manager := getTestGoroutinesManager() + en.setHash(manager) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 4, 5, 6}), "dog"), } @@ -1281,9 +1274,10 @@ func TestExtensionNode_deleteBatch(t *testing.T) { assert.Nil(t, err) newEn, _ := en.insert(data, goRoutinesManager, nil) - newEn.setHash(getTestGoroutinesManager()) - err = newEn.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + newEn.setHash(manager) + assert.Nil(t, manager.GetError()) + newEn.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) dataForRemoval := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 7, 7, 8, 9}), "dog"), @@ -1302,9 +1296,11 @@ func TestExtensionNode_deleteBatch(t *testing.T) { t.Parallel() en := getEn() - en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.setHash(manager) + assert.Nil(t, manager.GetError()) + en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), diff --git a/trie/interface.go b/trie/interface.go index b01c009d57..e8a79f475c 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -44,7 +44,7 @@ type node interface { getVersion() (core.TrieNodeVersion, error) collectLeavesForMigration(migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder) (bool, error) - commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error + commitDirty(level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) commitSnapshot(originDb common.TrieStorageInteractor, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, depthLevel int) error sizeInBytes() int diff --git a/trie/leafNode.go b/trie/leafNode.go index a6b44a9411..6c9c1a4180 100644 --- a/trie/leafNode.go +++ b/trie/leafNode.go @@ -68,20 +68,30 @@ func (ln *leafNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(ln) } -func (ln *leafNode) commitDirty(_ byte, _ uint, _ common.TrieStorageInteractor, targetDb common.BaseStorer) error { - err := ln.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } - +func (ln *leafNode) commitDirty( + _ byte, + _ uint, + goRoutinesManager common.TrieGoroutinesManager, + _ common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { if !ln.dirty { - return nil + return } ln.dirty = false - _, err = encodeNodeAndCommitToDB(ln, targetDb) + encNode, err := ln.getEncodedNode() + if err != nil { + goRoutinesManager.SetError(err) + return + } + hash := ln.hasher.Compute(string(encNode)) + ln.hash = hash - return err + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + } } func (ln *leafNode) commitSnapshot( diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index 97ff0c7e6d..5ecb93c497 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -131,8 +131,9 @@ func TestLeafNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(ln) ln.setHash(getTestGoroutinesManager()) - err := ln.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + ln.commitDirty(0, 5, manager, db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, ln.marsh, ln.hasher) @@ -141,24 +142,6 @@ func TestLeafNode_commit(t *testing.T) { assert.Equal(t, ln, n) } -func TestLeafNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - ln := &leafNode{} - - err := ln.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyLeafNode)) -} - -func TestLeafNode_commitNilNode(t *testing.T) { - t.Parallel() - - var ln *leafNode - - err := ln.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilLeafNode)) -} - func TestLeafNode_getEncodedNode(t *testing.T) { t.Parallel() @@ -301,7 +284,7 @@ func TestLeafNode_insertInStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion("dog", "dogs")} @@ -321,7 +304,7 @@ func TestLeafNode_insertInStoredLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() marsh, hasher := getTestMarshalizerAndHasher() ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3}), "dog"), marsh, hasher) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs")} @@ -389,7 +372,7 @@ func TestLeafNode_deleteFromStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) lnHash := ln.getHash() data := []core.TrieData{{Key: []byte("dog")}} @@ -408,7 +391,7 @@ func TestLeafNode_deleteFromLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) wrongKey := []byte{1, 2, 3} data := []core.TrieData{{Key: wrongKey}} @@ -748,7 +731,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dogs")} - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -769,7 +752,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog")} - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -792,7 +775,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 3, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{3, 4, 5, 6, 16}), "dog"), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -822,7 +805,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{1, 2, 5, 6, 16}), "dog"), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -854,7 +837,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -876,7 +859,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) diff --git a/trie/node_test.go b/trie/node_test.go index 864ca00297..fd5225c8ae 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -136,7 +136,7 @@ func TestNode_getNodeFromDBAndDecodeBranchNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) encNode, _ := bn.marsh.Marshal(collapsedBn) encNode = append(encNode, branch) @@ -156,7 +156,7 @@ func TestNode_getNodeFromDBAndDecodeExtensionNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) encNode, _ := en.marsh.Marshal(collapsedEn) encNode = append(encNode, extension) @@ -176,7 +176,7 @@ func TestNode_getNodeFromDBAndDecodeLeafNode(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) ln.setHash(getTestGoroutinesManager()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) encNode, _ := ln.marsh.Marshal(ln) encNode = append(encNode, leaf) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 0dc4562547..c3e8befe04 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -47,7 +47,7 @@ type patriciaMerkleTrie struct { batchManager common.TrieBatchManager goroutinesThrottler core.Throttler trieOperationInProgress *atomic.Flag - updateTrieMutex sync.RWMutex + trieMutex sync.RWMutex maxTrieLevelInMemory uint chanClose chan struct{} @@ -177,8 +177,8 @@ func (tr *patriciaMerkleTrie) Delete(key []byte) { } func (tr *patriciaMerkleTrie) updateTrie() error { - tr.updateTrieMutex.Lock() - defer tr.updateTrieMutex.Unlock() + tr.trieMutex.Lock() + defer tr.trieMutex.Unlock() batch, err := tr.batchManager.MarkTrieUpdateInProgress() if err != nil { @@ -293,6 +293,9 @@ func (tr *patriciaMerkleTrie) getRootHash() ([]byte, error) { return common.EmptyTrieHash, nil } + tr.trieMutex.Lock() + defer tr.trieMutex.Unlock() + hash := rootNode.getHash() if hash != nil { return hash, nil @@ -322,8 +325,11 @@ func (tr *patriciaMerkleTrie) Commit() error { return err } + tr.trieMutex.Lock() + defer tr.trieMutex.Unlock() + rootNode := tr.GetRootNode() - if rootNode == nil { + if check.IfNil(rootNode) { log.Trace("trying to commit empty trie") return nil } @@ -335,23 +341,17 @@ func (tr *patriciaMerkleTrie) Commit() error { return nil } - manager, err := NewGoroutinesManager(tr.goroutinesThrottler, errChan.NewErrChanWrapper(), tr.chanClose) - if err != nil { - return err - } - - rootNode.setHash(manager) - err = manager.GetError() - if err != nil { - return err - } - tr.ResetCollectedHashes() if log.GetLevel() == logger.LogTrace { log.Trace("started committing trie", "trie", rootNode.getHash()) } - return rootNode.commitDirty(0, tr.maxTrieLevelInMemory, tr.trieStorage, tr.trieStorage) + manager, err := NewGoroutinesManager(tr.goroutinesThrottler, errChan.NewErrChanWrapper(), tr.chanClose) + if err != nil { + return err + } + rootNode.commitDirty(0, tr.maxTrieLevelInMemory, manager, tr.trieStorage, tr.trieStorage) + return manager.GetError() } // Recreate returns a new trie that has the given root hash and database diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 58ce57cad6..5ec63bdf16 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -958,7 +958,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { numOperations := 1000 wg := sync.WaitGroup{} wg.Add(numOperations) - numFunctions := 18 + numFunctions := 7 initialRootHash, _ := tr.RootHash() @@ -993,48 +993,48 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { rootHashHolder := holders.NewRootHashHolder(initialRootHash, epoch) _, err := tr.RecreateFromEpoch(rootHashHolder) assert.Nil(t, err) - case 7: - _ = tr.String() - case 8: - _ = tr.GetObsoleteHashes() - case 9: - _, err := tr.GetDirtyHashes() - assert.Nil(t, err) - case 10: - _, err := tr.GetSerializedNode(initialRootHash) - assert.Nil(t, err) - case 11: - size1KB := uint64(1024 * 1024) - _, _, err := tr.GetSerializedNodes(initialRootHash, size1KB) - assert.Nil(t, err) - case 12: - trieIteratorChannels := &common.TrieIteratorChannels{ - LeavesChan: make(chan core.KeyValueHolder, 1000), - ErrChan: errChan.NewErrChanWrapper(), - } - - err := tr.GetAllLeavesOnChannel( - trieIteratorChannels, - context.Background(), - initialRootHash, - keyBuilder.NewKeyBuilder(), - parsers.NewMainTrieLeafParser(), - ) - assert.Nil(t, err) - case 13: - _, _, _ = tr.GetProof(initialRootHash, initialRootHash) // this might error due to concurrent operations that change the roothash - case 14: - // extremely hard to compute an existing hash due to concurrent changes. - _, _ = tr.VerifyProof([]byte("dog"), []byte("puppy"), [][]byte{[]byte("proof1")}) // this might error due to concurrent operations that change the roothash - case 15: - sm := tr.GetStorageManager() - assert.NotNil(t, sm) - case 16: - _ = tr.GetOldRoot() - case 17: - trieStatsHandler := tr.(common.TrieStats) - _, err := trieStatsHandler.GetTrieStats("address", initialRootHash) - assert.Nil(t, err) + //case 7: + // _ = tr.String() + //case 8: + // _ = tr.GetObsoleteHashes() + //case 9: + // _, err := tr.GetDirtyHashes() + // assert.Nil(t, err) + //case 10: + // _, err := tr.GetSerializedNode(initialRootHash) + // assert.Nil(t, err) + //case 11: + // size1KB := uint64(1024 * 1024) + // _, _, err := tr.GetSerializedNodes(initialRootHash, size1KB) + // assert.Nil(t, err) + //case 12: + // trieIteratorChannels := &common.TrieIteratorChannels{ + // LeavesChan: make(chan core.KeyValueHolder, 1000), + // ErrChan: errChan.NewErrChanWrapper(), + // } + // + // err := tr.GetAllLeavesOnChannel( + // trieIteratorChannels, + // context.Background(), + // initialRootHash, + // keyBuilder.NewKeyBuilder(), + // parsers.NewMainTrieLeafParser(), + // ) + // assert.Nil(t, err) + //case 13: + // _, _, _ = tr.GetProof(initialRootHash, initialRootHash) // this might error due to concurrent operations that change the roothash + //case 14: + // // extremely hard to compute an existing hash due to concurrent changes. + // _, _ = tr.VerifyProof([]byte("dog"), []byte("puppy"), [][]byte{[]byte("proof1")}) // this might error due to concurrent operations that change the roothash + //case 15: + // sm := tr.GetStorageManager() + // assert.NotNil(t, sm) + //case 16: + // _ = tr.GetOldRoot() + //case 17: + // trieStatsHandler := tr.(common.TrieStats) + // _, err := trieStatsHandler.GetTrieStats("address", initialRootHash) + // assert.Nil(t, err) default: assert.Fail(t, fmt.Sprintf("invalid numFunctions value %d, operation: %d", numFunctions, operation)) } From 27288b2eea6bfa9e926ccb955f4c5d728d401534 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 20 Nov 2024 11:21:07 +0200 Subject: [PATCH 2/6] use hashesCollector during commit --- common/interface.go | 15 ++- .../benchmarks/loadFromTrie_test.go | 3 +- .../longTests/storage/storage_test.go | 3 +- .../state/genesisState/genesisState_test.go | 12 +- .../state/stateTrie/stateTrie_test.go | 5 +- .../stateTrieClose/stateTrieClose_test.go | 7 +- .../state/stateTrieSync/stateTrieSync_test.go | 5 +- .../vm/txsFee/migrateDataTrie_test.go | 5 +- state/accountsDB.go | 45 +++----- state/accountsDB_test.go | 51 +++++---- .../dataTrieHashesCollector.go | 57 ++++++++++ .../disabledHashesCollector.go | 29 +++++ state/hashesCollector/hashesCollector.go | 32 ++++++ state/interface.go | 2 +- .../disabled/disabledStoragePruningManager.go | 2 +- .../storagePruningManager.go | 5 +- state/syncer/userAccountSyncer_test.go | 3 +- state/syncer/userAccountsSyncer_test.go | 9 +- .../state/storagePruningManagerStub.go | 6 +- testscommon/trie/trieStub.go | 33 +----- trie/baseIterator_test.go | 9 +- trie/branchNode.go | 31 +----- trie/branchNode_test.go | 49 +++----- trie/depthFirstSync_test.go | 9 +- trie/dfsIterator_test.go | 7 +- trie/doubleListSync_test.go | 9 +- trie/extensionNode.go | 27 +---- trie/extensionNode_test.go | 62 ++++------- trie/interceptedNode_test.go | 3 +- trie/interface.go | 3 +- trie/leafNode.go | 16 +-- trie/leafNode_test.go | 31 +++--- trie/node_test.go | 56 ++-------- trie/patriciaMerkleTrie.go | 81 +++----------- trie/patriciaMerkleTrie_test.go | 105 ++++++++---------- trie/rootManager.go | 7 +- update/genesis/import.go | 5 +- update/sync/coordinator_test.go | 4 +- update/sync/syncAccountsDBs.go | 3 +- 39 files changed, 382 insertions(+), 464 deletions(-) create mode 100644 state/hashesCollector/dataTrieHashesCollector.go create mode 100644 state/hashesCollector/disabledHashesCollector.go create mode 100644 state/hashesCollector/hashesCollector.go diff --git a/common/interface.go b/common/interface.go index 1980975447..cf04aaf885 100644 --- a/common/interface.go +++ b/common/interface.go @@ -41,13 +41,10 @@ type Trie interface { Update(key, value []byte) error Delete(key []byte) RootHash() ([]byte, error) - Commit() error + Commit(collector TrieHashesCollector) error Recreate(root []byte) (Trie, error) RecreateFromEpoch(options RootHashHolder) (Trie, error) - String() string - GetObsoleteHashes() [][]byte - GetDirtyHashes() (ModifiedHashes, error) - GetOldRoot() []byte + ToString() string GetSerializedNodes([]byte, uint64) ([][]byte, uint64, error) GetSerializedNode([]byte) ([]byte, error) GetAllLeavesOnChannel(allLeavesChan *TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder KeyBuilder, trieLeafParser TrieLeafParser) error @@ -403,3 +400,11 @@ type TrieGoroutinesManager interface { GetError() error IsInterfaceNil() bool } + +// TrieHashesCollector defines the methods needed for collecting trie hashes +type TrieHashesCollector interface { + AddDirtyHash(hash []byte) + GetDirtyHashes() ModifiedHashes + AddObsoleteHashes(oldRootHash []byte, oldHashes [][]byte) + GetCollectedData() ([]byte, ModifiedHashes, ModifiedHashes) +} diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index c3c7a99f57..75451f3f79 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-go/common" disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -120,7 +121,7 @@ func insertKeysIntoTrie(t *testing.T, tr common.Trie, numTrieLevels int, numChil _, depth, _ := tr.Get(key) require.Equal(t, uint32(numTrieLevels), depth+1) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return key } diff --git a/integrationTests/longTests/storage/storage_test.go b/integrationTests/longTests/storage/storage_test.go index bea274856d..e47fd932c5 100644 --- a/integrationTests/longTests/storage/storage_test.go +++ b/integrationTests/longTests/storage/storage_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" @@ -130,7 +131,7 @@ func TestWriteContinuouslyInTree(t *testing.T) { if i%written == 0 { endTime := time.Now() diff := endTime.Sub(startTime) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) fmt.Printf("Written %d, total %d in %f s\n", written, i, diff.Seconds()) startTime = time.Now() } diff --git a/integrationTests/state/genesisState/genesisState_test.go b/integrationTests/state/genesisState/genesisState_test.go index cf7c1b1519..cefe1c2e44 100644 --- a/integrationTests/state/genesisState/genesisState_test.go +++ b/integrationTests/state/genesisState/genesisState_test.go @@ -85,7 +85,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { _ = tr1.Update([]byte(key3), []byte(val)) fmt.Println() - strTr1 := tr1.String() + strTr1 := tr1.ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -98,7 +98,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { fmt.Printf("root hash2: %s\n", base64.StdEncoding.EncodeToString(hash2)) fmt.Println() - strTr2 := tr2.String() + strTr2 := tr2.ToString() fmt.Println(strTr2) assert.Equal(t, hash1, hash2) @@ -126,7 +126,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr1.Update([]byte(key4), []byte(val)) fmt.Println() - strTr1 := tr1.String() + strTr1 := tr1.ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -140,7 +140,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr2.Update([]byte(key6), []byte(val)) fmt.Println() - strTr2 := tr2.String() + strTr2 := tr2.ToString() fmt.Println(strTr2) hash2, _ := tr2.RootHash() @@ -299,12 +299,12 @@ func printTestDebugLines( fmt.Println() fmt.Println("Reference trie:") - strRefTrie := referenceTrie.String() + strRefTrie := referenceTrie.ToString() fmt.Println(strRefTrie) fmt.Println() fmt.Println("Actual trie:") - strTr := tr.String() + strTr := tr.ToString() fmt.Println(strTr) } diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index d20b9d819e..db15862d79 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -36,6 +36,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/storagePruningManager" @@ -279,7 +280,7 @@ func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { _ = tr1.Update(key, value) h1, _ := tr1.RootHash() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) tr2, err := tr1.Recreate(h1) @@ -994,7 +995,7 @@ func BenchmarkCreateOneMillionAccountsWithMockDB(b *testing.B) { core.ConvertBytes(rtm.Sys), ) - _ = tr.String() + _ = tr.ToString() } func BenchmarkCreateOneMillionAccounts(b *testing.B) { diff --git a/integrationTests/state/stateTrieClose/stateTrieClose_test.go b/integrationTests/state/stateTrieClose/stateTrieClose_test.go index 9d99a17848..0f6f4718ef 100644 --- a/integrationTests/state/stateTrieClose/stateTrieClose_test.go +++ b/integrationTests/state/stateTrieClose/stateTrieClose_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" @@ -28,7 +29,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { for i := 0; i < numLeavesToAdd; i++ { _ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) time.Sleep(time.Second * 2) // allow the commit go routines to finish completely as to not alter the further counters gc := goroutines.NewGoCounter(goroutines.TestsRelevantGoRoutines) @@ -70,7 +71,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { assert.Nil(t, err) _ = tr.Update([]byte("god"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() leavesChannel1 = &common.TrieIteratorChannels{ @@ -91,7 +92,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { assert.Nil(t, err) _ = tr.Update([]byte("eggod"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() leavesChannel2 := &common.TrieIteratorChannels{ diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index cb933aedba..eccca46cfe 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage" @@ -103,7 +104,7 @@ func testNodeRequestInterceptTrieNodesWithMessenger(t *testing.T, version int) { _ = resolverTrie.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = resolverTrie.Commit() + _ = resolverTrie.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := resolverTrie.RootHash() numLeaves := getNumLeaves(t, resolverTrie, rootHash) @@ -224,7 +225,7 @@ func testNodeRequestInterceptTrieNodesWithMessengerNotSyncingShouldErr(t *testin _ = resolverTrie.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = resolverTrie.Commit() + _ = resolverTrie.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := resolverTrie.RootHash() numLeaves := getNumLeaves(t, resolverTrie, rootHash) diff --git a/integrationTests/vm/txsFee/migrateDataTrie_test.go b/integrationTests/vm/txsFee/migrateDataTrie_test.go index a33d57883d..689215f217 100644 --- a/integrationTests/vm/txsFee/migrateDataTrie_test.go +++ b/integrationTests/vm/txsFee/migrateDataTrie_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/vm" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/stretchr/testify/require" ) @@ -110,7 +111,7 @@ func TestMigrateDataTrieBuiltInFunc(t *testing.T) { migrateDataTrie(t, testContext, sndAddr, gasPrice, gasLimit, vmcommon.Ok) testGasConsumed(t, testContext, gasLimit, 20000) - err = dtr.Commit() + err = dtr.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) rootHash, err = dtr.RootHash() @@ -162,7 +163,7 @@ func TestMigrateDataTrieBuiltInFunc(t *testing.T) { migrateDataTrie(t, testContext, sndAddr, gasPrice, gasLimit, vmcommon.Ok) numMigrateDataTrieCalls++ - err = dtr.Commit() + err = dtr.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) rootHash, err = dtr.RootHash() diff --git a/state/accountsDB.go b/state/accountsDB.go index e42627c9d6..b8590c3ecd 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" @@ -777,22 +778,26 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - oldHashes := make(common.ModifiedHashes) - newHashes := make(common.ModifiedHashes) + hc := hashesCollector.NewDisabledHashesCollector() + if adb.mainTrie.GetStorageManager().IsPruningEnabled() { + hc = hashesCollector.NewDataTrieHashesCollector() + } + // Step 1. commit all data tries dataTries := adb.dataTries.GetAll() for i := 0; i < len(dataTries); i++ { - err := adb.commitTrie(dataTries[i], oldHashes, newHashes) + err := dataTries[i].Commit(hc) if err != nil { return nil, err } } adb.dataTries.Reset() - oldRoot := adb.mainTrie.GetOldRoot() - // Step 2. commit main trie - err := adb.commitTrie(adb.mainTrie, oldHashes, newHashes) + if adb.mainTrie.GetStorageManager().IsPruningEnabled() { + hc = hashesCollector.NewHashesCollector(hc) + } + err := adb.mainTrie.Commit(hc) if err != nil { return nil, err } @@ -803,7 +808,7 @@ func (adb *AccountsDB) commit() ([]byte, error) { return nil, err } - err = adb.markForEviction(oldRoot, newRoot, oldHashes, newHashes) + err = adb.markForEviction(newRoot, hc) if err != nil { return nil, err } @@ -818,36 +823,14 @@ func (adb *AccountsDB) commit() ([]byte, error) { } func (adb *AccountsDB) markForEviction( - oldRoot []byte, newRoot []byte, - oldHashes common.ModifiedHashes, - newHashes common.ModifiedHashes, + collector common.TrieHashesCollector, ) error { if !adb.mainTrie.GetStorageManager().IsPruningEnabled() { return nil } - return adb.storagePruningManager.MarkForEviction(oldRoot, newRoot, oldHashes, newHashes) -} - -func (adb *AccountsDB) commitTrie(tr common.Trie, oldHashes common.ModifiedHashes, newHashes common.ModifiedHashes) error { - if adb.mainTrie.GetStorageManager().IsPruningEnabled() { - oldTrieHashes := tr.GetObsoleteHashes() - newTrieHashes, err := tr.GetDirtyHashes() - if err != nil { - return err - } - - for _, hash := range oldTrieHashes { - oldHashes[string(hash)] = struct{}{} - } - - for hash := range newTrieHashes { - newHashes[hash] = struct{}{} - } - } - - return tr.Commit() + return adb.storagePruningManager.MarkForEviction(newRoot, collector) } // RootHash returns the main trie's root hash diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index ffd5ae1457..ad6e1f8d69 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/rand" + "encoding/hex" "errors" "fmt" mathRand "math/rand" @@ -26,6 +27,7 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/parsers" @@ -863,7 +865,7 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { marshaller := &marshallerMock.MarshalizerMock{} serializedAccount, _ := marshaller.Marshal(stateMock.AccountWrapMock{}) trieStub := trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { commitCalled++ return nil @@ -882,7 +884,7 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { UpdateWithVersionCalled: func(key, value []byte, version core.TrieNodeVersion) error { return nil }, - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { commitCalled++ return nil @@ -1826,22 +1828,29 @@ func TestAccountsDB_TrieDatabasePruning(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _, err := adb.Commit() + assert.Nil(t, err) rootHash, _ := tr.RootHash() _ = tr.Update([]byte("dog"), []byte("doee")) - oldHashes := tr.GetObsoleteHashes() - assert.Equal(t, 4, len(oldHashes)) - _, err := adb.Commit() + _, err = adb.Commit() assert.Nil(t, err) adb.CancelPrune(rootHash, state.NewRoot) adb.PruneTrie(rootHash, state.OldRoot, state.NewPruningHandler(state.EnableDataRemoval)) time.Sleep(trieDbOperationDelay) - for i := range oldHashes { - encNode, errGet := tr.GetStorageManager().Get(oldHashes[i]) + oldHashes := []string{ + "aa07e09a40ff7d332c2c50fa1619c751629f06babc3c1793ab3db8dd0c298345", + "a21e06e29734f85c99da88ae69d71b532f3929c2f96a43b95813e8c077151d11", + "d2b566879acfb873caa9a2282d48d196aefaa2a6168a84447298c472a1ba8f1c", + "7f20b853d6edc10f3e6840c7779fe842b7e87b2ad6488d4bb2465c0e389b7272", + } + + for _, hexKey := range oldHashes { + key, _ := hex.DecodeString(hexKey) + encNode, errGet := tr.GetStorageManager().Get(key) assert.Nil(t, encNode) assert.NotNil(t, errGet) } @@ -1923,12 +1932,6 @@ func TestAccountsDB_Prune(t *testing.T) { assert.Equal(t, trie.ErrKeyNotFound, err) } -func mergeMaps(map1 common.ModifiedHashes, map2 common.ModifiedHashes) { - for key, val := range map2 { - map1[key] = val - } -} - func generateAccounts(t testing.TB, numAccounts int, adb state.AccountsAdapter) [][]byte { accountsAddresses := make([][]byte, numAccounts) for i := 0; i < numAccounts; i++ { @@ -1951,23 +1954,18 @@ func generateRandomByteArray(size int) []byte { return r } -func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.AccountsDB) common.ModifiedHashes { +func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.AccountsDB) { acc, _ := adb.LoadAccount(accountsAddresses[0]) err := acc.(state.UserAccountHandler).SaveKeyValue([]byte("key1"), []byte("value1")) assert.Nil(t, err) err = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key2"), []byte("value2")) assert.Nil(t, err) _ = adb.SaveAccount(acc) - newHashes, _ := acc.(state.UserAccountHandler).DataTrie().(common.Trie).GetDirtyHashes() acc, _ = adb.LoadAccount(accountsAddresses[1]) err = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key2"), []byte("value2")) assert.Nil(t, err) _ = adb.SaveAccount(acc) - newHashesDataTrie, _ := acc.(state.UserAccountHandler).DataTrie().(common.Trie).GetDirtyHashes() - mergeMaps(newHashes, newHashesDataTrie) - - return newHashes } func TestAccountsDB_GetCode(t *testing.T) { @@ -2112,7 +2110,7 @@ func TestAccountsDB_GetTrie(t *testing.T) { _, adb := getDefaultTrieAndAccountsDb() addresses := generateAccounts(t, 2, adb) - _ = modifyDataTries(t, addresses, adb) + modifyDataTries(t, addresses, adb) _, _ = adb.Commit() @@ -2693,10 +2691,10 @@ func prepareTrie(tr common.Trie, numKeys int) common.ModifiedHashes { _ = tr.Update([]byte(key), []byte(val)) } - hashes, _ := tr.GetDirtyHashes() - _ = tr.Commit() - - return hashes + hc := hashesCollector.NewDataTrieHashesCollector() + _ = tr.Commit(hc) + _, _, newHashes := hc.GetCollectedData() + return newHashes } func addDataTries(accountsAddresses [][]byte, adb *state.AccountsDB) { @@ -2814,7 +2812,8 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { // check that the data trie was completely reverted. The rootHash of the user account should be present // in both old and new hashes. This means that after the revert, the rootHash is the same as before markForEvictionCalled := false - spm.MarkForEvictionCalled = func(_ []byte, _ []byte, oldHashes common.ModifiedHashes, newHashes common.ModifiedHashes) error { + spm.MarkForEvictionCalled = func(_ []byte, collector common.TrieHashesCollector) error { + _, oldHashes, newHashes := collector.GetCollectedData() _, ok := oldHashes[string(userAccRootHash)] require.True(t, ok) _, ok = newHashes[string(userAccRootHash)] diff --git a/state/hashesCollector/dataTrieHashesCollector.go b/state/hashesCollector/dataTrieHashesCollector.go new file mode 100644 index 0000000000..e0aff5d13c --- /dev/null +++ b/state/hashesCollector/dataTrieHashesCollector.go @@ -0,0 +1,57 @@ +package hashesCollector + +import ( + "sync" + + "github.com/multiversx/mx-chain-go/common" +) + +type dataTrieHashesCollector struct { + oldHashes common.ModifiedHashes + newHashes common.ModifiedHashes + + sync.RWMutex +} + +// NewDataTrieHashesCollector creates a new instance of dataTrieHashesCollector. +// This collector is used to collect hashes related to the data trie. +func NewDataTrieHashesCollector() *dataTrieHashesCollector { + return &dataTrieHashesCollector{ + oldHashes: make(common.ModifiedHashes), + newHashes: make(common.ModifiedHashes), + } +} + +// AddDirtyHash adds the new hash to the collector. +func (hc *dataTrieHashesCollector) AddDirtyHash(hash []byte) { + hc.Lock() + defer hc.Unlock() + + hc.newHashes[string(hash)] = struct{}{} +} + +// GetDirtyHashes returns the new hashes. +func (hc *dataTrieHashesCollector) GetDirtyHashes() common.ModifiedHashes { + hc.RLock() + defer hc.RUnlock() + + return hc.newHashes +} + +// AddObsoleteHashes adds the old hashes to the collector. +func (hc *dataTrieHashesCollector) AddObsoleteHashes(_ []byte, oldHashes [][]byte) { + hc.Lock() + defer hc.Unlock() + + for _, hash := range oldHashes { + hc.oldHashes[string(hash)] = struct{}{} + } +} + +// GetCollectedData returns the old hashes and the new hashes. +func (hc *dataTrieHashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + hc.RLock() + defer hc.RUnlock() + + return nil, hc.oldHashes, hc.newHashes +} diff --git a/state/hashesCollector/disabledHashesCollector.go b/state/hashesCollector/disabledHashesCollector.go new file mode 100644 index 0000000000..af016ac763 --- /dev/null +++ b/state/hashesCollector/disabledHashesCollector.go @@ -0,0 +1,29 @@ +package hashesCollector + +import "github.com/multiversx/mx-chain-go/common" + +type disabledHashesCollector struct { +} + +// NewDisabledHashesCollector creates a new instance of disabledHashesCollector. +func NewDisabledHashesCollector() common.TrieHashesCollector { + return &disabledHashesCollector{} +} + +// AddDirtyHash does nothing. +func (hc *disabledHashesCollector) AddDirtyHash(_ []byte) { +} + +// GetDirtyHashes returns an empty map. +func (hc *disabledHashesCollector) GetDirtyHashes() common.ModifiedHashes { + return make(common.ModifiedHashes) +} + +// AddObsoleteHashes does nothing. +func (hc *disabledHashesCollector) AddObsoleteHashes(_ []byte, _ [][]byte) { +} + +// GetCollectedData returns nil data +func (hc *disabledHashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + return nil, nil, nil +} diff --git a/state/hashesCollector/hashesCollector.go b/state/hashesCollector/hashesCollector.go new file mode 100644 index 0000000000..c7740d8e5f --- /dev/null +++ b/state/hashesCollector/hashesCollector.go @@ -0,0 +1,32 @@ +package hashesCollector + +import ( + "github.com/multiversx/mx-chain-go/common" +) + +type hashesCollector struct { + common.TrieHashesCollector + + oldRootHash []byte +} + +// NewHashesCollector creates a new instance of hashesCollector. +// This collector is used to collect hashes related to the main trie. +func NewHashesCollector(collector common.TrieHashesCollector) *hashesCollector { + return &hashesCollector{ + TrieHashesCollector: collector, + oldRootHash: nil, + } +} + +// AddObsoleteHashes adds the old root hash and the old hashes to the collector. +func (hc *hashesCollector) AddObsoleteHashes(oldRootHash []byte, oldHashes [][]byte) { + hc.TrieHashesCollector.AddObsoleteHashes(oldRootHash, oldHashes) + hc.oldRootHash = oldRootHash +} + +// GetCollectedData returns the old root hash, the old hashes and the new hashes. +func (hc *hashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + _, oldHashes, newHashes := hc.TrieHashesCollector.GetCollectedData() + return hc.oldRootHash, oldHashes, newHashes +} diff --git a/state/interface.go b/state/interface.go index 06050f95fc..7920c3c4ad 100644 --- a/state/interface.go +++ b/state/interface.go @@ -147,7 +147,7 @@ type AtomicBuffer interface { // StoragePruningManager is used to manage all state pruning operations type StoragePruningManager interface { - MarkForEviction([]byte, []byte, common.ModifiedHashes, common.ModifiedHashes) error + MarkForEviction([]byte, common.TrieHashesCollector) error PruneTrie(rootHash []byte, identifier TriePruningIdentifier, tsm common.StorageManager, handler PruningHandler) CancelPrune(rootHash []byte, identifier TriePruningIdentifier, tsm common.StorageManager) Close() error diff --git a/state/storagePruningManager/disabled/disabledStoragePruningManager.go b/state/storagePruningManager/disabled/disabledStoragePruningManager.go index 6de7e2b084..5229166a49 100644 --- a/state/storagePruningManager/disabled/disabledStoragePruningManager.go +++ b/state/storagePruningManager/disabled/disabledStoragePruningManager.go @@ -14,7 +14,7 @@ func NewDisabledStoragePruningManager() *disabledStoragePruningManager { } // MarkForEviction does nothing for this implementation -func (i *disabledStoragePruningManager) MarkForEviction(_ []byte, _ []byte, _ common.ModifiedHashes, _ common.ModifiedHashes) error { +func (i *disabledStoragePruningManager) MarkForEviction(_ []byte, _ common.TrieHashesCollector) error { return nil } diff --git a/state/storagePruningManager/storagePruningManager.go b/state/storagePruningManager/storagePruningManager.go index 757d04cc9e..2f6d7fae8e 100644 --- a/state/storagePruningManager/storagePruningManager.go +++ b/state/storagePruningManager/storagePruningManager.go @@ -44,11 +44,10 @@ func NewStoragePruningManager( // MarkForEviction adds the given hashes to the underlying evictionWaitingList func (spm *storagePruningManager) MarkForEviction( - oldRoot []byte, newRoot []byte, - oldHashes common.ModifiedHashes, - newHashes common.ModifiedHashes, + collector common.TrieHashesCollector, ) error { + oldRoot, oldHashes, newHashes := collector.GetCollectedData() if bytes.Equal(newRoot, oldRoot) { log.Trace("old root and new root are identical", "rootHash", newRoot) return nil diff --git a/state/syncer/userAccountSyncer_test.go b/state/syncer/userAccountSyncer_test.go index eefdd96778..d7b51390f8 100644 --- a/state/syncer/userAccountSyncer_test.go +++ b/state/syncer/userAccountSyncer_test.go @@ -1,6 +1,7 @@ package syncer import ( + "github.com/multiversx/mx-chain-go/state/hashesCollector" "testing" "time" @@ -93,7 +94,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { value := []byte("value") _ = tr.Update(key, value) rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) args.Cacher = &testscommon.CacherStub{ GetCalled: func(key []byte) (value interface{}, ok bool) { diff --git a/state/syncer/userAccountsSyncer_test.go b/state/syncer/userAccountsSyncer_test.go index 176a4ec749..af30d51f98 100644 --- a/state/syncer/userAccountsSyncer_test.go +++ b/state/syncer/userAccountsSyncer_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/testscommon" @@ -111,7 +112,7 @@ func getSerializedTrieNode( tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) _ = tr.Update(key, []byte("value")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return serializedLeafNode } @@ -248,7 +249,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), accountBytes) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) leavesChannels := &common.TrieIteratorChannels{ LeavesChan: make(chan core.KeyValueHolder, common.TrieLeavesChannelDefaultCapacity), @@ -305,7 +306,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), accountBytes) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) leavesChannels := &common.TrieIteratorChannels{ LeavesChan: make(chan core.KeyValueHolder, common.TrieLeavesChannelDefaultCapacity), @@ -364,7 +365,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { value := []byte("value") _ = tr.Update(key, value) rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) args.Cacher = &testscommon.CacherStub{ GetCalled: func(key []byte) (value interface{}, ok bool) { diff --git a/testscommon/state/storagePruningManagerStub.go b/testscommon/state/storagePruningManagerStub.go index 92c697c522..045d1392d7 100644 --- a/testscommon/state/storagePruningManagerStub.go +++ b/testscommon/state/storagePruningManagerStub.go @@ -7,16 +7,16 @@ import ( // StoragePruningManagerStub - type StoragePruningManagerStub struct { - MarkForEvictionCalled func(bytes []byte, bytes2 []byte, hashes common.ModifiedHashes, hashes2 common.ModifiedHashes) error + MarkForEvictionCalled func(bytes []byte, collector common.TrieHashesCollector) error PruneTrieCalled func(rootHash []byte, identifier state.TriePruningIdentifier, tsm common.StorageManager, handler state.PruningHandler) CancelPruneCalled func(rootHash []byte, identifier state.TriePruningIdentifier, tsm common.StorageManager) CloseCalled func() error } // MarkForEviction - -func (stub *StoragePruningManagerStub) MarkForEviction(bytes []byte, bytes2 []byte, hashes common.ModifiedHashes, hashes2 common.ModifiedHashes) error { +func (stub *StoragePruningManagerStub) MarkForEviction(bytes []byte, collector common.TrieHashesCollector) error { if stub.MarkForEvictionCalled != nil { - return stub.MarkForEvictionCalled(bytes, bytes2, hashes, hashes2) + return stub.MarkForEvictionCalled(bytes, collector) } return nil diff --git a/testscommon/trie/trieStub.go b/testscommon/trie/trieStub.go index 8ab3ab07c1..185a5fa2ce 100644 --- a/testscommon/trie/trieStub.go +++ b/testscommon/trie/trieStub.go @@ -18,10 +18,9 @@ type TrieStub struct { UpdateWithVersionCalled func(key, value []byte, version core.TrieNodeVersion) error DeleteCalled func(key []byte) RootCalled func() ([]byte, error) - CommitCalled func() error + CommitCalled func(collector common.TrieHashesCollector) error RecreateCalled func(root []byte) (common.Trie, error) RecreateFromEpochCalled func(options common.RootHashHolder) (common.Trie, error) - GetObsoleteHashesCalled func() [][]byte AppendToOldHashesCalled func([][]byte) GetSerializedNodesCalled func([]byte, uint64) ([][]byte, uint64, error) GetAllLeavesOnChannelCalled func(leavesChannels *common.TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder common.KeyBuilder, trieLeafParser common.TrieLeafParser) error @@ -29,7 +28,6 @@ type TrieStub struct { VerifyProofCalled func(rootHash []byte, key []byte, proof [][]byte) (bool, error) GetStorageManagerCalled func() common.StorageManager GetSerializedNodeCalled func(bytes []byte) ([]byte, error) - GetOldRootCalled func() []byte CloseCalled func() error CollectLeavesForMigrationCalled func(args vmcommon.ArgsMigrateDataTrieLeaves) error IsMigratedToLatestVersionCalled func() (bool, error) @@ -124,9 +122,9 @@ func (ts *TrieStub) RootHash() ([]byte, error) { } // Commit - -func (ts *TrieStub) Commit() error { +func (ts *TrieStub) Commit(hc common.TrieHashesCollector) error { if ts.CommitCalled != nil { - return ts.CommitCalled() + return ts.CommitCalled(hc) } return errNotImplemented @@ -151,7 +149,7 @@ func (ts *TrieStub) RecreateFromEpoch(options common.RootHashHolder) (common.Tri } // String - -func (ts *TrieStub) String() string { +func (ts *TrieStub) ToString() string { return "stub trie" } @@ -160,15 +158,6 @@ func (ts *TrieStub) IsInterfaceNil() bool { return ts == nil } -// GetObsoleteHashes resets the oldHashes and oldRoot variables and returns the old hashes -func (ts *TrieStub) GetObsoleteHashes() [][]byte { - if ts.GetObsoleteHashesCalled != nil { - return ts.GetObsoleteHashesCalled() - } - - return nil -} - // GetSerializedNodes - func (ts *TrieStub) GetSerializedNodes(hash []byte, maxBuffToSend uint64) ([][]byte, uint64, error) { if ts.GetSerializedNodesCalled != nil { @@ -177,11 +166,6 @@ func (ts *TrieStub) GetSerializedNodes(hash []byte, maxBuffToSend uint64) ([][]b return nil, 0, nil } -// GetDirtyHashes - -func (ts *TrieStub) GetDirtyHashes() (common.ModifiedHashes, error) { - return nil, nil -} - // SetNewHashes - func (ts *TrieStub) SetNewHashes(_ common.ModifiedHashes) { } @@ -195,15 +179,6 @@ func (ts *TrieStub) GetSerializedNode(bytes []byte) ([]byte, error) { return nil, nil } -// GetOldRoot - -func (ts *TrieStub) GetOldRoot() []byte { - if ts.GetOldRootCalled != nil { - return ts.GetOldRootCalled() - } - - return nil -} - // IsMigratedToLatestVersion - func (ts *TrieStub) IsMigratedToLatestVersion() (bool, error) { if ts.IsMigratedToLatestVersionCalled != nil { diff --git a/trie/baseIterator_test.go b/trie/baseIterator_test.go index c65fc89f0c..a899d18f2b 100644 --- a/trie/baseIterator_test.go +++ b/trie/baseIterator_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,13 +37,13 @@ func TestBaseIterator_HasNext(t *testing.T) { tr := emptyTrie() _ = tr.Update([]byte("dog"), []byte("dog")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewBaseIterator(tr, rootHash) assert.False(t, it.HasNext()) _ = tr.Update([]byte("doe"), []byte("doe")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() it, _ = trie.NewBaseIterator(tr, rootHash) assert.True(t, it.HasNext()) @@ -68,7 +69,7 @@ func TestBaseIterator_GetHash(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewBaseIterator(tr, rootHash) @@ -84,7 +85,7 @@ func TestIterator_Search(t *testing.T) { _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) _ = tr.Update([]byte("ddoge"), []byte("foo")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) expectedHashes := []string{ "ecc2304769996585131ad6276c1422265813a2b79d60392130c4baa19a9b4e06", diff --git a/trie/branchNode.go b/trie/branchNode.go index 4d6bda9904..8084b0e8ae 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -182,6 +182,7 @@ func (bn *branchNode) commitDirty( level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, originDb common.TrieStorageInteractor, targetDb common.BaseStorer, ) { @@ -207,7 +208,7 @@ func (bn *branchNode) commitDirty( } if !goRoutinesManager.CanStartGoRoutine() { - child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) if !goRoutinesManager.ShouldContinueProcessing() { return } @@ -221,7 +222,7 @@ func (bn *branchNode) commitDirty( waitGroup.Add(1) go func(childPos int) { - child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) if !goRoutinesManager.ShouldContinueProcessing() { waitGroup.Done() return @@ -246,6 +247,7 @@ func (bn *branchNode) commitDirty( } hash := bn.hasher.Compute(string(encNode)) bn.hash = hash + hashesCollector.AddDirtyHash(hash) err = targetDb.Put(hash, encNode) if err != nil { @@ -849,31 +851,6 @@ func (bn *branchNode) print(writer io.Writer, index int, db common.TrieStorageIn } } -func (bn *branchNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := bn.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !bn.isDirty() { - return nil - } - - for i := range bn.children { - if bn.children[i] == nil { - continue - } - - err = bn.children[i].getDirtyHashes(hashes) - if err != nil { - return err - } - } - - hashes[string(bn.getHash())] = struct{}{} - return nil -} - func (bn *branchNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { err := bn.isEmptyOrNil() if err != nil { diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 7687551a07..2dcaa77b00 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "testing" "github.com/multiversx/mx-chain-core-go/core" @@ -104,7 +105,7 @@ func initTrie() *patriciaMerkleTrie { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return tr } @@ -269,7 +270,7 @@ func TestBranchNode_commit(t *testing.T) { bn.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - bn.commitDirty(0, 5, manager, db, db) + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) @@ -320,7 +321,7 @@ func TestBranchNode_resolveIfCollapsed(t *testing.T) { childPos := byte(2) bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) resolved, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) resolved.dirty = false resolved.hash = bn.EncodedChildren[childPos] @@ -409,7 +410,7 @@ func TestBranchNode_tryGetCollapsedNode(t *testing.T) { bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -521,7 +522,7 @@ func TestBranchNode_insertCollapsedNode(t *testing.T) { key := append([]byte{childPos}, []byte("dog")...) bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) @@ -545,7 +546,7 @@ func TestBranchNode_insertInStoredBnOnExistingPos(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(key, db) lnHash := ln.getHash() @@ -571,7 +572,7 @@ func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) { nilChildPos := byte(11) key := append([]byte{nilChildPos}, []byte("dog")...) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() expectedHashes := [][]byte{bnHash} @@ -658,7 +659,7 @@ func TestBranchNode_deleteFromStoredBn(t *testing.T) { childPos := byte(2) lnKey := append([]byte{childPos}, []byte("dog")...) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(lnKey, db) lnHash := ln.getHash() @@ -734,7 +735,7 @@ func TestBranchNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -983,7 +984,7 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) { _ = tr.Update([]byte("aaa"), []byte("aaa")) _ = tr.Update([]byte("nnn"), []byte("nnn")) _ = tr.Update([]byte("zzz"), []byte("zzz")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) collapsedRoot, _ := tr.GetRootNode().getCollapsed() tr.Delete([]byte("zzz")) @@ -991,7 +992,7 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) { assert.True(t, collapsedRoot.isDirty()) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.False(t, collapsedRoot.isDirty()) } @@ -1085,7 +1086,7 @@ func TestBranchNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testin collapsedBn.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - bn.commitDirty(0, 1, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + bn.commitDirty(0, 1, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedBn.EncodedChildren, bn.EncodedChildren) @@ -1126,20 +1127,6 @@ func TestBranchNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { assert.Equal(t, bnWriter.Bytes(), collapsedBnWriter.Bytes()) } -func TestBranchNode_getDirtyHashesFromCleanNode(t *testing.T) { - t.Parallel() - - db := testscommon.NewMemDbMock() - bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) - dirtyHashes := make(common.ModifiedHashes) - - err := bn.getDirtyHashes(dirtyHashes) - assert.Nil(t, err) - assert.Equal(t, 0, len(dirtyHashes)) -} - func TestBranchNode_getNextHashAndKey(t *testing.T) { t.Parallel() @@ -1551,7 +1538,7 @@ func TestBranchNode_insertOnNilChild(t *testing.T) { bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - bn.commitDirty(0, 5, manager, db, db) + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() @@ -1657,7 +1644,7 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { }, } manager := getTestGoroutinesManager() - bn.commitDirty(0, 5, manager, db, db) + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() @@ -1698,7 +1685,7 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { }, } manager := getTestGoroutinesManager() - bn.commitDirty(0, 5, manager, db, db) + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) @@ -1747,7 +1734,7 @@ func TestBranchNode_insertBatch(t *testing.T) { }, } manager := getTestGoroutinesManager() - bn.commitDirty(0, 5, manager, db, db) + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) @@ -1787,7 +1774,7 @@ func getNewBn() *branchNode { bn, _ := newBranchNode(marsh, hasher) bn.children = children bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) return bn } diff --git a/trie/depthFirstSync_test.go b/trie/depthFirstSync_test.go index 456c1b1f3e..de0861fdd2 100644 --- a/trie/depthFirstSync_test.go +++ b/trie/depthFirstSync_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "sync" "testing" "time" @@ -69,7 +70,7 @@ func TestDepthFirstTrieSyncer_StartSyncingCanTimeout(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -87,7 +88,7 @@ func TestDepthFirstTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -103,7 +104,7 @@ func TestDepthFirstTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { numKeysValues := 100 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -165,7 +166,7 @@ func TestDepthFirstTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi numKeysValues := 100 trSource, memUnitSource := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) diff --git a/trie/dfsIterator_test.go b/trie/dfsIterator_test.go index bc2f2b893f..887e123cc1 100644 --- a/trie/dfsIterator_test.go +++ b/trie/dfsIterator_test.go @@ -2,7 +2,8 @@ package trie_test import ( "testing" - + + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" ) @@ -21,7 +22,7 @@ func TestNewDFSIterator(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, err := trie.NewDFSIterator(tr, rootHash) @@ -34,7 +35,7 @@ func TestDFSIterator_Next(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewDFSIterator(tr, rootHash) diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 6bb2a5eb97..eea0fb7549 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" @@ -151,7 +152,7 @@ func TestDoubleListTrieSyncer_StartSyncingCanTimeout(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -169,7 +170,7 @@ func TestDoubleListTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -185,7 +186,7 @@ func TestDoubleListTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { numKeysValues := 100 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -246,7 +247,7 @@ func TestDoubleListTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi numKeysValues := 100 trSource, memUnitSource := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) diff --git a/trie/extensionNode.go b/trie/extensionNode.go index f52aa8ef34..4167255144 100644 --- a/trie/extensionNode.go +++ b/trie/extensionNode.go @@ -131,6 +131,7 @@ func (en *extensionNode) commitDirty( level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, originDb common.TrieStorageInteractor, targetDb common.BaseStorer, ) { @@ -149,7 +150,7 @@ func (en *extensionNode) commitDirty( en.childMutex.RUnlock() if child != nil { - child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, originDb, targetDb) + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) if !goRoutinesManager.ShouldContinueProcessing() { return } @@ -165,6 +166,7 @@ func (en *extensionNode) commitDirty( } hash := en.hasher.Compute(string(encNode)) en.hash = hash + hashesCollector.AddDirtyHash(hash) err = targetDb.Put(hash, encNode) if err != nil { @@ -639,29 +641,6 @@ func (en *extensionNode) print(writer io.Writer, index int, db common.TrieStorag en.child.print(writer, index+len(str), db) } -func (en *extensionNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := en.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !en.isDirty() { - return nil - } - - if en.child == nil { - return nil - } - - err = en.child.getDirtyHashes(hashes) - if err != nil { - return err - } - hashes[string(en.getHash())] = struct{}{} - - return nil -} - func (en *extensionNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { err := en.isEmptyOrNil() if err != nil { diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index 26c47e5797..f4b204260f 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -9,8 +9,8 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/throttler" - "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -184,7 +184,7 @@ func TestExtensionNode_commit(t *testing.T) { en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, db, db) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) @@ -205,7 +205,7 @@ func TestExtensionNode_commitCollapsedNode(t *testing.T) { collapsedEn.dirty = true manager := getTestGoroutinesManager() - collapsedEn.commitDirty(0, 5, manager, db, db) + collapsedEn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) @@ -255,7 +255,7 @@ func TestExtensionNode_resolveCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) _, resolved := getBnAndCollapsedBn(en.marsh, en.hasher) child, err := collapsedEn.resolveIfCollapsed(db) @@ -329,7 +329,7 @@ func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -402,7 +402,7 @@ func TestExtensionNode_insertCollapsedNode(t *testing.T) { key := []byte{100, 15, 5, 6} en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) @@ -426,7 +426,7 @@ func TestExtensionNode_insertInStoredEnSameKey(t *testing.T) { key := append(enKey, []byte{11, 12}...) en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enHash := en.getHash() bn, _, _ := en.getNext(enKey, db) bnHash := bn.getHash() @@ -453,7 +453,7 @@ func TestExtensionNode_insertInStoredEnDifferentKey(t *testing.T) { nodeKey := []byte{11, 12} en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) expectedHashes := [][]byte{en.getHash()} th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -543,7 +543,7 @@ func TestExtensionNode_deleteFromStoredEn(t *testing.T) { lnPathKey := key en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bn, key, _ := en.getNext(key, db) ln, _, _ := bn.getNext(key, db) expectedHashes := [][]byte{ln.getHash(), bn.getHash(), en.getHash()} @@ -582,7 +582,7 @@ func TestExtensionNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -627,13 +627,13 @@ func TestExtensionNode_reduceNodeCollapsedNode(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() collapsedTrie, _ := tr.Recreate(rootHash) collapsedTrie.Delete([]byte("doe")) - err := collapsedTrie.Commit() + err := collapsedTrie.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -672,7 +672,7 @@ func TestExtensionNode_getChildrenCollapsedEn(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) children, err := collapsedEn.getChildren(db) assert.Nil(t, err) @@ -695,7 +695,7 @@ func TestExtensionNode_loadChildren(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) tr.GetRootNode().setHash(getTestGoroutinesManager()) nodes, _ := getEncodedTrieNodesAndHashes(tr) nodesCacher, _ := cache.NewLRUCache(100) @@ -779,7 +779,7 @@ func TestExtensionNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *tes en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 1, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 1, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedEn.EncodedChild, en.EncodedChild) @@ -797,7 +797,7 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) collapsedEn.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) _ = collapsedEn.commitSnapshot(db, nil, nil, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) en.print(enWriter, 0, db) @@ -806,20 +806,6 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { assert.Equal(t, enWriter.Bytes(), collapsedEnWriter.Bytes()) } -func TestExtensionNode_getDirtyHashesFromCleanNode(t *testing.T) { - t.Parallel() - - db := testscommon.NewMemDbMock() - en, _ := getEnAndCollapsedEn() - en.setHash(getTestGoroutinesManager()) - _ = en.commitSnapshot(db, nil, nil, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - dirtyHashes := make(common.ModifiedHashes) - - err := en.getDirtyHashes(dirtyHashes) - assert.Nil(t, err) - assert.Equal(t, 0, len(dirtyHashes)) -} - func TestExtensionNode_getNextHashAndKey(t *testing.T) { t.Parallel() @@ -1087,7 +1073,7 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1111,7 +1097,7 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1147,7 +1133,7 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1181,7 +1167,7 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1214,7 +1200,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1238,7 +1224,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ @@ -1276,7 +1262,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { newEn, _ := en.insert(data, goRoutinesManager, nil) newEn.setHash(manager) assert.Nil(t, manager.GetError()) - newEn.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + newEn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) dataForRemoval := []core.TrieData{ @@ -1299,7 +1285,7 @@ func TestExtensionNode_deleteBatch(t *testing.T) { manager := getTestGoroutinesManager() en.setHash(manager) assert.Nil(t, manager.GetError()) - en.commitDirty(0, 5, manager, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.Nil(t, manager.GetError()) data := []core.TrieData{ diff --git a/trie/interceptedNode_test.go b/trie/interceptedNode_test.go index ec413063aa..071023f706 100644 --- a/trie/interceptedNode_test.go +++ b/trie/interceptedNode_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" @@ -14,7 +15,7 @@ import ( func getDefaultInterceptedTrieNodeParameters() ([]byte, hashing.Hasher) { tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) nodes, _ := getEncodedTrieNodesAndHashes(tr) return nodes[0], &testscommon.KeccakMock{} diff --git a/trie/interface.go b/trie/interface.go index e8a79f475c..5f04270612 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -34,7 +34,6 @@ type node interface { reduceNode(pos int, db common.TrieStorageInteractor) (node, bool, error) isEmptyOrNil() error print(writer io.Writer, index int, db common.TrieStorageInteractor) - getDirtyHashes(common.ModifiedHashes) error getChildren(db common.TrieStorageInteractor) ([]node, error) loadChildren(func([]byte) (node, error)) ([][]byte, []node, error) @@ -44,7 +43,7 @@ type node interface { getVersion() (core.TrieNodeVersion, error) collectLeavesForMigration(migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder) (bool, error) - commitDirty(level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) + commitDirty(level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, hashesCollector common.TrieHashesCollector, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) commitSnapshot(originDb common.TrieStorageInteractor, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, depthLevel int) error sizeInBytes() int diff --git a/trie/leafNode.go b/trie/leafNode.go index 6c9c1a4180..e34ff8aca9 100644 --- a/trie/leafNode.go +++ b/trie/leafNode.go @@ -72,6 +72,7 @@ func (ln *leafNode) commitDirty( _ byte, _ uint, goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, _ common.TrieStorageInteractor, targetDb common.BaseStorer, ) { @@ -87,6 +88,7 @@ func (ln *leafNode) commitDirty( } hash := ln.hasher.Compute(string(encNode)) ln.hash = hash + hashesCollector.AddDirtyHash(hash) err = targetDb.Put(hash, encNode) if err != nil { @@ -346,20 +348,6 @@ func (ln *leafNode) print(writer io.Writer, _ int, _ common.TrieStorageInteracto _, _ = fmt.Fprintf(writer, "L: key= %v, (%v) - %v\n", ln.Key, hex.EncodeToString(ln.hash), ln.dirty) } -func (ln *leafNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := ln.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !ln.isDirty() { - return nil - } - - hashes[string(ln.getHash())] = struct{}{} - return nil -} - func (ln *leafNode) getChildren(_ common.TrieStorageInteractor) ([]node, error) { return nil, nil } diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index 5ecb93c497..ffe8d99a7d 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common/errChan" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -132,7 +133,7 @@ func TestLeafNode_commit(t *testing.T) { ln.setHash(getTestGoroutinesManager()) manager := getTestGoroutinesManager() - ln.commitDirty(0, 5, manager, db, db) + ln.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) @@ -284,7 +285,7 @@ func TestLeafNode_insertInStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion("dog", "dogs")} @@ -304,7 +305,7 @@ func TestLeafNode_insertInStoredLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() marsh, hasher := getTestMarshalizerAndHasher() ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3}), "dog"), marsh, hasher) - ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs")} @@ -372,7 +373,7 @@ func TestLeafNode_deleteFromStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{{Key: []byte("dog")}} @@ -391,7 +392,7 @@ func TestLeafNode_deleteFromLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) wrongKey := []byte{1, 2, 3} data := []core.TrieData{{Key: wrongKey}} @@ -505,7 +506,7 @@ func TestInsertSameNodeShouldNotSetDirtyBnRoot(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -521,7 +522,7 @@ func TestInsertSameNodeShouldNotSetDirtyEnRoot(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("log"), []byte("wood")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -536,7 +537,7 @@ func TestInsertSameNodeShouldNotSetDirtyLnRoot(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -550,7 +551,7 @@ func TestLeafNode_deleteDifferentKeyShouldNotModifyTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("ddoe"), []byte{}) @@ -731,7 +732,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dogs")} - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -752,7 +753,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog")} - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -775,7 +776,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 3, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{3, 4, 5, 6, 16}), "dog"), } - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -805,7 +806,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{1, 2, 5, 6, 16}), "dog"), } - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -837,7 +838,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -859,7 +860,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - ln.commitDirty(0, 5, getTestGoroutinesManager(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) diff --git a/trie/node_test.go b/trie/node_test.go index fd5225c8ae..481b39bd84 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" dataMock "github.com/multiversx/mx-chain-go/dataRetriever/mock" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/trie/keyBuilder" @@ -136,7 +137,7 @@ func TestNode_getNodeFromDBAndDecodeBranchNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - bn.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := bn.marsh.Marshal(collapsedBn) encNode = append(encNode, branch) @@ -156,7 +157,7 @@ func TestNode_getNodeFromDBAndDecodeExtensionNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - en.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := en.marsh.Marshal(collapsedEn) encNode = append(encNode, extension) @@ -176,7 +177,7 @@ func TestNode_getNodeFromDBAndDecodeLeafNode(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) ln.setHash(getTestGoroutinesManager()) - ln.commitDirty(0, 5, getTestGoroutinesManager(), db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := ln.marsh.Marshal(ln) encNode = append(encNode, leaf) @@ -365,7 +366,7 @@ func TestGetOldHashesIfNodeIsCollapsed(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.GetRootNode().(*branchNode) for i := 0; i < nrOfChildren; i++ { @@ -383,61 +384,26 @@ func TestClearOldHashesAndOldRootOnCommit(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.RootHash() _ = tr.Update([]byte("dog"), []byte("value of dog")) ExecuteUpdatesFromBatch(tr) assert.Equal(t, 4, len(tr.GetOldHashes())) - assert.Equal(t, root, tr.GetOldRoot()) + assert.Equal(t, root, tr.GetOldRootHash()) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Equal(t, 0, len(tr.GetOldHashes())) assert.Equal(t, 0, len(tr.RootManager.(*rootManager).oldRootHash)) } -func TestTrieGetObsoleteHashes(t *testing.T) { - t.Parallel() - - tr := initTrie() - _ = tr.Commit() - - _ = tr.Update([]byte("doeee"), []byte("value of doeee")) - ExecuteUpdatesFromBatch(tr) - - assert.NotEqual(t, 0, len(tr.GetOldHashes())) - assert.NotEqual(t, 0, len(tr.GetOldRoot())) - - expectedHashes := tr.GetOldHashes() - hashes := tr.GetObsoleteHashes() - assert.Equal(t, expectedHashes, hashes) -} - -func TestNode_getDirtyHashes(t *testing.T) { - t.Parallel() - - tr, _ := newEmptyTrie() - _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Update([]byte("ddog"), []byte("cat")) - ExecuteUpdatesFromBatch(tr) - - tr.GetRootNode().setHash(getTestGoroutinesManager()) - hashes := make(map[string]struct{}) - err := tr.GetRootNode().getDirtyHashes(hashes) - - assert.Nil(t, err) - assert.NotNil(t, hashes) - assert.Equal(t, 6, len(hashes)) -} - func TestPatriciaMerkleTrie_GetAllLeavesCollapsedTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.GetRootNode().(*branchNode) for i := 0; i < nrOfChildren; i++ { @@ -470,12 +436,12 @@ func TestPatriciaMerkleTrie_oldRootAndOldHashesAreResetAfterEveryCommit(t *testi t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) _ = tr.Update([]byte("doe"), []byte("deer")) _ = tr.Update([]byte("doe"), []byte("reindeer")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) assert.Equal(t, 0, len(tr.GetOldHashes())) assert.Equal(t, 0, len(tr.RootManager.(*rootManager).oldRootHash)) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index c3e8befe04..196d9d5bed 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -101,6 +101,7 @@ func NewTrie( batchManager: trieBatchManager.NewTrieBatchManager(), goroutinesThrottler: trieThrottler, trieOperationInProgress: &atomic.Flag{}, + trieMutex: sync.RWMutex{}, }, nil } @@ -316,7 +317,7 @@ func (tr *patriciaMerkleTrie) getRootHash() ([]byte, error) { } // Commit adds all the dirty nodes to the database -func (tr *patriciaMerkleTrie) Commit() error { +func (tr *patriciaMerkleTrie) Commit(hashesCollector common.TrieHashesCollector) error { tr.trieOperationInProgress.SetValue(true) defer tr.trieOperationInProgress.Reset() @@ -341,7 +342,7 @@ func (tr *patriciaMerkleTrie) Commit() error { return nil } - tr.ResetCollectedHashes() + oldRootHash := tr.GetOldRootHash() if log.GetLevel() == logger.LogTrace { log.Trace("started committing trie", "trie", rootNode.getHash()) } @@ -350,8 +351,20 @@ func (tr *patriciaMerkleTrie) Commit() error { if err != nil { return err } - rootNode.commitDirty(0, tr.maxTrieLevelInMemory, manager, tr.trieStorage, tr.trieStorage) - return manager.GetError() + rootNode.commitDirty(0, tr.maxTrieLevelInMemory, manager, hashesCollector, tr.trieStorage, tr.trieStorage) + err = manager.GetError() + if err != nil { + return err + } + + oldHashes := tr.GetOldHashes() + hashesCollector.AddObsoleteHashes(oldRootHash, oldHashes) + + logArrayWithTrace("old trie hash", "hash", oldHashes) + logMapWithTrace("new trie hash", "hash", hashesCollector.GetDirtyHashes()) + + tr.ResetCollectedHashes() + return nil } // Recreate returns a new trie that has the given root hash and database @@ -403,7 +416,7 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( } // String outputs a graphical view of the trie. Mainly used in tests/debugging -func (tr *patriciaMerkleTrie) String() string { +func (tr *patriciaMerkleTrie) ToString() string { tr.trieOperationInProgress.SetValue(true) defer tr.trieOperationInProgress.Reset() @@ -430,59 +443,6 @@ func (tr *patriciaMerkleTrie) IsInterfaceNil() bool { return tr == nil } -// GetObsoleteHashes resets the oldHashes and oldRoot variables and returns the old hashes -func (tr *patriciaMerkleTrie) GetObsoleteHashes() [][]byte { - tr.trieOperationInProgress.SetValue(true) - defer tr.trieOperationInProgress.Reset() - - err := tr.updateTrie() - if err != nil { - log.Warn("get obsolete hashes - could not save batched changes", "error", err) - } - - oldHashes := tr.GetOldHashes() - logArrayWithTrace("old trie hash", "hash", oldHashes) - - return oldHashes -} - -// GetDirtyHashes returns all the dirty hashes from the trie -func (tr *patriciaMerkleTrie) GetDirtyHashes() (common.ModifiedHashes, error) { - tr.trieOperationInProgress.SetValue(true) - defer tr.trieOperationInProgress.Reset() - - err := tr.updateTrie() - if err != nil { - return nil, err - } - - rootNode := tr.GetRootNode() - if rootNode == nil { - return nil, nil - } - - manager, err := NewGoroutinesManager(tr.goroutinesThrottler, errChan.NewErrChanWrapper(), tr.chanClose) - if err != nil { - return nil, err - } - - rootNode.setHash(manager) - err = manager.GetError() - if err != nil { - return nil, err - } - - dirtyHashes := make(common.ModifiedHashes) - err = rootNode.getDirtyHashes(dirtyHashes) - if err != nil { - return nil, err - } - - logMapWithTrace("new trie hash", "hash", dirtyHashes) - - return dirtyHashes, nil -} - func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.StorageManager) (*patriciaMerkleTrie, snapshotNode, error) { newTr, err := NewTrie( tsm, @@ -746,11 +706,6 @@ func (tr *patriciaMerkleTrie) GetStorageManager() common.StorageManager { return tr.trieStorage } -// GetOldRoot returns the rootHash of the trie before the latest changes -func (tr *patriciaMerkleTrie) GetOldRoot() []byte { - return tr.GetOldRootHash() -} - // GetTrieStats will collect and return the statistics for the given rootHash func (tr *patriciaMerkleTrie) GetTrieStats(address string, rootHash []byte) (common.TrieStatisticsHandler, error) { newTrie, err := tr.recreate(rootHash, tr.trieStorage) diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 5ec63bdf16..43d12132fe 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -21,6 +21,7 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" errorsCommon "github.com/multiversx/mx-chain-go/errors" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/storageManager" @@ -73,7 +74,7 @@ func initTrieMultipleValues(nr int) (common.Trie, [][]byte) { func initTrie() common.Trie { tr := emptyTrie() addDefaultDataToTrie(tr) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return tr } @@ -208,7 +209,7 @@ func TestPatriciaMerkleTree_DeleteEmptyTrie(t *testing.T) { tr := emptyTrie() tr.Delete([]byte("dog")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -291,7 +292,7 @@ func TestPatriciaMerkleTree_Commit(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -304,7 +305,7 @@ func TestPatriciaMerkleTree_CommitCollapsesTrieOk(t *testing.T) { _ = tr.Update([]byte("doggo"), []byte("doggo")) _ = tr.Update([]byte("doggless"), []byte("doggless")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -313,8 +314,8 @@ func TestPatriciaMerkleTree_CommitAfterCommit(t *testing.T) { tr := initTrie() - _ = tr.Commit() - err := tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -323,7 +324,7 @@ func TestPatriciaMerkleTree_CommitEmptyRoot(t *testing.T) { tr := emptyTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -332,7 +333,7 @@ func TestPatriciaMerkleTree_GetAfterCommit(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) val, _, err := tr.Get([]byte("dog")) @@ -346,7 +347,7 @@ func TestPatriciaMerkleTree_InsertAfterCommit(t *testing.T) { tr1 := initTrie() tr2 := initTrie() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) _ = tr1.Update([]byte("doge"), []byte("coin")) @@ -364,7 +365,7 @@ func TestPatriciaMerkleTree_DeleteAfterCommit(t *testing.T) { tr1 := initTrie() tr2 := initTrie() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) tr1.Delete([]byte("dogglesworth")) @@ -381,11 +382,11 @@ func TestPatriciaMerkleTree_DeleteNotPresent(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) tr.Delete([]byte("adog")) - err = tr.Commit() + err = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -394,7 +395,7 @@ func TestPatriciaMerkleTrie_Recreate(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) newTr, err := tr.Recreate(rootHash) assert.Nil(t, err) @@ -422,7 +423,7 @@ func TestPatriciaMerkleTrie_RecreateFromEpoch(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHashHolder := holders.NewRootHashHolder(rootHash, core.OptionalUint32{}) newTr, err := tr.RecreateFromEpoch(rootHashHolder) @@ -436,7 +437,7 @@ func TestPatriciaMerkleTrie_RecreateFromEpoch(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) optionalUint32 := core.OptionalUint32{ Value: 5, @@ -465,7 +466,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodes(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() maxBuffToSend := uint64(500) @@ -479,7 +480,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodesTinyBufferShouldNotGetAllNodes(t * t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() maxBuffToSend := uint64(150) @@ -493,11 +494,11 @@ func TestPatriciaMerkleTrie_String(t *testing.T) { t.Parallel() tr := initTrie() - str := tr.String() + str := tr.ToString() assert.NotEqual(t, 0, len(str)) tr = emptyTrie() - str = tr.String() + str = tr.ToString() assert.Equal(t, "*** EMPTY TRIE ***\n", str) } @@ -512,14 +513,14 @@ func TestPatriciaMerkleTree_reduceBranchNodeReturnsOldHashesCorrectly(t *testing tr := emptyTrie() _ = tr.Update(key1, val1) _ = tr.Update(key2, val2) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) _ = tr.Update(key1, nil) _ = tr.Update(key1, val1) + hc := hashesCollector.NewDataTrieHashesCollector() + _ = tr.Commit(hc) - oldHashes := tr.GetObsoleteHashes() - newHashes, _ := tr.GetDirtyHashes() - + _, oldHashes, newHashes := hc.GetCollectedData() assert.Equal(t, len(oldHashes), len(newHashes)) } @@ -610,7 +611,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -646,7 +647,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { tr := emptyTrie() _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -695,7 +696,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { "dog": []byte("puppy"), "ddog": []byte("cat"), } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -720,7 +721,7 @@ func TestPatriciaMerkleTree_Prove(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, value, err := tr.GetProof([]byte("dog"), rootHash) @@ -734,7 +735,7 @@ func TestPatriciaMerkleTree_ProveCollapsedTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, err := tr.GetProof([]byte("dog"), rootHash) @@ -757,7 +758,7 @@ func TestPatriciaMerkleTree_VerifyProof(t *testing.T) { t.Parallel() tr, val := initTrieMultipleValues(50) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() for i := range val { @@ -780,7 +781,7 @@ func TestPatriciaMerkleTrie_VerifyProofBranchNodeWantHashShouldWork(t *testing.T _ = tr.Update([]byte("dog"), []byte("cat")) _ = tr.Update([]byte("zebra"), []byte("horse")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, _ := tr.GetProof([]byte("dog"), rootHash) @@ -796,7 +797,7 @@ func TestPatriciaMerkleTrie_VerifyProofExtensionNodeWantHashShouldWork(t *testin _ = tr.Update([]byte("dog"), []byte("cat")) _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, _ := tr.GetProof([]byte("dog"), rootHash) @@ -840,7 +841,7 @@ func TestPatriciaMerkleTrie_VerifyProofFromDifferentTrieShouldNotWork(t *testing _ = tr2.Update([]byte("doe"), []byte("reindeer")) _ = tr2.Update([]byte("dog"), []byte("puppy")) _ = tr2.Update([]byte("dogglesworth"), []byte("caterpillar")) - _ = tr2.Commit() + _ = tr2.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash2, _ := tr2.RootHash() rootHash, _ := tr1.RootHash() @@ -866,7 +867,7 @@ func TestPatriciaMerkleTrie_GetAndVerifyProof(t *testing.T) { _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() for i := 0; i < numRuns; i++ { randNum := rand.Intn(nrLeaves) @@ -889,7 +890,7 @@ func TestPatriciaMerkleTrie_GetAndVerifyProof(t *testing.T) { } func dumpTrieContents(tr common.Trie, values [][]byte) { - fmt.Println(tr.String()) + fmt.Println(tr.ToString()) for _, val := range values { fmt.Println(val) } @@ -903,7 +904,7 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { _ = tr.Update([]byte("dog"), []byte("reindeer")) _ = tr.Update([]byte("fog"), []byte("puppy")) _ = tr.Update([]byte("dogglesworth"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() address := "address" @@ -921,20 +922,6 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { assert.Equal(t, uint32(3), stats.GetMaxTrieDepth()) } -func TestPatriciaMerkleTrie_GetOldRoot(t *testing.T) { - t.Parallel() - - tr := emptyTrie() - _ = tr.Update([]byte("eod"), []byte("reindeer")) - _ = tr.Update([]byte("god"), []byte("puppy")) - _ = tr.Commit() - expecterOldRoot, _ := tr.RootHash() - - _ = tr.Update([]byte("eggod"), []byte("cat")) - trie.ExecuteUpdatesFromBatch(tr) - assert.Equal(t, expecterOldRoot, tr.GetOldRoot()) -} - func TestPatriciaMerkleTree_GetValueReturnsTrieDepth(t *testing.T) { t.Parallel() @@ -954,7 +941,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) numOperations := 1000 wg := sync.WaitGroup{} wg.Add(numOperations) @@ -980,7 +967,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { _, err := tr.RootHash() assert.Nil(t, err) case 4: - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) case 5: _, err := tr.Recreate(initialRootHash) @@ -994,7 +981,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { _, err := tr.RecreateFromEpoch(rootHashHolder) assert.Nil(t, err) //case 7: - // _ = tr.String() + // _ = tr.ToString() //case 8: // _ = tr.GetObsoleteHashes() //case 9: @@ -1374,7 +1361,7 @@ func TestPatriciaMerkleTrie_CollectLeavesForMigration(t *testing.T) { }, ) addDefaultDataToTrie(tr) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() collapsedTrie, _ := tr.Recreate(rootHash) dtr := collapsedTrie.(dataTrie) @@ -1566,7 +1553,7 @@ func BenchmarkPatriciaMerkleTree_InsertCollapsedTrie(b *testing.B) { for i := 0; i < nrValuesNotInTrie; i++ { values[i] = hsh.Compute(strconv.Itoa(i + nrValuesInTrie)) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1604,7 +1591,7 @@ func BenchmarkPatriciaMerkleTree_DeleteCollapsedTrie(b *testing.B) { _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1641,7 +1628,7 @@ func BenchmarkPatriciaMerkleTree_GetCollapsedTrie(b *testing.B) { values[i] = hsh.Compute(strconv.Itoa(i)) _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1661,7 +1648,7 @@ func BenchmarkPatriciaMerkleTree_Commit(b *testing.B) { } b.StartTimer() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) } } @@ -1680,7 +1667,7 @@ func BenchmarkPatriciaMerkleTrie_RootHashAfterChanging30000Nodes(b *testing.B) { _ = tr.Update(key, value) values[i] = key } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1709,7 +1696,7 @@ func BenchmarkPatriciaMerkleTrie_RootHashAfterChanging30000NodesInBatchesOf200(b _ = tr.Update(key, value) values[i] = key } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/trie/rootManager.go b/trie/rootManager.go index b9231e0ae6..2d7f3d2297 100644 --- a/trie/rootManager.go +++ b/trie/rootManager.go @@ -1,8 +1,9 @@ package trie import ( - "github.com/multiversx/mx-chain-core-go/core/check" "sync" + + "github.com/multiversx/mx-chain-core-go/core/check" ) type rootManager struct { @@ -58,9 +59,7 @@ func (rm *rootManager) GetOldHashes() [][]byte { rm.mutOperation.RLock() defer rm.mutOperation.RUnlock() - oldHashes := make([][]byte, len(rm.oldHashes)) - copy(oldHashes, rm.oldHashes) - return oldHashes + return rm.oldHashes } func (rm *rootManager) GetOldRootHash() []byte { diff --git a/update/genesis/import.go b/update/genesis/import.go index 6092a7ceaa..a2e6c25eff 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "strings" "github.com/multiversx/mx-chain-core-go/core" @@ -352,7 +353,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b } if common.IsEmptyTrie(originalRootHash) { - err = dataTrie.Commit() + err = dataTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return err } @@ -386,7 +387,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w identifier: %s", err, identifier) } - err = dataTrie.Commit() + err = dataTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return err } diff --git a/update/sync/coordinator_test.go b/update/sync/coordinator_test.go index b56b2d8f99..15d4b62980 100644 --- a/update/sync/coordinator_test.go +++ b/update/sync/coordinator_test.go @@ -125,7 +125,7 @@ func createSyncTrieState(retErr bool) update.EpochStartTriesSyncHandler { RecreateAllTriesCalled: func(rootHash []byte) (map[string]common.Trie, error) { tries := make(map[string]common.Trie) tries[string(rootHash)] = &trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { if retErr { return errors.New("err") } @@ -140,7 +140,7 @@ func createSyncTrieState(retErr bool) update.EpochStartTriesSyncHandler { RecreateAllTriesCalled: func(rootHash []byte) (map[string]common.Trie, error) { tries := make(map[string]common.Trie) tries[string(rootHash)] = &trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { if retErr { return errors.New("err") } diff --git a/update/sync/syncAccountsDBs.go b/update/sync/syncAccountsDBs.go index 803460bd91..82e21adc17 100644 --- a/update/sync/syncAccountsDBs.go +++ b/update/sync/syncAccountsDBs.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie/storageMarker" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/genesis" @@ -194,7 +195,7 @@ func (st *syncAccountsDBs) tryRecreateTrie(shardId uint32, id string, trieID sta } for _, recreatedTrie := range tries { - err = recreatedTrie.Commit() + err = recreatedTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return false } From 4620444039912ce881ca9bf69b88123ea40f6859 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Fri, 22 Nov 2024 11:23:14 +0200 Subject: [PATCH 3/6] remove ToString from trie interface --- common/interface.go | 1 - .../state/genesisState/genesisState_test.go | 17 +++++++++++------ .../state/stateTrie/stateTrie_test.go | 7 ++++++- testscommon/trie/trieStub.go | 5 ----- trie/patriciaMerkleTrie.go | 8 +------- trie/patriciaMerkleTrie_test.go | 11 ++++++++--- 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/common/interface.go b/common/interface.go index cf04aaf885..c0d63f48e1 100644 --- a/common/interface.go +++ b/common/interface.go @@ -44,7 +44,6 @@ type Trie interface { Commit(collector TrieHashesCollector) error Recreate(root []byte) (Trie, error) RecreateFromEpoch(options RootHashHolder) (Trie, error) - ToString() string GetSerializedNodes([]byte, uint64) ([][]byte, uint64, error) GetSerializedNode([]byte) ([]byte, error) GetAllLeavesOnChannel(allLeavesChan *TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder KeyBuilder, trieLeafParser TrieLeafParser) error diff --git a/integrationTests/state/genesisState/genesisState_test.go b/integrationTests/state/genesisState/genesisState_test.go index cefe1c2e44..32b070f719 100644 --- a/integrationTests/state/genesisState/genesisState_test.go +++ b/integrationTests/state/genesisState/genesisState_test.go @@ -29,6 +29,11 @@ type testPair struct { const generate32ByteSlices = 0 const generate32HexByteSlices = 1 +type trieWithToString interface { + common.Trie + ToString() string +} + func TestCreationOfTheGenesisState(t *testing.T) { if testing.Short() { t.Skip("this is not a short test") @@ -85,7 +90,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { _ = tr1.Update([]byte(key3), []byte(val)) fmt.Println() - strTr1 := tr1.ToString() + strTr1 := tr1.(trieWithToString).ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -98,7 +103,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { fmt.Printf("root hash2: %s\n", base64.StdEncoding.EncodeToString(hash2)) fmt.Println() - strTr2 := tr2.ToString() + strTr2 := tr2.(trieWithToString).ToString() fmt.Println(strTr2) assert.Equal(t, hash1, hash2) @@ -126,7 +131,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr1.Update([]byte(key4), []byte(val)) fmt.Println() - strTr1 := tr1.ToString() + strTr1 := tr1.(trieWithToString).ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -140,7 +145,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr2.Update([]byte(key6), []byte(val)) fmt.Println() - strTr2 := tr2.ToString() + strTr2 := tr2.(trieWithToString).ToString() fmt.Println(strTr2) hash2, _ := tr2.RootHash() @@ -299,12 +304,12 @@ func printTestDebugLines( fmt.Println() fmt.Println("Reference trie:") - strRefTrie := referenceTrie.ToString() + strRefTrie := referenceTrie.(trieWithToString).ToString() fmt.Println(strRefTrie) fmt.Println() fmt.Println("Actual trie:") - strTr := tr.ToString() + strTr := tr.(trieWithToString).ToString() fmt.Println(strTr) } diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index db15862d79..3faf1403ce 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -962,6 +962,11 @@ func TestAccountsDB_ExecALotOfBalanceTxOKorNOK(t *testing.T) { integrationTests.PrintShardAccount(acntDest.(state.UserAccountHandler), "Destination") } +type trieWithToString interface { + common.Trie + ToString() string +} + func BenchmarkCreateOneMillionAccountsWithMockDB(b *testing.B) { nrOfAccounts := 1000000 balance := 1500000 @@ -995,7 +1000,7 @@ func BenchmarkCreateOneMillionAccountsWithMockDB(b *testing.B) { core.ConvertBytes(rtm.Sys), ) - _ = tr.ToString() + _ = tr.(trieWithToString).ToString() } func BenchmarkCreateOneMillionAccounts(b *testing.B) { diff --git a/testscommon/trie/trieStub.go b/testscommon/trie/trieStub.go index 185a5fa2ce..51e889e74f 100644 --- a/testscommon/trie/trieStub.go +++ b/testscommon/trie/trieStub.go @@ -148,11 +148,6 @@ func (ts *TrieStub) RecreateFromEpoch(options common.RootHashHolder) (common.Tri return nil, errNotImplemented } -// String - -func (ts *TrieStub) ToString() string { - return "stub trie" -} - // IsInterfaceNil returns true if there is no value under the interface func (ts *TrieStub) IsInterfaceNil() bool { return ts == nil diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 196d9d5bed..832a2dcf24 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -415,17 +415,11 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( return newTr, nil } -// String outputs a graphical view of the trie. Mainly used in tests/debugging +// ToString outputs a graphical view of the trie. Mainly used in tests/debugging func (tr *patriciaMerkleTrie) ToString() string { tr.trieOperationInProgress.SetValue(true) defer tr.trieOperationInProgress.Reset() - err := tr.updateTrie() - if err != nil { - log.Warn("print trie - could not save batched changes", "error", err) - return "" - } - writer := bytes.NewBuffer(make([]byte, 0)) rootNode := tr.GetRootNode() diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 43d12132fe..6b0d4c6772 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -490,15 +490,20 @@ func TestPatriciaMerkleTrie_GetSerializedNodesTinyBufferShouldNotGetAllNodes(t * assert.Equal(t, expectedNodes, len(serializedNodes)) } +type trieWithToString interface { + common.Trie + ToString() string +} + func TestPatriciaMerkleTrie_String(t *testing.T) { t.Parallel() tr := initTrie() - str := tr.ToString() + str := tr.(trieWithToString).ToString() assert.NotEqual(t, 0, len(str)) tr = emptyTrie() - str = tr.ToString() + str = tr.(trieWithToString).ToString() assert.Equal(t, "*** EMPTY TRIE ***\n", str) } @@ -890,7 +895,7 @@ func TestPatriciaMerkleTrie_GetAndVerifyProof(t *testing.T) { } func dumpTrieContents(tr common.Trie, values [][]byte) { - fmt.Println(tr.ToString()) + fmt.Println(tr.(trieWithToString).ToString()) for _, val := range values { fmt.Println(val) } From 804ea57721ed145802ee212604789b193259c866 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Wed, 27 Nov 2024 14:21:54 +0200 Subject: [PATCH 4/6] fix after merge --- trie/extensionNode_test.go | 1 + trie/patriciaMerkleTrie.go | 11 ++--- trie/patriciaMerkleTrie_test.go | 77 +++++++++++++++------------------ 3 files changed, 39 insertions(+), 50 deletions(-) diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index 2425f7bfb1..187005b026 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/throttler" + "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 94d0505614..79fa85ae8b 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -47,7 +47,7 @@ type patriciaMerkleTrie struct { batchManager common.TrieBatchManager goroutinesThrottler core.Throttler trieOperationInProgress *atomic.Flag - trieMutex sync.RWMutex + updateTrieMutex sync.RWMutex maxTrieLevelInMemory uint chanClose chan struct{} @@ -101,7 +101,7 @@ func NewTrie( batchManager: trieBatchManager.NewTrieBatchManager(), goroutinesThrottler: trieThrottler, trieOperationInProgress: &atomic.Flag{}, - trieMutex: sync.RWMutex{}, + updateTrieMutex: sync.RWMutex{}, }, nil } @@ -178,8 +178,8 @@ func (tr *patriciaMerkleTrie) Delete(key []byte) { } func (tr *patriciaMerkleTrie) updateTrie() error { - tr.trieMutex.Lock() - defer tr.trieMutex.Unlock() + tr.updateTrieMutex.Lock() + defer tr.updateTrieMutex.Unlock() batch, err := tr.batchManager.MarkTrieUpdateInProgress() if err != nil { @@ -301,9 +301,6 @@ func (tr *patriciaMerkleTrie) getRootHash() ([]byte, error) { return common.EmptyTrieHash, nil } - tr.trieMutex.Lock() - defer tr.trieMutex.Unlock() - hash := rootNode.getHash() if hash != nil { return hash, nil diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 6b0d4c6772..c03f5307d1 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -950,7 +950,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { numOperations := 1000 wg := sync.WaitGroup{} wg.Add(numOperations) - numFunctions := 7 + numFunctions := 14 initialRootHash, _ := tr.RootHash() @@ -985,48 +985,39 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { rootHashHolder := holders.NewRootHashHolder(initialRootHash, epoch) _, err := tr.RecreateFromEpoch(rootHashHolder) assert.Nil(t, err) - //case 7: - // _ = tr.ToString() - //case 8: - // _ = tr.GetObsoleteHashes() - //case 9: - // _, err := tr.GetDirtyHashes() - // assert.Nil(t, err) - //case 10: - // _, err := tr.GetSerializedNode(initialRootHash) - // assert.Nil(t, err) - //case 11: - // size1KB := uint64(1024 * 1024) - // _, _, err := tr.GetSerializedNodes(initialRootHash, size1KB) - // assert.Nil(t, err) - //case 12: - // trieIteratorChannels := &common.TrieIteratorChannels{ - // LeavesChan: make(chan core.KeyValueHolder, 1000), - // ErrChan: errChan.NewErrChanWrapper(), - // } - // - // err := tr.GetAllLeavesOnChannel( - // trieIteratorChannels, - // context.Background(), - // initialRootHash, - // keyBuilder.NewKeyBuilder(), - // parsers.NewMainTrieLeafParser(), - // ) - // assert.Nil(t, err) - //case 13: - // _, _, _ = tr.GetProof(initialRootHash, initialRootHash) // this might error due to concurrent operations that change the roothash - //case 14: - // // extremely hard to compute an existing hash due to concurrent changes. - // _, _ = tr.VerifyProof([]byte("dog"), []byte("puppy"), [][]byte{[]byte("proof1")}) // this might error due to concurrent operations that change the roothash - //case 15: - // sm := tr.GetStorageManager() - // assert.NotNil(t, sm) - //case 16: - // _ = tr.GetOldRoot() - //case 17: - // trieStatsHandler := tr.(common.TrieStats) - // _, err := trieStatsHandler.GetTrieStats("address", initialRootHash) - // assert.Nil(t, err) + case 7: + _, err := tr.GetSerializedNode(initialRootHash) + assert.Nil(t, err) + case 8: + size1KB := uint64(1024 * 1024) + _, _, err := tr.GetSerializedNodes(initialRootHash, size1KB) + assert.Nil(t, err) + case 9: + trieIteratorChannels := &common.TrieIteratorChannels{ + LeavesChan: make(chan core.KeyValueHolder, 1000), + ErrChan: errChan.NewErrChanWrapper(), + } + + err := tr.GetAllLeavesOnChannel( + trieIteratorChannels, + context.Background(), + initialRootHash, + keyBuilder.NewKeyBuilder(), + parsers.NewMainTrieLeafParser(), + ) + assert.Nil(t, err) + case 10: + _, _, _ = tr.GetProof(initialRootHash, initialRootHash) // this might error due to concurrent operations that change the roothash + case 11: + // extremely hard to compute an existing hash due to concurrent changes. + _, _ = tr.VerifyProof([]byte("dog"), []byte("puppy"), [][]byte{[]byte("proof1")}) // this might error due to concurrent operations that change the roothash + case 12: + sm := tr.GetStorageManager() + assert.NotNil(t, sm) + case 13: + trieStatsHandler := tr.(common.TrieStats) + _, err := trieStatsHandler.GetTrieStats("address", initialRootHash) + assert.Nil(t, err) default: assert.Fail(t, fmt.Sprintf("invalid numFunctions value %d, operation: %d", numFunctions, operation)) } From 6ad3b9667570214b1eb5215d14855dec8303e660 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 12 Dec 2024 14:18:41 +0200 Subject: [PATCH 5/6] fix after merge --- trie/patriciaMerkleTrie_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 0ddf458aa5..b88cc01f4a 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -1763,7 +1763,7 @@ func TestPatriciaMerkleTrie_Get(t *testing.T) { for i := 0; i < numTrieValues; i++ { _ = tr.Update([]byte("dog"+strconv.Itoa(i)), []byte("reindeer"+strconv.Itoa(i))) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) // collapse the trie rootHash, _ := tr.RootHash() From 22912c93de4cfe64e8cbcf742b1e09674d4c08a7 Mon Sep 17 00:00:00 2001 From: BeniaminDrasovean Date: Thu, 12 Dec 2024 15:54:03 +0200 Subject: [PATCH 6/6] commitDirty end goroutine processing --- trie/branchNode.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/trie/branchNode.go b/trie/branchNode.go index 0639bb1026..debfcee80b 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -214,17 +214,19 @@ func (bn *branchNode) commitDirty( waitGroup.Add(1) go func(childPos int) { + defer func() { + goRoutinesManager.EndGoRoutineProcessing() + waitGroup.Done() + }() + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) if !goRoutinesManager.ShouldContinueProcessing() { - waitGroup.Done() return } bn.childrenMutexes[childPos].Lock() bn.EncodedChildren[childPos] = child.getHash() bn.childrenMutexes[childPos].Unlock() - - waitGroup.Done() }(i) }