diff --git a/common/interface.go b/common/interface.go index e0dd4ba4b88..7afba588e9f 100644 --- a/common/interface.go +++ b/common/interface.go @@ -41,13 +41,9 @@ type Trie interface { Update(key, value []byte) error Delete(key []byte) RootHash() ([]byte, error) - Commit() error + Commit(collector TrieHashesCollector) error Recreate(root []byte) (Trie, error) RecreateFromEpoch(options RootHashHolder) (Trie, error) - String() string - GetObsoleteHashes() [][]byte - GetDirtyHashes() (ModifiedHashes, error) - GetOldRoot() []byte GetSerializedNodes([]byte, uint64) ([][]byte, uint64, error) GetSerializedNode([]byte) ([]byte, error) GetAllLeavesOnChannel(allLeavesChan *TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder KeyBuilder, trieLeafParser TrieLeafParser) error @@ -410,3 +406,11 @@ type AtomicBytesSlice interface { Append(data [][]byte) Get() [][]byte } + +// TrieHashesCollector defines the methods needed for collecting trie hashes +type TrieHashesCollector interface { + AddDirtyHash(hash []byte) + GetDirtyHashes() ModifiedHashes + AddObsoleteHashes(oldRootHash []byte, oldHashes [][]byte) + GetCollectedData() ([]byte, ModifiedHashes, ModifiedHashes) +} diff --git a/integrationTests/benchmarks/loadFromTrie_test.go b/integrationTests/benchmarks/loadFromTrie_test.go index c3c7a99f573..75451f3f796 100644 --- a/integrationTests/benchmarks/loadFromTrie_test.go +++ b/integrationTests/benchmarks/loadFromTrie_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-go/common" disabledStatistics "github.com/multiversx/mx-chain-go/common/statistics/disabled" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/database" "github.com/multiversx/mx-chain-go/storage/storageunit" @@ -120,7 +121,7 @@ func insertKeysIntoTrie(t *testing.T, tr common.Trie, numTrieLevels int, numChil _, depth, _ := tr.Get(key) require.Equal(t, uint32(numTrieLevels), depth+1) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return key } diff --git a/integrationTests/longTests/storage/storage_test.go b/integrationTests/longTests/storage/storage_test.go index bea274856d8..e47fd932c52 100644 --- a/integrationTests/longTests/storage/storage_test.go +++ b/integrationTests/longTests/storage/storage_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/hashing/blake2b" "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/storage" "github.com/multiversx/mx-chain-go/trie" @@ -130,7 +131,7 @@ func TestWriteContinuouslyInTree(t *testing.T) { if i%written == 0 { endTime := time.Now() diff := endTime.Sub(startTime) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) fmt.Printf("Written %d, total %d in %f s\n", written, i, diff.Seconds()) startTime = time.Now() } diff --git a/integrationTests/state/genesisState/genesisState_test.go b/integrationTests/state/genesisState/genesisState_test.go index cf7c1b15197..32b070f719c 100644 --- a/integrationTests/state/genesisState/genesisState_test.go +++ b/integrationTests/state/genesisState/genesisState_test.go @@ -29,6 +29,11 @@ type testPair struct { const generate32ByteSlices = 0 const generate32HexByteSlices = 1 +type trieWithToString interface { + common.Trie + ToString() string +} + func TestCreationOfTheGenesisState(t *testing.T) { if testing.Short() { t.Skip("this is not a short test") @@ -85,7 +90,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { _ = tr1.Update([]byte(key3), []byte(val)) fmt.Println() - strTr1 := tr1.String() + strTr1 := tr1.(trieWithToString).ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -98,7 +103,7 @@ func TestExtensionNodeToBranchEdgeCaseSet1(t *testing.T) { fmt.Printf("root hash2: %s\n", base64.StdEncoding.EncodeToString(hash2)) fmt.Println() - strTr2 := tr2.String() + strTr2 := tr2.(trieWithToString).ToString() fmt.Println(strTr2) assert.Equal(t, hash1, hash2) @@ -126,7 +131,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr1.Update([]byte(key4), []byte(val)) fmt.Println() - strTr1 := tr1.String() + strTr1 := tr1.(trieWithToString).ToString() fmt.Println(strTr1) hash1, _ := tr1.RootHash() @@ -140,7 +145,7 @@ func TestExtensionNodeToBranchEdgeCaseSet2(t *testing.T) { _ = tr2.Update([]byte(key6), []byte(val)) fmt.Println() - strTr2 := tr2.String() + strTr2 := tr2.(trieWithToString).ToString() fmt.Println(strTr2) hash2, _ := tr2.RootHash() @@ -299,12 +304,12 @@ func printTestDebugLines( fmt.Println() fmt.Println("Reference trie:") - strRefTrie := referenceTrie.String() + strRefTrie := referenceTrie.(trieWithToString).ToString() fmt.Println(strRefTrie) fmt.Println() fmt.Println("Actual trie:") - strTr := tr.String() + strTr := tr.(trieWithToString).ToString() fmt.Println(strTr) } diff --git a/integrationTests/state/stateTrie/stateTrie_test.go b/integrationTests/state/stateTrie/stateTrie_test.go index d20b9d819e3..3faf1403ce9 100644 --- a/integrationTests/state/stateTrie/stateTrie_test.go +++ b/integrationTests/state/stateTrie/stateTrie_test.go @@ -36,6 +36,7 @@ import ( "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/storagePruningManager" @@ -279,7 +280,7 @@ func TestTrieDB_RecreateFromStorageShouldWork(t *testing.T) { _ = tr1.Update(key, value) h1, _ := tr1.RootHash() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) tr2, err := tr1.Recreate(h1) @@ -961,6 +962,11 @@ func TestAccountsDB_ExecALotOfBalanceTxOKorNOK(t *testing.T) { integrationTests.PrintShardAccount(acntDest.(state.UserAccountHandler), "Destination") } +type trieWithToString interface { + common.Trie + ToString() string +} + func BenchmarkCreateOneMillionAccountsWithMockDB(b *testing.B) { nrOfAccounts := 1000000 balance := 1500000 @@ -994,7 +1000,7 @@ func BenchmarkCreateOneMillionAccountsWithMockDB(b *testing.B) { core.ConvertBytes(rtm.Sys), ) - _ = tr.String() + _ = tr.(trieWithToString).ToString() } func BenchmarkCreateOneMillionAccounts(b *testing.B) { diff --git a/integrationTests/state/stateTrieClose/stateTrieClose_test.go b/integrationTests/state/stateTrieClose/stateTrieClose_test.go index 9d99a178484..0f6f4718ef7 100644 --- a/integrationTests/state/stateTrieClose/stateTrieClose_test.go +++ b/integrationTests/state/stateTrieClose/stateTrieClose_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/integrationTests" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/goroutines" @@ -28,7 +29,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { for i := 0; i < numLeavesToAdd; i++ { _ = tr.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) time.Sleep(time.Second * 2) // allow the commit go routines to finish completely as to not alter the further counters gc := goroutines.NewGoCounter(goroutines.TestsRelevantGoRoutines) @@ -70,7 +71,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { assert.Nil(t, err) _ = tr.Update([]byte("god"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() leavesChannel1 = &common.TrieIteratorChannels{ @@ -91,7 +92,7 @@ func TestPatriciaMerkleTrie_Close(t *testing.T) { assert.Nil(t, err) _ = tr.Update([]byte("eggod"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() leavesChannel2 := &common.TrieIteratorChannels{ diff --git a/integrationTests/state/stateTrieSync/stateTrieSync_test.go b/integrationTests/state/stateTrieSync/stateTrieSync_test.go index cb933aedba6..eccca46cfe6 100644 --- a/integrationTests/state/stateTrieSync/stateTrieSync_test.go +++ b/integrationTests/state/stateTrieSync/stateTrieSync_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests" "github.com/multiversx/mx-chain-go/process/factory" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/storage" @@ -103,7 +104,7 @@ func testNodeRequestInterceptTrieNodesWithMessenger(t *testing.T, version int) { _ = resolverTrie.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = resolverTrie.Commit() + _ = resolverTrie.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := resolverTrie.RootHash() numLeaves := getNumLeaves(t, resolverTrie, rootHash) @@ -224,7 +225,7 @@ func testNodeRequestInterceptTrieNodesWithMessengerNotSyncingShouldErr(t *testin _ = resolverTrie.Update([]byte(strconv.Itoa(i)), []byte(strconv.Itoa(i))) } - _ = resolverTrie.Commit() + _ = resolverTrie.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := resolverTrie.RootHash() numLeaves := getNumLeaves(t, resolverTrie, rootHash) diff --git a/integrationTests/vm/txsFee/migrateDataTrie_test.go b/integrationTests/vm/txsFee/migrateDataTrie_test.go index a33d57883d4..689215f2170 100644 --- a/integrationTests/vm/txsFee/migrateDataTrie_test.go +++ b/integrationTests/vm/txsFee/migrateDataTrie_test.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/integrationTests/vm" "github.com/multiversx/mx-chain-go/sharding" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" vmcommon "github.com/multiversx/mx-chain-vm-common-go" "github.com/stretchr/testify/require" ) @@ -110,7 +111,7 @@ func TestMigrateDataTrieBuiltInFunc(t *testing.T) { migrateDataTrie(t, testContext, sndAddr, gasPrice, gasLimit, vmcommon.Ok) testGasConsumed(t, testContext, gasLimit, 20000) - err = dtr.Commit() + err = dtr.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) rootHash, err = dtr.RootHash() @@ -162,7 +163,7 @@ func TestMigrateDataTrieBuiltInFunc(t *testing.T) { migrateDataTrie(t, testContext, sndAddr, gasPrice, gasLimit, vmcommon.Ok) numMigrateDataTrieCalls++ - err = dtr.Commit() + err = dtr.Commit(hashesCollector.NewDisabledHashesCollector()) require.Nil(t, err) rootHash, err = dtr.RootHash() diff --git a/state/accountsDB.go b/state/accountsDB.go index e42627c9d68..b8590c3ecd4 100644 --- a/state/accountsDB.go +++ b/state/accountsDB.go @@ -18,6 +18,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/trie/keyBuilder" "github.com/multiversx/mx-chain-go/trie/statistics" @@ -777,22 +778,26 @@ func (adb *AccountsDB) commit() ([]byte, error) { log.Trace("accountsDB.Commit started") adb.entries = make([]JournalEntry, 0) - oldHashes := make(common.ModifiedHashes) - newHashes := make(common.ModifiedHashes) + hc := hashesCollector.NewDisabledHashesCollector() + if adb.mainTrie.GetStorageManager().IsPruningEnabled() { + hc = hashesCollector.NewDataTrieHashesCollector() + } + // Step 1. commit all data tries dataTries := adb.dataTries.GetAll() for i := 0; i < len(dataTries); i++ { - err := adb.commitTrie(dataTries[i], oldHashes, newHashes) + err := dataTries[i].Commit(hc) if err != nil { return nil, err } } adb.dataTries.Reset() - oldRoot := adb.mainTrie.GetOldRoot() - // Step 2. commit main trie - err := adb.commitTrie(adb.mainTrie, oldHashes, newHashes) + if adb.mainTrie.GetStorageManager().IsPruningEnabled() { + hc = hashesCollector.NewHashesCollector(hc) + } + err := adb.mainTrie.Commit(hc) if err != nil { return nil, err } @@ -803,7 +808,7 @@ func (adb *AccountsDB) commit() ([]byte, error) { return nil, err } - err = adb.markForEviction(oldRoot, newRoot, oldHashes, newHashes) + err = adb.markForEviction(newRoot, hc) if err != nil { return nil, err } @@ -818,36 +823,14 @@ func (adb *AccountsDB) commit() ([]byte, error) { } func (adb *AccountsDB) markForEviction( - oldRoot []byte, newRoot []byte, - oldHashes common.ModifiedHashes, - newHashes common.ModifiedHashes, + collector common.TrieHashesCollector, ) error { if !adb.mainTrie.GetStorageManager().IsPruningEnabled() { return nil } - return adb.storagePruningManager.MarkForEviction(oldRoot, newRoot, oldHashes, newHashes) -} - -func (adb *AccountsDB) commitTrie(tr common.Trie, oldHashes common.ModifiedHashes, newHashes common.ModifiedHashes) error { - if adb.mainTrie.GetStorageManager().IsPruningEnabled() { - oldTrieHashes := tr.GetObsoleteHashes() - newTrieHashes, err := tr.GetDirtyHashes() - if err != nil { - return err - } - - for _, hash := range oldTrieHashes { - oldHashes[string(hash)] = struct{}{} - } - - for hash := range newTrieHashes { - newHashes[hash] = struct{}{} - } - } - - return tr.Commit() + return adb.storagePruningManager.MarkForEviction(newRoot, collector) } // RootHash returns the main trie's root hash diff --git a/state/accountsDB_test.go b/state/accountsDB_test.go index ffd5ae1457b..ad6e1f8d69d 100644 --- a/state/accountsDB_test.go +++ b/state/accountsDB_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/rand" + "encoding/hex" "errors" "fmt" mathRand "math/rand" @@ -26,6 +27,7 @@ import ( "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" "github.com/multiversx/mx-chain-go/state/factory" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/iteratorChannelsProvider" "github.com/multiversx/mx-chain-go/state/lastSnapshotMarker" "github.com/multiversx/mx-chain-go/state/parsers" @@ -863,7 +865,7 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { marshaller := &marshallerMock.MarshalizerMock{} serializedAccount, _ := marshaller.Marshal(stateMock.AccountWrapMock{}) trieStub := trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { commitCalled++ return nil @@ -882,7 +884,7 @@ func TestAccountsDB_CommitShouldCallCommitFromTrie(t *testing.T) { UpdateWithVersionCalled: func(key, value []byte, version core.TrieNodeVersion) error { return nil }, - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { commitCalled++ return nil @@ -1826,22 +1828,29 @@ func TestAccountsDB_TrieDatabasePruning(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _, err := adb.Commit() + assert.Nil(t, err) rootHash, _ := tr.RootHash() _ = tr.Update([]byte("dog"), []byte("doee")) - oldHashes := tr.GetObsoleteHashes() - assert.Equal(t, 4, len(oldHashes)) - _, err := adb.Commit() + _, err = adb.Commit() assert.Nil(t, err) adb.CancelPrune(rootHash, state.NewRoot) adb.PruneTrie(rootHash, state.OldRoot, state.NewPruningHandler(state.EnableDataRemoval)) time.Sleep(trieDbOperationDelay) - for i := range oldHashes { - encNode, errGet := tr.GetStorageManager().Get(oldHashes[i]) + oldHashes := []string{ + "aa07e09a40ff7d332c2c50fa1619c751629f06babc3c1793ab3db8dd0c298345", + "a21e06e29734f85c99da88ae69d71b532f3929c2f96a43b95813e8c077151d11", + "d2b566879acfb873caa9a2282d48d196aefaa2a6168a84447298c472a1ba8f1c", + "7f20b853d6edc10f3e6840c7779fe842b7e87b2ad6488d4bb2465c0e389b7272", + } + + for _, hexKey := range oldHashes { + key, _ := hex.DecodeString(hexKey) + encNode, errGet := tr.GetStorageManager().Get(key) assert.Nil(t, encNode) assert.NotNil(t, errGet) } @@ -1923,12 +1932,6 @@ func TestAccountsDB_Prune(t *testing.T) { assert.Equal(t, trie.ErrKeyNotFound, err) } -func mergeMaps(map1 common.ModifiedHashes, map2 common.ModifiedHashes) { - for key, val := range map2 { - map1[key] = val - } -} - func generateAccounts(t testing.TB, numAccounts int, adb state.AccountsAdapter) [][]byte { accountsAddresses := make([][]byte, numAccounts) for i := 0; i < numAccounts; i++ { @@ -1951,23 +1954,18 @@ func generateRandomByteArray(size int) []byte { return r } -func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.AccountsDB) common.ModifiedHashes { +func modifyDataTries(t *testing.T, accountsAddresses [][]byte, adb *state.AccountsDB) { acc, _ := adb.LoadAccount(accountsAddresses[0]) err := acc.(state.UserAccountHandler).SaveKeyValue([]byte("key1"), []byte("value1")) assert.Nil(t, err) err = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key2"), []byte("value2")) assert.Nil(t, err) _ = adb.SaveAccount(acc) - newHashes, _ := acc.(state.UserAccountHandler).DataTrie().(common.Trie).GetDirtyHashes() acc, _ = adb.LoadAccount(accountsAddresses[1]) err = acc.(state.UserAccountHandler).SaveKeyValue([]byte("key2"), []byte("value2")) assert.Nil(t, err) _ = adb.SaveAccount(acc) - newHashesDataTrie, _ := acc.(state.UserAccountHandler).DataTrie().(common.Trie).GetDirtyHashes() - mergeMaps(newHashes, newHashesDataTrie) - - return newHashes } func TestAccountsDB_GetCode(t *testing.T) { @@ -2112,7 +2110,7 @@ func TestAccountsDB_GetTrie(t *testing.T) { _, adb := getDefaultTrieAndAccountsDb() addresses := generateAccounts(t, 2, adb) - _ = modifyDataTries(t, addresses, adb) + modifyDataTries(t, addresses, adb) _, _ = adb.Commit() @@ -2693,10 +2691,10 @@ func prepareTrie(tr common.Trie, numKeys int) common.ModifiedHashes { _ = tr.Update([]byte(key), []byte(val)) } - hashes, _ := tr.GetDirtyHashes() - _ = tr.Commit() - - return hashes + hc := hashesCollector.NewDataTrieHashesCollector() + _ = tr.Commit(hc) + _, _, newHashes := hc.GetCollectedData() + return newHashes } func addDataTries(accountsAddresses [][]byte, adb *state.AccountsDB) { @@ -2814,7 +2812,8 @@ func TestAccountsDB_RevertTxWhichMigratesDataRemovesMigratedData(t *testing.T) { // check that the data trie was completely reverted. The rootHash of the user account should be present // in both old and new hashes. This means that after the revert, the rootHash is the same as before markForEvictionCalled := false - spm.MarkForEvictionCalled = func(_ []byte, _ []byte, oldHashes common.ModifiedHashes, newHashes common.ModifiedHashes) error { + spm.MarkForEvictionCalled = func(_ []byte, collector common.TrieHashesCollector) error { + _, oldHashes, newHashes := collector.GetCollectedData() _, ok := oldHashes[string(userAccRootHash)] require.True(t, ok) _, ok = newHashes[string(userAccRootHash)] diff --git a/state/hashesCollector/dataTrieHashesCollector.go b/state/hashesCollector/dataTrieHashesCollector.go new file mode 100644 index 00000000000..e0aff5d13cc --- /dev/null +++ b/state/hashesCollector/dataTrieHashesCollector.go @@ -0,0 +1,57 @@ +package hashesCollector + +import ( + "sync" + + "github.com/multiversx/mx-chain-go/common" +) + +type dataTrieHashesCollector struct { + oldHashes common.ModifiedHashes + newHashes common.ModifiedHashes + + sync.RWMutex +} + +// NewDataTrieHashesCollector creates a new instance of dataTrieHashesCollector. +// This collector is used to collect hashes related to the data trie. +func NewDataTrieHashesCollector() *dataTrieHashesCollector { + return &dataTrieHashesCollector{ + oldHashes: make(common.ModifiedHashes), + newHashes: make(common.ModifiedHashes), + } +} + +// AddDirtyHash adds the new hash to the collector. +func (hc *dataTrieHashesCollector) AddDirtyHash(hash []byte) { + hc.Lock() + defer hc.Unlock() + + hc.newHashes[string(hash)] = struct{}{} +} + +// GetDirtyHashes returns the new hashes. +func (hc *dataTrieHashesCollector) GetDirtyHashes() common.ModifiedHashes { + hc.RLock() + defer hc.RUnlock() + + return hc.newHashes +} + +// AddObsoleteHashes adds the old hashes to the collector. +func (hc *dataTrieHashesCollector) AddObsoleteHashes(_ []byte, oldHashes [][]byte) { + hc.Lock() + defer hc.Unlock() + + for _, hash := range oldHashes { + hc.oldHashes[string(hash)] = struct{}{} + } +} + +// GetCollectedData returns the old hashes and the new hashes. +func (hc *dataTrieHashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + hc.RLock() + defer hc.RUnlock() + + return nil, hc.oldHashes, hc.newHashes +} diff --git a/state/hashesCollector/disabledHashesCollector.go b/state/hashesCollector/disabledHashesCollector.go new file mode 100644 index 00000000000..af016ac7634 --- /dev/null +++ b/state/hashesCollector/disabledHashesCollector.go @@ -0,0 +1,29 @@ +package hashesCollector + +import "github.com/multiversx/mx-chain-go/common" + +type disabledHashesCollector struct { +} + +// NewDisabledHashesCollector creates a new instance of disabledHashesCollector. +func NewDisabledHashesCollector() common.TrieHashesCollector { + return &disabledHashesCollector{} +} + +// AddDirtyHash does nothing. +func (hc *disabledHashesCollector) AddDirtyHash(_ []byte) { +} + +// GetDirtyHashes returns an empty map. +func (hc *disabledHashesCollector) GetDirtyHashes() common.ModifiedHashes { + return make(common.ModifiedHashes) +} + +// AddObsoleteHashes does nothing. +func (hc *disabledHashesCollector) AddObsoleteHashes(_ []byte, _ [][]byte) { +} + +// GetCollectedData returns nil data +func (hc *disabledHashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + return nil, nil, nil +} diff --git a/state/hashesCollector/hashesCollector.go b/state/hashesCollector/hashesCollector.go new file mode 100644 index 00000000000..c7740d8e5f1 --- /dev/null +++ b/state/hashesCollector/hashesCollector.go @@ -0,0 +1,32 @@ +package hashesCollector + +import ( + "github.com/multiversx/mx-chain-go/common" +) + +type hashesCollector struct { + common.TrieHashesCollector + + oldRootHash []byte +} + +// NewHashesCollector creates a new instance of hashesCollector. +// This collector is used to collect hashes related to the main trie. +func NewHashesCollector(collector common.TrieHashesCollector) *hashesCollector { + return &hashesCollector{ + TrieHashesCollector: collector, + oldRootHash: nil, + } +} + +// AddObsoleteHashes adds the old root hash and the old hashes to the collector. +func (hc *hashesCollector) AddObsoleteHashes(oldRootHash []byte, oldHashes [][]byte) { + hc.TrieHashesCollector.AddObsoleteHashes(oldRootHash, oldHashes) + hc.oldRootHash = oldRootHash +} + +// GetCollectedData returns the old root hash, the old hashes and the new hashes. +func (hc *hashesCollector) GetCollectedData() ([]byte, common.ModifiedHashes, common.ModifiedHashes) { + _, oldHashes, newHashes := hc.TrieHashesCollector.GetCollectedData() + return hc.oldRootHash, oldHashes, newHashes +} diff --git a/state/interface.go b/state/interface.go index 06050f95fcb..7920c3c4ad5 100644 --- a/state/interface.go +++ b/state/interface.go @@ -147,7 +147,7 @@ type AtomicBuffer interface { // StoragePruningManager is used to manage all state pruning operations type StoragePruningManager interface { - MarkForEviction([]byte, []byte, common.ModifiedHashes, common.ModifiedHashes) error + MarkForEviction([]byte, common.TrieHashesCollector) error PruneTrie(rootHash []byte, identifier TriePruningIdentifier, tsm common.StorageManager, handler PruningHandler) CancelPrune(rootHash []byte, identifier TriePruningIdentifier, tsm common.StorageManager) Close() error diff --git a/state/storagePruningManager/disabled/disabledStoragePruningManager.go b/state/storagePruningManager/disabled/disabledStoragePruningManager.go index 6de7e2b0845..5229166a498 100644 --- a/state/storagePruningManager/disabled/disabledStoragePruningManager.go +++ b/state/storagePruningManager/disabled/disabledStoragePruningManager.go @@ -14,7 +14,7 @@ func NewDisabledStoragePruningManager() *disabledStoragePruningManager { } // MarkForEviction does nothing for this implementation -func (i *disabledStoragePruningManager) MarkForEviction(_ []byte, _ []byte, _ common.ModifiedHashes, _ common.ModifiedHashes) error { +func (i *disabledStoragePruningManager) MarkForEviction(_ []byte, _ common.TrieHashesCollector) error { return nil } diff --git a/state/storagePruningManager/storagePruningManager.go b/state/storagePruningManager/storagePruningManager.go index 757d04cc9ed..2f6d7fae8e6 100644 --- a/state/storagePruningManager/storagePruningManager.go +++ b/state/storagePruningManager/storagePruningManager.go @@ -44,11 +44,10 @@ func NewStoragePruningManager( // MarkForEviction adds the given hashes to the underlying evictionWaitingList func (spm *storagePruningManager) MarkForEviction( - oldRoot []byte, newRoot []byte, - oldHashes common.ModifiedHashes, - newHashes common.ModifiedHashes, + collector common.TrieHashesCollector, ) error { + oldRoot, oldHashes, newHashes := collector.GetCollectedData() if bytes.Equal(newRoot, oldRoot) { log.Trace("old root and new root are identical", "rootHash", newRoot) return nil diff --git a/state/syncer/userAccountSyncer_test.go b/state/syncer/userAccountSyncer_test.go index eefdd96778f..d7b51390f82 100644 --- a/state/syncer/userAccountSyncer_test.go +++ b/state/syncer/userAccountSyncer_test.go @@ -1,6 +1,7 @@ package syncer import ( + "github.com/multiversx/mx-chain-go/state/hashesCollector" "testing" "time" @@ -93,7 +94,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { value := []byte("value") _ = tr.Update(key, value) rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) args.Cacher = &testscommon.CacherStub{ GetCalled: func(key []byte) (value interface{}, ok bool) { diff --git a/state/syncer/userAccountsSyncer_test.go b/state/syncer/userAccountsSyncer_test.go index 176a4ec7497..af30d51f989 100644 --- a/state/syncer/userAccountsSyncer_test.go +++ b/state/syncer/userAccountsSyncer_test.go @@ -17,6 +17,7 @@ import ( "github.com/multiversx/mx-chain-go/config" "github.com/multiversx/mx-chain-go/state" "github.com/multiversx/mx-chain-go/state/accounts" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/state/syncer" "github.com/multiversx/mx-chain-go/testscommon" @@ -111,7 +112,7 @@ func getSerializedTrieNode( tr, _ := trie.NewTrie(tsm, marshaller, hasher, &enableEpochsHandlerMock.EnableEpochsHandlerStub{}, 5) _ = tr.Update(key, []byte("value")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return serializedLeafNode } @@ -248,7 +249,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), accountBytes) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) leavesChannels := &common.TrieIteratorChannels{ LeavesChan: make(chan core.KeyValueHolder, common.TrieLeavesChannelDefaultCapacity), @@ -305,7 +306,7 @@ func TestUserAccountsSyncer_SyncAccountDataTries(t *testing.T) { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), accountBytes) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) leavesChannels := &common.TrieIteratorChannels{ LeavesChan: make(chan core.KeyValueHolder, common.TrieLeavesChannelDefaultCapacity), @@ -364,7 +365,7 @@ func TestUserAccountsSyncer_MissingDataTrieNodeFound(t *testing.T) { value := []byte("value") _ = tr.Update(key, value) rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) args.Cacher = &testscommon.CacherStub{ GetCalled: func(key []byte) (value interface{}, ok bool) { diff --git a/testscommon/state/storagePruningManagerStub.go b/testscommon/state/storagePruningManagerStub.go index 92c697c5224..045d1392d79 100644 --- a/testscommon/state/storagePruningManagerStub.go +++ b/testscommon/state/storagePruningManagerStub.go @@ -7,16 +7,16 @@ import ( // StoragePruningManagerStub - type StoragePruningManagerStub struct { - MarkForEvictionCalled func(bytes []byte, bytes2 []byte, hashes common.ModifiedHashes, hashes2 common.ModifiedHashes) error + MarkForEvictionCalled func(bytes []byte, collector common.TrieHashesCollector) error PruneTrieCalled func(rootHash []byte, identifier state.TriePruningIdentifier, tsm common.StorageManager, handler state.PruningHandler) CancelPruneCalled func(rootHash []byte, identifier state.TriePruningIdentifier, tsm common.StorageManager) CloseCalled func() error } // MarkForEviction - -func (stub *StoragePruningManagerStub) MarkForEviction(bytes []byte, bytes2 []byte, hashes common.ModifiedHashes, hashes2 common.ModifiedHashes) error { +func (stub *StoragePruningManagerStub) MarkForEviction(bytes []byte, collector common.TrieHashesCollector) error { if stub.MarkForEvictionCalled != nil { - return stub.MarkForEvictionCalled(bytes, bytes2, hashes, hashes2) + return stub.MarkForEvictionCalled(bytes, collector) } return nil diff --git a/testscommon/trie/trieStub.go b/testscommon/trie/trieStub.go index 8ab3ab07c16..51e889e74f4 100644 --- a/testscommon/trie/trieStub.go +++ b/testscommon/trie/trieStub.go @@ -18,10 +18,9 @@ type TrieStub struct { UpdateWithVersionCalled func(key, value []byte, version core.TrieNodeVersion) error DeleteCalled func(key []byte) RootCalled func() ([]byte, error) - CommitCalled func() error + CommitCalled func(collector common.TrieHashesCollector) error RecreateCalled func(root []byte) (common.Trie, error) RecreateFromEpochCalled func(options common.RootHashHolder) (common.Trie, error) - GetObsoleteHashesCalled func() [][]byte AppendToOldHashesCalled func([][]byte) GetSerializedNodesCalled func([]byte, uint64) ([][]byte, uint64, error) GetAllLeavesOnChannelCalled func(leavesChannels *common.TrieIteratorChannels, ctx context.Context, rootHash []byte, keyBuilder common.KeyBuilder, trieLeafParser common.TrieLeafParser) error @@ -29,7 +28,6 @@ type TrieStub struct { VerifyProofCalled func(rootHash []byte, key []byte, proof [][]byte) (bool, error) GetStorageManagerCalled func() common.StorageManager GetSerializedNodeCalled func(bytes []byte) ([]byte, error) - GetOldRootCalled func() []byte CloseCalled func() error CollectLeavesForMigrationCalled func(args vmcommon.ArgsMigrateDataTrieLeaves) error IsMigratedToLatestVersionCalled func() (bool, error) @@ -124,9 +122,9 @@ func (ts *TrieStub) RootHash() ([]byte, error) { } // Commit - -func (ts *TrieStub) Commit() error { +func (ts *TrieStub) Commit(hc common.TrieHashesCollector) error { if ts.CommitCalled != nil { - return ts.CommitCalled() + return ts.CommitCalled(hc) } return errNotImplemented @@ -150,25 +148,11 @@ func (ts *TrieStub) RecreateFromEpoch(options common.RootHashHolder) (common.Tri return nil, errNotImplemented } -// String - -func (ts *TrieStub) String() string { - return "stub trie" -} - // IsInterfaceNil returns true if there is no value under the interface func (ts *TrieStub) IsInterfaceNil() bool { return ts == nil } -// GetObsoleteHashes resets the oldHashes and oldRoot variables and returns the old hashes -func (ts *TrieStub) GetObsoleteHashes() [][]byte { - if ts.GetObsoleteHashesCalled != nil { - return ts.GetObsoleteHashesCalled() - } - - return nil -} - // GetSerializedNodes - func (ts *TrieStub) GetSerializedNodes(hash []byte, maxBuffToSend uint64) ([][]byte, uint64, error) { if ts.GetSerializedNodesCalled != nil { @@ -177,11 +161,6 @@ func (ts *TrieStub) GetSerializedNodes(hash []byte, maxBuffToSend uint64) ([][]b return nil, 0, nil } -// GetDirtyHashes - -func (ts *TrieStub) GetDirtyHashes() (common.ModifiedHashes, error) { - return nil, nil -} - // SetNewHashes - func (ts *TrieStub) SetNewHashes(_ common.ModifiedHashes) { } @@ -195,15 +174,6 @@ func (ts *TrieStub) GetSerializedNode(bytes []byte) ([]byte, error) { return nil, nil } -// GetOldRoot - -func (ts *TrieStub) GetOldRoot() []byte { - if ts.GetOldRootCalled != nil { - return ts.GetOldRootCalled() - } - - return nil -} - // IsMigratedToLatestVersion - func (ts *TrieStub) IsMigratedToLatestVersion() (bool, error) { if ts.IsMigratedToLatestVersionCalled != nil { diff --git a/trie/baseIterator_test.go b/trie/baseIterator_test.go index c65fc89f0cc..a899d18f2b7 100644 --- a/trie/baseIterator_test.go +++ b/trie/baseIterator_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,13 +37,13 @@ func TestBaseIterator_HasNext(t *testing.T) { tr := emptyTrie() _ = tr.Update([]byte("dog"), []byte("dog")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewBaseIterator(tr, rootHash) assert.False(t, it.HasNext()) _ = tr.Update([]byte("doe"), []byte("doe")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ = tr.RootHash() it, _ = trie.NewBaseIterator(tr, rootHash) assert.True(t, it.HasNext()) @@ -68,7 +69,7 @@ func TestBaseIterator_GetHash(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewBaseIterator(tr, rootHash) @@ -84,7 +85,7 @@ func TestIterator_Search(t *testing.T) { _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) _ = tr.Update([]byte("ddoge"), []byte("foo")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) expectedHashes := []string{ "ecc2304769996585131ad6276c1422265813a2b79d60392130c4baa19a9b4e06", diff --git a/trie/branchNode.go b/trie/branchNode.go index 86ebe1aff4d..debfcee80ba 100644 --- a/trie/branchNode.go +++ b/trie/branchNode.go @@ -170,44 +170,94 @@ func (bn *branchNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(bn) } -func (bn *branchNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { +func (bn *branchNode) commitDirty( + level byte, + maxTrieLevelInMemory uint, + goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, + originDb common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { level++ - err := bn.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } if !bn.dirty { - return nil + return } - for i := range bn.children { - if bn.children[i] == nil { + waitGroup := sync.WaitGroup{} + + for i := 0; i < nrOfChildren; i++ { + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + bn.childrenMutexes[i].RLock() + child := bn.children[i] + bn.childrenMutexes[i].RUnlock() + + if child == nil { continue } - err = bn.children[i].commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) - if err != nil { - return err + if !goRoutinesManager.CanStartGoRoutine() { + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + bn.childrenMutexes[i].Lock() + bn.EncodedChildren[i] = child.getHash() + bn.childrenMutexes[i].Unlock() + + continue } + + waitGroup.Add(1) + go func(childPos int) { + defer func() { + goRoutinesManager.EndGoRoutineProcessing() + waitGroup.Done() + }() + + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + bn.childrenMutexes[childPos].Lock() + bn.EncodedChildren[childPos] = child.getHash() + bn.childrenMutexes[childPos].Unlock() + }(i) + } + + waitGroup.Wait() + bn.dirty = false - _, err = encodeNodeAndCommitToDB(bn, targetDb) + encNode, err := bn.getEncodedNode() if err != nil { - return err + goRoutinesManager.SetError(err) + return + } + hash := bn.hasher.Compute(string(encNode)) + bn.hash = hash + hashesCollector.AddDirtyHash(hash) + + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + return } + if uint(level) == maxTrieLevelInMemory { log.Trace("collapse branch node on commit") - var collapsedBn *branchNode - collapsedBn, err = bn.getCollapsedBn() - if err != nil { - return err + for i := range bn.children { + bn.childrenMutexes[i].Lock() + bn.children[i] = nil + bn.childrenMutexes[i].Unlock() } - - *bn = *collapsedBn } - return nil } func (bn *branchNode) commitSnapshot( @@ -749,31 +799,6 @@ func (bn *branchNode) print(writer io.Writer, index int, db common.TrieStorageIn } } -func (bn *branchNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := bn.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !bn.isDirty() { - return nil - } - - for i := range bn.children { - if bn.children[i] == nil { - continue - } - - err = bn.children[i].getDirtyHashes(hashes) - if err != nil { - return err - } - } - - hashes[string(bn.getHash())] = struct{}{} - return nil -} - func (bn *branchNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { err := bn.isEmptyOrNil() if err != nil { diff --git a/trie/branchNode_test.go b/trie/branchNode_test.go index 47ea35a9df5..6c6b471dd4d 100644 --- a/trie/branchNode_test.go +++ b/trie/branchNode_test.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" @@ -106,7 +107,7 @@ func initTrie() *patriciaMerkleTrie { _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return tr } @@ -270,8 +271,9 @@ func TestBranchNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedBn) bn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, marsh, hasher) @@ -280,24 +282,6 @@ func TestBranchNode_commit(t *testing.T) { assert.Equal(t, h1, h2) } -func TestBranchNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - bn := emptyDirtyBranchNode() - - err := bn.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyBranchNode)) -} - -func TestBranchNode_commitNilNode(t *testing.T) { - t.Parallel() - - var bn *branchNode - - err := bn.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilBranchNode)) -} - func TestBranchNode_getEncodedNode(t *testing.T) { t.Parallel() @@ -339,7 +323,7 @@ func TestBranchNode_resolveIfCollapsed(t *testing.T) { childPos := byte(2) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) resolved, _ := newLeafNode(getTrieDataWithDefaultVersion("dog", "dog"), bn.marsh, bn.hasher) resolved.dirty = false resolved.hash = bn.EncodedChildren[childPos] @@ -428,7 +412,7 @@ func TestBranchNode_tryGetCollapsedNode(t *testing.T) { bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -534,7 +518,7 @@ func TestBranchNode_insertCollapsedNode(t *testing.T) { key := append([]byte{childPos}, []byte("dog")...) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) goRoutinesManager := getTestGoroutinesManager() data := []core.TrieData{getTrieDataWithDefaultVersion(string(key), "dogs")} @@ -556,7 +540,7 @@ func TestBranchNode_insertInStoredBnOnExistingPos(t *testing.T) { childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(key, db) lnHash := ln.getHash() @@ -603,7 +587,7 @@ func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) { nilChildPos := byte(11) key := append([]byte{nilChildPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() expectedHashes := [][]byte{bnHash} @@ -684,7 +668,7 @@ func TestBranchNode_deleteFromStoredBn(t *testing.T) { childPos := byte(2) lnKey := append([]byte{childPos}, []byte("dog")...) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bnHash := bn.getHash() ln, _, _ := bn.getNext(lnKey, db) lnHash := ln.getHash() @@ -750,7 +734,7 @@ func TestBranchNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) childPos := byte(2) key := append([]byte{childPos}, []byte("dog")...) @@ -993,7 +977,7 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) { _ = tr.Update([]byte("aaa"), []byte("aaa")) _ = tr.Update([]byte("nnn"), []byte("nnn")) _ = tr.Update([]byte("zzz"), []byte("zzz")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) collapsedRoot, _ := tr.GetRootNode().getCollapsed() tr.Delete([]byte("zzz")) @@ -1001,7 +985,7 @@ func TestPatriciaMerkleTrie_CommitCollapsedDirtyTrieShouldWork(t *testing.T) { assert.True(t, collapsedRoot.isDirty()) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.False(t, collapsedRoot.isDirty()) } @@ -1094,8 +1078,9 @@ func TestBranchNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testin bn.setHash(getTestGoroutinesManager()) collapsedBn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 1, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedBn.EncodedChildren, bn.EncodedChildren) assert.Equal(t, collapsedBn.children, bn.children) @@ -1135,20 +1120,6 @@ func TestBranchNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { assert.Equal(t, bnWriter.Bytes(), collapsedBnWriter.Bytes()) } -func TestBranchNode_getDirtyHashesFromCleanNode(t *testing.T) { - t.Parallel() - - db := testscommon.NewMemDbMock() - bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) - bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) - dirtyHashes := make(common.ModifiedHashes) - - err := bn.getDirtyHashes(dirtyHashes) - assert.Nil(t, err) - assert.Equal(t, 0, len(dirtyHashes)) -} - func TestBranchNode_getNextHashAndKey(t *testing.T) { t.Parallel() @@ -1553,8 +1524,9 @@ func TestBranchNode_insertOnNilChild(t *testing.T) { db := testscommon.NewMemDbMock() bn, _ := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() assert.True(t, len(originalHash) > 0) @@ -1666,8 +1638,9 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { Version: core.AutoBalanceEnabled, }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) originalHash := bn.getHash() assert.True(t, len(originalHash) > 0) @@ -1721,8 +1694,9 @@ func TestBranchNode_insertOnExistingChild(t *testing.T) { Version: core.NotSpecified, }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) goRoutinesManager := getTestGoroutinesManager() @@ -1769,8 +1743,9 @@ func TestBranchNode_insertBatch(t *testing.T) { Value: []byte("value4"), }, } - err := bn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + bn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) assert.False(t, bn.dirty) goRoutinesManager := getTestGoroutinesManager() @@ -1808,7 +1783,7 @@ func getNewBn() *branchNode { bn, _ := newBranchNode(marsh, hasher) bn.children = children bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) return bn } diff --git a/trie/depthFirstSync_test.go b/trie/depthFirstSync_test.go index 456c1b1f3e8..de0861fdd2e 100644 --- a/trie/depthFirstSync_test.go +++ b/trie/depthFirstSync_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "sync" "testing" "time" @@ -69,7 +70,7 @@ func TestDepthFirstTrieSyncer_StartSyncingCanTimeout(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -87,7 +88,7 @@ func TestDepthFirstTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -103,7 +104,7 @@ func TestDepthFirstTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { numKeysValues := 100 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -165,7 +166,7 @@ func TestDepthFirstTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi numKeysValues := 100 trSource, memUnitSource := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) diff --git a/trie/dfsIterator_test.go b/trie/dfsIterator_test.go index bc2f2b893fa..887e123cc15 100644 --- a/trie/dfsIterator_test.go +++ b/trie/dfsIterator_test.go @@ -2,7 +2,8 @@ package trie_test import ( "testing" - + + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" ) @@ -21,7 +22,7 @@ func TestNewDFSIterator(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, err := trie.NewDFSIterator(tr, rootHash) @@ -34,7 +35,7 @@ func TestDFSIterator_Next(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() it, _ := trie.NewDFSIterator(tr, rootHash) diff --git a/trie/doubleListSync_test.go b/trie/doubleListSync_test.go index 6bb2a5eb977..eea0fb75496 100644 --- a/trie/doubleListSync_test.go +++ b/trie/doubleListSync_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage" "github.com/multiversx/mx-chain-go/storage/storageunit" "github.com/multiversx/mx-chain-go/testscommon" @@ -151,7 +152,7 @@ func TestDoubleListTrieSyncer_StartSyncingCanTimeout(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -169,7 +170,7 @@ func TestDoubleListTrieSyncer_StartSyncingTimeoutNoNodesReceived(t *testing.T) { numKeysValues := 10 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -185,7 +186,7 @@ func TestDoubleListTrieSyncer_StartSyncingNewTrieShouldWork(t *testing.T) { numKeysValues := 100 trSource, _ := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) @@ -246,7 +247,7 @@ func TestDoubleListTrieSyncer_StartSyncingPartiallyFilledTrieShouldWork(t *testi numKeysValues := 100 trSource, memUnitSource := createInMemoryTrie() addDataToTrie(numKeysValues, trSource) - _ = trSource.Commit() + _ = trSource.Commit(hashesCollector.NewDisabledHashesCollector()) roothash, _ := trSource.RootHash() log.Info("source trie", "root hash", roothash) diff --git a/trie/export_test.go b/trie/export_test.go index 9489ffa6fb0..5d24134bbfb 100644 --- a/trie/export_test.go +++ b/trie/export_test.go @@ -1,8 +1,6 @@ package trie import ( - "time" - "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/statistics" @@ -34,25 +32,6 @@ func (ts *trieSyncer) trieNodeIntercepted(hash []byte, val interface{}) { } } -// WaitForOperationToComplete - -func WaitForOperationToComplete(tsm common.StorageManager) { - for tsm.IsPruningBlocked() { - time.Sleep(10 * time.Millisecond) - } -} - -// CreateSmallTestTrieAndStorageManager - -func CreateSmallTestTrieAndStorageManager() (*patriciaMerkleTrie, *trieStorageManager) { - tr, trieStorage := newEmptyTrie() - _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Update([]byte("dogglesworth"), []byte("cat")) - - _ = tr.Commit() - - return tr, trieStorage -} - // WriteInChanNonBlocking - func WriteInChanNonBlocking(errChan chan error, err error) { select { diff --git a/trie/extensionNode.go b/trie/extensionNode.go index 8dc44724418..73f38c78d56 100644 --- a/trie/extensionNode.go +++ b/trie/extensionNode.go @@ -127,41 +127,60 @@ func (en *extensionNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(en) } -func (en *extensionNode) commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error { +func (en *extensionNode) commitDirty( + level byte, + maxTrieLevelInMemory uint, + goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, + originDb common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { level++ - err := en.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } if !en.dirty { - return nil + return } - if en.child != nil { - err = en.child.commitDirty(level, maxTrieLevelInMemory, originDb, targetDb) - if err != nil { - return err + if !goRoutinesManager.ShouldContinueProcessing() { + return + } + + en.childMutex.RLock() + child := en.child + en.childMutex.RUnlock() + + if child != nil { + child.commitDirty(level, maxTrieLevelInMemory, goRoutinesManager, hashesCollector, originDb, targetDb) + if !goRoutinesManager.ShouldContinueProcessing() { + return } + + en.EncodedChild = child.getHash() } en.dirty = false - _, err = encodeNodeAndCommitToDB(en, targetDb) + encNode, err := en.getEncodedNode() if err != nil { - return err + goRoutinesManager.SetError(err) + return + } + hash := en.hasher.Compute(string(encNode)) + en.hash = hash + hashesCollector.AddDirtyHash(hash) + + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + return } + if uint(level) == maxTrieLevelInMemory { log.Trace("collapse extension node on commit") - var collapsedEn *extensionNode - collapsedEn, err = en.getCollapsedEn() - if err != nil { - return err - } - - *en = *collapsedEn + en.childMutex.Lock() + en.child = nil + en.childMutex.Unlock() } - return nil } func (en *extensionNode) commitSnapshot( @@ -624,29 +643,6 @@ func (en *extensionNode) print(writer io.Writer, index int, db common.TrieStorag en.child.print(writer, index+len(str), db) } -func (en *extensionNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := en.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !en.isDirty() { - return nil - } - - if en.child == nil { - return nil - } - - err = en.child.getDirtyHashes(hashes) - if err != nil { - return err - } - hashes[string(en.getHash())] = struct{}{} - - return nil -} - func (en *extensionNode) getChildren(db common.TrieStorageInteractor) ([]node, error) { err := en.isEmptyOrNil() if err != nil { diff --git a/trie/extensionNode_test.go b/trie/extensionNode_test.go index 3194830bbf3..946f587c63c 100644 --- a/trie/extensionNode_test.go +++ b/trie/extensionNode_test.go @@ -11,6 +11,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/throttler" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -183,8 +184,9 @@ func TestExtensionNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(collapsedEn) en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, en.marsh, en.hasher) @@ -194,24 +196,6 @@ func TestExtensionNode_commit(t *testing.T) { assert.Equal(t, h1, h2) } -func TestExtensionNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - en := &extensionNode{} - - err := en.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyExtensionNode)) -} - -func TestExtensionNode_commitNilNode(t *testing.T) { - t.Parallel() - - var en *extensionNode - - err := en.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilExtensionNode)) -} - func TestExtensionNode_commitCollapsedNode(t *testing.T) { t.Parallel() @@ -221,8 +205,9 @@ func TestExtensionNode_commitCollapsedNode(t *testing.T) { collapsedEn.setHash(getTestGoroutinesManager()) collapsedEn.dirty = true - err := collapsedEn.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + collapsedEn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, collapsedEn.marsh, collapsedEn.hasher) @@ -271,7 +256,7 @@ func TestExtensionNode_resolveCollapsed(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) _, resolved := getBnAndCollapsedBn(en.marsh, en.hasher) child, err := collapsedEn.resolveIfCollapsed(db) @@ -345,7 +330,7 @@ func TestExtensionNode_tryGetCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -416,7 +401,7 @@ func TestExtensionNode_insertCollapsedNode(t *testing.T) { key := []byte{100, 15, 5, 6} en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) goRoutinesManager := getTestGoroutinesManager() data := []core.TrieData{getTrieDataWithDefaultVersion(string(key), "dogs")} @@ -438,7 +423,7 @@ func TestExtensionNode_insertInStoredEnSameKey(t *testing.T) { key := append(enKey, []byte{11, 12}...) en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enHash := en.getHash() bn, _, _ := en.getNext(enKey, db) bnHash := bn.getHash() @@ -464,7 +449,7 @@ func TestExtensionNode_insertInStoredEnDifferentKey(t *testing.T) { nodeKey := []byte{11, 12} en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) expectedHashes := [][]byte{en.getHash()} goRoutinesManager := getTestGoroutinesManager() @@ -548,7 +533,7 @@ func TestExtensionNode_deleteFromStoredEn(t *testing.T) { lnPathKey := key en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) bn, key, _ := en.getNext(key, db) ln, _, _ := bn.getNext(key, db) expectedHashes := [][]byte{ln.getHash(), bn.getHash(), en.getHash()} @@ -583,7 +568,7 @@ func TestExtensionNode_deleteCollapsedNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) enKey := []byte{100} bnKey := []byte{2} @@ -625,13 +610,13 @@ func TestExtensionNode_reduceNodeCollapsedNode(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() collapsedTrie, _ := tr.Recreate(rootHash) collapsedTrie.Delete([]byte("doe")) - err := collapsedTrie.Commit() + err := collapsedTrie.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -670,7 +655,7 @@ func TestExtensionNode_getChildrenCollapsedEn(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) children, err := collapsedEn.getChildren(db) assert.Nil(t, err) @@ -693,7 +678,7 @@ func TestExtensionNode_loadChildren(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("ddog"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) tr.GetRootNode().setHash(getTestGoroutinesManager()) nodes, _ := getEncodedTrieNodesAndHashes(tr) nodesCacher, _ := cache.NewLRUCache(100) @@ -776,8 +761,9 @@ func TestExtensionNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *tes collapsedEn.setHash(getTestGoroutinesManager()) en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 1, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 1, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) assert.Equal(t, collapsedEn.EncodedChild, en.EncodedChild) assert.Equal(t, collapsedEn.child, en.child) @@ -794,7 +780,7 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) collapsedEn.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) _ = collapsedEn.commitSnapshot(db, nil, nil, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) en.print(enWriter, 0, db) @@ -803,20 +789,6 @@ func TestExtensionNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) { assert.Equal(t, enWriter.Bytes(), collapsedEnWriter.Bytes()) } -func TestExtensionNode_getDirtyHashesFromCleanNode(t *testing.T) { - t.Parallel() - - db := testscommon.NewMemDbMock() - en, _ := getEnAndCollapsedEn() - en.setHash(getTestGoroutinesManager()) - _ = en.commitSnapshot(db, nil, nil, context.Background(), statistics.NewTrieStatistics(), &testscommon.ProcessStatusHandlerStub{}, 0) - dirtyHashes := make(common.ModifiedHashes) - - err := en.getDirtyHashes(dirtyHashes) - assert.Nil(t, err) - assert.Equal(t, 0, len(dirtyHashes)) -} - func TestExtensionNode_getNextHashAndKey(t *testing.T) { t.Parallel() @@ -1083,8 +1055,9 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), @@ -1108,8 +1081,9 @@ func TestExtensionNode_insertInSameEn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 6, 7, 16}), "dog"), @@ -1145,8 +1119,9 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 3, 6, 7, 16}), "dog"), @@ -1180,8 +1155,9 @@ func TestExtensionNode_insertInNewBn(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{2, 3, 6, 7, 16}), "dog"), @@ -1214,8 +1190,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{2, 3, 6, 7, 16}), "dog"), @@ -1239,8 +1216,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { en := getEn() en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), @@ -1265,7 +1243,9 @@ func TestExtensionNode_deleteBatch(t *testing.T) { t.Parallel() en := getEn() - en.setHash(getTestGoroutinesManager()) + manager := getTestGoroutinesManager() + en.setHash(manager) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 4, 5, 6}), "dog"), } @@ -1275,9 +1255,10 @@ func TestExtensionNode_deleteBatch(t *testing.T) { assert.Nil(t, err) newEn := en.insert(data, goRoutinesManager, common.NewModifiedHashesSlice(initialModifiedHashesCapacity), nil) - newEn.setHash(getTestGoroutinesManager()) - err = newEn.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + newEn.setHash(manager) + assert.Nil(t, manager.GetError()) + newEn.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) dataForRemoval := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 7, 7, 8, 9}), "dog"), @@ -1298,9 +1279,11 @@ func TestExtensionNode_deleteBatch(t *testing.T) { t.Parallel() en := getEn() - en.setHash(getTestGoroutinesManager()) - err := en.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + en.setHash(manager) + assert.Nil(t, manager.GetError()) + en.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + assert.Nil(t, manager.GetError()) data := []core.TrieData{ getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 3, 4, 5}), "dog"), diff --git a/trie/interceptedNode_test.go b/trie/interceptedNode_test.go index ec413063aa6..071023f706f 100644 --- a/trie/interceptedNode_test.go +++ b/trie/interceptedNode_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core/check" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-go/common" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/trie" "github.com/stretchr/testify/assert" @@ -14,7 +15,7 @@ import ( func getDefaultInterceptedTrieNodeParameters() ([]byte, hashing.Hasher) { tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) nodes, _ := getEncodedTrieNodesAndHashes(tr) return nodes[0], &testscommon.KeccakMock{} diff --git a/trie/interface.go b/trie/interface.go index e77fae76c8f..ba2ec655610 100644 --- a/trie/interface.go +++ b/trie/interface.go @@ -34,7 +34,6 @@ type node interface { reduceNode(pos int, db common.TrieStorageInteractor) (node, bool, error) isEmptyOrNil() error print(writer io.Writer, index int, db common.TrieStorageInteractor) - getDirtyHashes(common.ModifiedHashes) error getChildren(db common.TrieStorageInteractor) ([]node, error) loadChildren(func([]byte) (node, error)) ([][]byte, []node, error) @@ -44,7 +43,7 @@ type node interface { getVersion() (core.TrieNodeVersion, error) collectLeavesForMigration(migrationArgs vmcommon.ArgsMigrateDataTrieLeaves, db common.TrieStorageInteractor, keyBuilder common.KeyBuilder) (bool, error) - commitDirty(level byte, maxTrieLevelInMemory uint, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) error + commitDirty(level byte, maxTrieLevelInMemory uint, goRoutinesManager common.TrieGoroutinesManager, hashesCollector common.TrieHashesCollector, originDb common.TrieStorageInteractor, targetDb common.BaseStorer) commitSnapshot(originDb common.TrieStorageInteractor, leavesChan chan core.KeyValueHolder, missingNodesChan chan []byte, ctx context.Context, stats common.TrieStatisticsHandler, idleProvider IdleNodeProvider, depthLevel int) error sizeInBytes() int diff --git a/trie/leafNode.go b/trie/leafNode.go index f7d0243dda1..c85e2b6e906 100644 --- a/trie/leafNode.go +++ b/trie/leafNode.go @@ -68,20 +68,32 @@ func (ln *leafNode) hashNode() ([]byte, error) { return encodeNodeAndGetHash(ln) } -func (ln *leafNode) commitDirty(_ byte, _ uint, _ common.TrieStorageInteractor, targetDb common.BaseStorer) error { - err := ln.isEmptyOrNil() - if err != nil { - return fmt.Errorf("commit error %w", err) - } - +func (ln *leafNode) commitDirty( + _ byte, + _ uint, + goRoutinesManager common.TrieGoroutinesManager, + hashesCollector common.TrieHashesCollector, + _ common.TrieStorageInteractor, + targetDb common.BaseStorer, +) { if !ln.dirty { - return nil + return } ln.dirty = false - _, err = encodeNodeAndCommitToDB(ln, targetDb) + encNode, err := ln.getEncodedNode() + if err != nil { + goRoutinesManager.SetError(err) + return + } + hash := ln.hasher.Compute(string(encNode)) + ln.hash = hash + hashesCollector.AddDirtyHash(hash) - return err + err = targetDb.Put(hash, encNode) + if err != nil { + goRoutinesManager.SetError(err) + } } func (ln *leafNode) commitSnapshot( @@ -343,20 +355,6 @@ func (ln *leafNode) print(writer io.Writer, _ int, _ common.TrieStorageInteracto _, _ = fmt.Fprintf(writer, "L: key= %v, (%v) - %v\n", ln.Key, hex.EncodeToString(ln.hash), ln.dirty) } -func (ln *leafNode) getDirtyHashes(hashes common.ModifiedHashes) error { - err := ln.isEmptyOrNil() - if err != nil { - return fmt.Errorf("getDirtyHashes error %w", err) - } - - if !ln.isDirty() { - return nil - } - - hashes[string(ln.getHash())] = struct{}{} - return nil -} - func (ln *leafNode) getChildren(_ common.TrieStorageInteractor) ([]node, error) { return nil, nil } diff --git a/trie/leafNode_test.go b/trie/leafNode_test.go index 0b14bc82e3e..01fc6bdd83f 100644 --- a/trie/leafNode_test.go +++ b/trie/leafNode_test.go @@ -12,6 +12,7 @@ import ( "github.com/multiversx/mx-chain-core-go/marshal" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/storage/cache" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/testscommon/hashingMocks" @@ -132,8 +133,9 @@ func TestLeafNode_commit(t *testing.T) { hash, _ := encodeNodeAndGetHash(ln) ln.setHash(getTestGoroutinesManager()) - err := ln.commitDirty(0, 5, db, db) - assert.Nil(t, err) + manager := getTestGoroutinesManager() + ln.commitDirty(0, 5, manager, hashesCollector.NewDisabledHashesCollector(), db, db) + assert.Nil(t, manager.GetError()) encNode, _ := db.Get(hash) n, _ := decodeNode(encNode, ln.marsh, ln.hasher) @@ -142,24 +144,6 @@ func TestLeafNode_commit(t *testing.T) { assert.Equal(t, ln, n) } -func TestLeafNode_commitEmptyNode(t *testing.T) { - t.Parallel() - - ln := &leafNode{} - - err := ln.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrEmptyLeafNode)) -} - -func TestLeafNode_commitNilNode(t *testing.T) { - t.Parallel() - - var ln *leafNode - - err := ln.commitDirty(0, 5, nil, nil) - assert.True(t, errors.Is(err, ErrNilLeafNode)) -} - func TestLeafNode_getEncodedNode(t *testing.T) { t.Parallel() @@ -296,7 +280,7 @@ func TestLeafNode_insertInStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion("dog", "dogs")} @@ -314,7 +298,7 @@ func TestLeafNode_insertInStoredLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() marsh, hasher := getTestMarshalizerAndHasher() ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3}), "dog"), marsh, hasher) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{4, 5, 6}), "dogs")} @@ -373,7 +357,7 @@ func TestLeafNode_deleteFromStoredLnAtSameKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) lnHash := ln.getHash() data := []core.TrieData{{Key: []byte("dog")}} @@ -390,7 +374,7 @@ func TestLeafNode_deleteFromLnAtDifferentKey(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) wrongKey := []byte{1, 2, 3} data := []core.TrieData{{Key: wrongKey}} @@ -497,7 +481,7 @@ func TestInsertSameNodeShouldNotSetDirtyBnRoot(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -513,7 +497,7 @@ func TestInsertSameNodeShouldNotSetDirtyEnRoot(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) _ = tr.Update([]byte("log"), []byte("wood")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -528,7 +512,7 @@ func TestInsertSameNodeShouldNotSetDirtyLnRoot(t *testing.T) { tr, _ := newEmptyTrie() _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("dog"), []byte("puppy")) @@ -542,7 +526,7 @@ func TestLeafNode_deleteDifferentKeyShouldNotModifyTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash := tr.GetRootNode().getHash() _ = tr.Update([]byte("ddoe"), []byte{}) @@ -723,7 +707,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dogs")} - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -745,7 +729,7 @@ func TestLeafNode_insertBatch(t *testing.T) { ln, _ := newLeafNode(getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog"), marshaller, hasher) newData := []core.TrieData{getTrieDataWithDefaultVersion(string([]byte{1, 2, 3, 4, 16}), "dog")} - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -769,7 +753,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 3, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{3, 4, 5, 6, 16}), "dog"), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -800,7 +784,7 @@ func TestLeafNode_insertBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{1, 2, 4, 5, 16}), "dog"), getTrieDataWithDefaultVersion(string([]byte{1, 2, 5, 6, 16}), "dog"), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) assert.False(t, ln.dirty) originalHash := ln.getHash() @@ -833,7 +817,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) originalHash := ln.getHash() th, _ := throttler.NewNumGoRoutinesThrottler(5) @@ -856,7 +840,7 @@ func TestLeafNode_deleteBatch(t *testing.T) { getTrieDataWithDefaultVersion(string([]byte{2, 2, 3, 4, 16}), ""), getTrieDataWithDefaultVersion(string([]byte{3, 2, 3, 4, 16}), ""), } - _ = ln.commitDirty(0, 5, testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), testscommon.NewMemDbMock(), testscommon.NewMemDbMock()) th, _ := throttler.NewNumGoRoutinesThrottler(5) goRoutinesManager, err := NewGoroutinesManager(th, errChan.NewErrChanWrapper(), make(chan struct{})) diff --git a/trie/node_test.go b/trie/node_test.go index 864ca002970..481b39bd841 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -13,6 +13,7 @@ import ( "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/common/errChan" dataMock "github.com/multiversx/mx-chain-go/dataRetriever/mock" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon" "github.com/multiversx/mx-chain-go/trie/keyBuilder" @@ -136,7 +137,7 @@ func TestNode_getNodeFromDBAndDecodeBranchNode(t *testing.T) { db := testscommon.NewMemDbMock() bn, collapsedBn := getBnAndCollapsedBn(getTestMarshalizerAndHasher()) bn.setHash(getTestGoroutinesManager()) - _ = bn.commitDirty(0, 5, db, db) + bn.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := bn.marsh.Marshal(collapsedBn) encNode = append(encNode, branch) @@ -156,7 +157,7 @@ func TestNode_getNodeFromDBAndDecodeExtensionNode(t *testing.T) { db := testscommon.NewMemDbMock() en, collapsedEn := getEnAndCollapsedEn() en.setHash(getTestGoroutinesManager()) - _ = en.commitDirty(0, 5, db, db) + en.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := en.marsh.Marshal(collapsedEn) encNode = append(encNode, extension) @@ -176,7 +177,7 @@ func TestNode_getNodeFromDBAndDecodeLeafNode(t *testing.T) { db := testscommon.NewMemDbMock() ln := getLn(getTestMarshalizerAndHasher()) ln.setHash(getTestGoroutinesManager()) - _ = ln.commitDirty(0, 5, db, db) + ln.commitDirty(0, 5, getTestGoroutinesManager(), hashesCollector.NewDisabledHashesCollector(), db, db) encNode, _ := ln.marsh.Marshal(ln) encNode = append(encNode, leaf) @@ -365,7 +366,7 @@ func TestGetOldHashesIfNodeIsCollapsed(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.GetRootNode().(*branchNode) for i := 0; i < nrOfChildren; i++ { @@ -383,61 +384,26 @@ func TestClearOldHashesAndOldRootOnCommit(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.RootHash() _ = tr.Update([]byte("dog"), []byte("value of dog")) ExecuteUpdatesFromBatch(tr) assert.Equal(t, 4, len(tr.GetOldHashes())) - assert.Equal(t, root, tr.GetOldRoot()) + assert.Equal(t, root, tr.GetOldRootHash()) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Equal(t, 0, len(tr.GetOldHashes())) assert.Equal(t, 0, len(tr.RootManager.(*rootManager).oldRootHash)) } -func TestTrieGetObsoleteHashes(t *testing.T) { - t.Parallel() - - tr := initTrie() - _ = tr.Commit() - - _ = tr.Update([]byte("doeee"), []byte("value of doeee")) - ExecuteUpdatesFromBatch(tr) - - assert.NotEqual(t, 0, len(tr.GetOldHashes())) - assert.NotEqual(t, 0, len(tr.GetOldRoot())) - - expectedHashes := tr.GetOldHashes() - hashes := tr.GetObsoleteHashes() - assert.Equal(t, expectedHashes, hashes) -} - -func TestNode_getDirtyHashes(t *testing.T) { - t.Parallel() - - tr, _ := newEmptyTrie() - _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Update([]byte("ddog"), []byte("cat")) - ExecuteUpdatesFromBatch(tr) - - tr.GetRootNode().setHash(getTestGoroutinesManager()) - hashes := make(map[string]struct{}) - err := tr.GetRootNode().getDirtyHashes(hashes) - - assert.Nil(t, err) - assert.NotNil(t, hashes) - assert.Equal(t, 6, len(hashes)) -} - func TestPatriciaMerkleTrie_GetAllLeavesCollapsedTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) root, _ := tr.GetRootNode().(*branchNode) for i := 0; i < nrOfChildren; i++ { @@ -470,12 +436,12 @@ func TestPatriciaMerkleTrie_oldRootAndOldHashesAreResetAfterEveryCommit(t *testi t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) _ = tr.Update([]byte("doe"), []byte("deer")) _ = tr.Update([]byte("doe"), []byte("reindeer")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) assert.Equal(t, 0, len(tr.GetOldHashes())) assert.Equal(t, 0, len(tr.RootManager.(*rootManager).oldRootHash)) diff --git a/trie/patriciaMerkleTrie.go b/trie/patriciaMerkleTrie.go index 9823b71caa7..22ac6db1d5c 100644 --- a/trie/patriciaMerkleTrie.go +++ b/trie/patriciaMerkleTrie.go @@ -107,6 +107,7 @@ func NewTrie( batchManager: trieBatchManager.NewTrieBatchManager(), goRoutinesManager: goRoutinesManager, trieOperationInProgress: &atomic.Flag{}, + updateTrieMutex: sync.RWMutex{}, }, nil } @@ -328,7 +329,7 @@ func (tr *patriciaMerkleTrie) getRootHash() ([]byte, error) { } // Commit adds all the dirty nodes to the database -func (tr *patriciaMerkleTrie) Commit() error { +func (tr *patriciaMerkleTrie) Commit(hashesCollector common.TrieHashesCollector) error { tr.trieOperationInProgress.SetValue(true) defer tr.trieOperationInProgress.Reset() @@ -341,7 +342,7 @@ func (tr *patriciaMerkleTrie) Commit() error { defer tr.updateTrieMutex.Unlock() rootNode := tr.GetRootNode() - if rootNode == nil { + if check.IfNil(rootNode) { log.Trace("trying to commit empty trie") return nil } @@ -353,23 +354,30 @@ func (tr *patriciaMerkleTrie) Commit() error { return nil } + oldRootHash := tr.GetOldRootHash() + if log.GetLevel() == logger.LogTrace { + log.Trace("started committing trie", "trie", rootNode.getHash()) + } + err = tr.goRoutinesManager.SetNewErrorChannel(errChan.NewErrChanWrapper()) if err != nil { return err } - rootNode.setHash(tr.goRoutinesManager) + rootNode.commitDirty(0, tr.maxTrieLevelInMemory, tr.goRoutinesManager, hashesCollector, tr.trieStorage, tr.trieStorage) err = tr.goRoutinesManager.GetError() if err != nil { return err } - tr.ResetCollectedHashes() - if log.GetLevel() == logger.LogTrace { - log.Trace("started committing trie", "trie", rootNode.getHash()) - } + oldHashes := tr.GetOldHashes() + hashesCollector.AddObsoleteHashes(oldRootHash, oldHashes) + + logArrayWithTrace("old trie hash", "hash", oldHashes) + logMapWithTrace("new trie hash", "hash", hashesCollector.GetDirtyHashes()) - return rootNode.commitDirty(0, tr.maxTrieLevelInMemory, tr.trieStorage, tr.trieStorage) + tr.ResetCollectedHashes() + return nil } // Recreate returns a new trie that has the given root hash and database @@ -420,8 +428,8 @@ func (tr *patriciaMerkleTrie) recreate(root []byte, tsm common.StorageManager) ( return newTr, nil } -// String outputs a graphical view of the trie. Mainly used in tests/debugging -func (tr *patriciaMerkleTrie) String() string { +// ToString outputs a graphical view of the trie. Mainly used in tests/debugging +func (tr *patriciaMerkleTrie) ToString() string { tr.trieOperationInProgress.SetValue(true) defer tr.trieOperationInProgress.Reset() @@ -445,65 +453,6 @@ func (tr *patriciaMerkleTrie) IsInterfaceNil() bool { return tr == nil } -// GetObsoleteHashes resets the oldHashes and oldRoot variables and returns the old hashes -func (tr *patriciaMerkleTrie) GetObsoleteHashes() [][]byte { - tr.trieOperationInProgress.SetValue(true) - defer tr.trieOperationInProgress.Reset() - - err := tr.updateTrie() - if err != nil { - log.Warn("get obsolete hashes - could not save batched changes", "error", err) - } - - tr.updateTrieMutex.Lock() - defer tr.updateTrieMutex.Unlock() - - oldHashes := tr.GetOldHashes() - logArrayWithTrace("old trie hash", "hash", oldHashes) - - return oldHashes -} - -// GetDirtyHashes returns all the dirty hashes from the trie -func (tr *patriciaMerkleTrie) GetDirtyHashes() (common.ModifiedHashes, error) { - tr.trieOperationInProgress.SetValue(true) - defer tr.trieOperationInProgress.Reset() - - err := tr.updateTrie() - if err != nil { - return nil, err - } - - tr.updateTrieMutex.Lock() - defer tr.updateTrieMutex.Unlock() - - rootNode := tr.GetRootNode() - if rootNode == nil { - return nil, nil - } - - err = tr.goRoutinesManager.SetNewErrorChannel(errChan.NewErrChanWrapper()) - if err != nil { - return nil, err - } - - rootNode.setHash(tr.goRoutinesManager) - err = tr.goRoutinesManager.GetError() - if err != nil { - return nil, err - } - - dirtyHashes := make(common.ModifiedHashes) - err = rootNode.getDirtyHashes(dirtyHashes) - if err != nil { - return nil, err - } - - logMapWithTrace("new trie hash", "hash", dirtyHashes) - - return dirtyHashes, nil -} - func (tr *patriciaMerkleTrie) recreateFromDb(rootHash []byte, tsm common.StorageManager) (*patriciaMerkleTrie, snapshotNode, error) { newTr, err := NewTrie( tsm, @@ -767,11 +716,6 @@ func (tr *patriciaMerkleTrie) GetStorageManager() common.StorageManager { return tr.trieStorage } -// GetOldRoot returns the rootHash of the trie before the latest changes -func (tr *patriciaMerkleTrie) GetOldRoot() []byte { - return tr.GetOldRootHash() -} - // GetTrieStats will collect and return the statistics for the given rootHash func (tr *patriciaMerkleTrie) GetTrieStats(address string, rootHash []byte) (common.TrieStatisticsHandler, error) { newTrie, err := tr.recreate(rootHash, tr.trieStorage) diff --git a/trie/patriciaMerkleTrie_test.go b/trie/patriciaMerkleTrie_test.go index 31c9d059571..b88cc01f4a0 100644 --- a/trie/patriciaMerkleTrie_test.go +++ b/trie/patriciaMerkleTrie_test.go @@ -21,6 +21,7 @@ import ( "github.com/multiversx/mx-chain-go/common/errChan" "github.com/multiversx/mx-chain-go/common/holders" errorsCommon "github.com/multiversx/mx-chain-go/errors" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/state/parsers" "github.com/multiversx/mx-chain-go/testscommon/enableEpochsHandlerMock" "github.com/multiversx/mx-chain-go/testscommon/storageManager" @@ -73,7 +74,7 @@ func initTrieMultipleValues(nr int) (common.Trie, [][]byte) { func initTrie() common.Trie { tr := emptyTrie() addDefaultDataToTrie(tr) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) return tr } @@ -208,7 +209,7 @@ func TestPatriciaMerkleTree_DeleteEmptyTrie(t *testing.T) { tr := emptyTrie() tr.Delete([]byte("dog")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -291,7 +292,7 @@ func TestPatriciaMerkleTree_Commit(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -304,7 +305,7 @@ func TestPatriciaMerkleTree_CommitCollapsesTrieOk(t *testing.T) { _ = tr.Update([]byte("doggo"), []byte("doggo")) _ = tr.Update([]byte("doggless"), []byte("doggless")) - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -313,8 +314,8 @@ func TestPatriciaMerkleTree_CommitAfterCommit(t *testing.T) { tr := initTrie() - _ = tr.Commit() - err := tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -323,7 +324,7 @@ func TestPatriciaMerkleTree_CommitEmptyRoot(t *testing.T) { tr := emptyTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -332,7 +333,7 @@ func TestPatriciaMerkleTree_GetAfterCommit(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) val, _, err := tr.Get([]byte("dog")) @@ -346,7 +347,7 @@ func TestPatriciaMerkleTree_InsertAfterCommit(t *testing.T) { tr1 := initTrie() tr2 := initTrie() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) _ = tr1.Update([]byte("doge"), []byte("coin")) @@ -364,7 +365,7 @@ func TestPatriciaMerkleTree_DeleteAfterCommit(t *testing.T) { tr1 := initTrie() tr2 := initTrie() - err := tr1.Commit() + err := tr1.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) tr1.Delete([]byte("dogglesworth")) @@ -381,11 +382,11 @@ func TestPatriciaMerkleTree_DeleteNotPresent(t *testing.T) { tr := initTrie() - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) tr.Delete([]byte("adog")) - err = tr.Commit() + err = tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) } @@ -394,7 +395,7 @@ func TestPatriciaMerkleTrie_Recreate(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) newTr, err := tr.Recreate(rootHash) assert.Nil(t, err) @@ -422,7 +423,7 @@ func TestPatriciaMerkleTrie_RecreateFromEpoch(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHashHolder := holders.NewRootHashHolder(rootHash, core.OptionalUint32{}) newTr, err := tr.RecreateFromEpoch(rootHashHolder) @@ -436,7 +437,7 @@ func TestPatriciaMerkleTrie_RecreateFromEpoch(t *testing.T) { tr := initTrie() rootHash, _ := tr.RootHash() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) optionalUint32 := core.OptionalUint32{ Value: 5, @@ -465,7 +466,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodes(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() maxBuffToSend := uint64(500) @@ -479,7 +480,7 @@ func TestPatriciaMerkleTrie_GetSerializedNodesTinyBufferShouldNotGetAllNodes(t * t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() maxBuffToSend := uint64(150) @@ -489,15 +490,20 @@ func TestPatriciaMerkleTrie_GetSerializedNodesTinyBufferShouldNotGetAllNodes(t * assert.Equal(t, expectedNodes, len(serializedNodes)) } +type trieWithToString interface { + common.Trie + ToString() string +} + func TestPatriciaMerkleTrie_String(t *testing.T) { t.Parallel() tr := initTrie() - str := tr.String() + str := tr.(trieWithToString).ToString() assert.NotEqual(t, 0, len(str)) tr = emptyTrie() - str = tr.String() + str = tr.(trieWithToString).ToString() assert.Equal(t, "*** EMPTY TRIE ***\n", str) } @@ -512,14 +518,14 @@ func TestPatriciaMerkleTree_reduceBranchNodeReturnsOldHashesCorrectly(t *testing tr := emptyTrie() _ = tr.Update(key1, val1) _ = tr.Update(key2, val2) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) _ = tr.Update(key1, nil) _ = tr.Update(key1, val1) + hc := hashesCollector.NewDataTrieHashesCollector() + _ = tr.Commit(hc) - oldHashes := tr.GetObsoleteHashes() - newHashes, _ := tr.GetDirtyHashes() - + _, oldHashes, newHashes := hc.GetCollectedData() assert.Equal(t, len(oldHashes), len(newHashes)) } @@ -610,7 +616,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -646,7 +652,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { tr := emptyTrie() _ = tr.Update([]byte("doe"), []byte("reindeer")) _ = tr.Update([]byte("dog"), []byte("puppy")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -695,7 +701,7 @@ func TestPatriciaMerkleTrie_GetAllLeavesOnChannel(t *testing.T) { "dog": []byte("puppy"), "ddog": []byte("cat"), } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() leavesChannel := &common.TrieIteratorChannels{ @@ -720,7 +726,7 @@ func TestPatriciaMerkleTree_Prove(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, value, err := tr.GetProof([]byte("dog"), rootHash) @@ -734,7 +740,7 @@ func TestPatriciaMerkleTree_ProveCollapsedTrie(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, err := tr.GetProof([]byte("dog"), rootHash) @@ -757,7 +763,7 @@ func TestPatriciaMerkleTree_VerifyProof(t *testing.T) { t.Parallel() tr, val := initTrieMultipleValues(50) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() for i := range val { @@ -780,7 +786,7 @@ func TestPatriciaMerkleTrie_VerifyProofBranchNodeWantHashShouldWork(t *testing.T _ = tr.Update([]byte("dog"), []byte("cat")) _ = tr.Update([]byte("zebra"), []byte("horse")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, _ := tr.GetProof([]byte("dog"), rootHash) @@ -796,7 +802,7 @@ func TestPatriciaMerkleTrie_VerifyProofExtensionNodeWantHashShouldWork(t *testin _ = tr.Update([]byte("dog"), []byte("cat")) _ = tr.Update([]byte("doe"), []byte("reindeer")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() proof, _, _ := tr.GetProof([]byte("dog"), rootHash) @@ -840,7 +846,7 @@ func TestPatriciaMerkleTrie_VerifyProofFromDifferentTrieShouldNotWork(t *testing _ = tr2.Update([]byte("doe"), []byte("reindeer")) _ = tr2.Update([]byte("dog"), []byte("puppy")) _ = tr2.Update([]byte("dogglesworth"), []byte("caterpillar")) - _ = tr2.Commit() + _ = tr2.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash2, _ := tr2.RootHash() rootHash, _ := tr1.RootHash() @@ -866,7 +872,7 @@ func TestPatriciaMerkleTrie_GetAndVerifyProof(t *testing.T) { _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() for i := 0; i < numRuns; i++ { randNum := rand.Intn(nrLeaves) @@ -889,7 +895,7 @@ func TestPatriciaMerkleTrie_GetAndVerifyProof(t *testing.T) { } func dumpTrieContents(tr common.Trie, values [][]byte) { - fmt.Println(tr.String()) + fmt.Println(tr.(trieWithToString).ToString()) for _, val := range values { fmt.Println(val) } @@ -903,7 +909,7 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { _ = tr.Update([]byte("dog"), []byte("reindeer")) _ = tr.Update([]byte("fog"), []byte("puppy")) _ = tr.Update([]byte("dogglesworth"), []byte("cat")) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() address := "address" @@ -921,20 +927,6 @@ func TestPatriciaMerkleTrie_GetTrieStats(t *testing.T) { assert.Equal(t, uint32(3), stats.GetMaxTrieDepth()) } -func TestPatriciaMerkleTrie_GetOldRoot(t *testing.T) { - t.Parallel() - - tr := emptyTrie() - _ = tr.Update([]byte("eod"), []byte("reindeer")) - _ = tr.Update([]byte("god"), []byte("puppy")) - _ = tr.Commit() - expecterOldRoot, _ := tr.RootHash() - - _ = tr.Update([]byte("eggod"), []byte("cat")) - trie.ExecuteUpdatesFromBatch(tr) - assert.Equal(t, expecterOldRoot, tr.GetOldRoot()) -} - func TestPatriciaMerkleTree_GetValueReturnsTrieDepth(t *testing.T) { t.Parallel() @@ -954,11 +946,11 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { t.Parallel() tr := initTrie() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) numOperations := 1000 wg := sync.WaitGroup{} wg.Add(numOperations) - numFunctions := 18 + numFunctions := 14 initialRootHash, _ := tr.RootHash() @@ -980,7 +972,7 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { _, err := tr.RootHash() assert.Nil(t, err) case 4: - err := tr.Commit() + err := tr.Commit(hashesCollector.NewDisabledHashesCollector()) assert.Nil(t, err) case 5: _, err := tr.Recreate(initialRootHash) @@ -994,20 +986,13 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { _, err := tr.RecreateFromEpoch(rootHashHolder) assert.Nil(t, err) case 7: - _ = tr.String() - case 8: - _ = tr.GetObsoleteHashes() - case 9: - _, err := tr.GetDirtyHashes() - assert.Nil(t, err) - case 10: _, err := tr.GetSerializedNode(initialRootHash) assert.Nil(t, err) - case 11: + case 8: size1KB := uint64(1024 * 1024) _, _, err := tr.GetSerializedNodes(initialRootHash, size1KB) assert.Nil(t, err) - case 12: + case 9: trieIteratorChannels := &common.TrieIteratorChannels{ LeavesChan: make(chan core.KeyValueHolder, 1000), ErrChan: errChan.NewErrChanWrapper(), @@ -1021,17 +1006,15 @@ func TestPatriciaMerkleTrie_ConcurrentOperations(t *testing.T) { parsers.NewMainTrieLeafParser(), ) assert.Nil(t, err) - case 13: + case 10: _, _, _ = tr.GetProof(initialRootHash, initialRootHash) // this might error due to concurrent operations that change the roothash - case 14: + case 11: // extremely hard to compute an existing hash due to concurrent changes. _, _ = tr.VerifyProof([]byte("dog"), []byte("puppy"), [][]byte{[]byte("proof1")}) // this might error due to concurrent operations that change the roothash - case 15: + case 12: sm := tr.GetStorageManager() assert.NotNil(t, sm) - case 16: - _ = tr.GetOldRoot() - case 17: + case 13: trieStatsHandler := tr.(common.TrieStats) _, err := trieStatsHandler.GetTrieStats("address", initialRootHash) assert.Nil(t, err) @@ -1374,7 +1357,7 @@ func TestPatriciaMerkleTrie_CollectLeavesForMigration(t *testing.T) { }, ) addDefaultDataToTrie(tr) - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) rootHash, _ := tr.RootHash() collapsedTrie, _ := tr.Recreate(rootHash) dtr := collapsedTrie.(dataTrie) @@ -1780,7 +1763,7 @@ func TestPatriciaMerkleTrie_Get(t *testing.T) { for i := 0; i < numTrieValues; i++ { _ = tr.Update([]byte("dog"+strconv.Itoa(i)), []byte("reindeer"+strconv.Itoa(i))) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) // collapse the trie rootHash, _ := tr.RootHash() @@ -1954,7 +1937,7 @@ func BenchmarkPatriciaMerkleTree_InsertCollapsedTrie(b *testing.B) { for i := 0; i < nrValuesNotInTrie; i++ { values[i] = hsh.Compute(strconv.Itoa(i + nrValuesInTrie)) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1992,7 +1975,7 @@ func BenchmarkPatriciaMerkleTree_DeleteCollapsedTrie(b *testing.B) { _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -2029,7 +2012,7 @@ func BenchmarkPatriciaMerkleTree_GetCollapsedTrie(b *testing.B) { values[i] = hsh.Compute(strconv.Itoa(i)) _ = tr.Update(values[i], values[i]) } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -2049,7 +2032,7 @@ func BenchmarkPatriciaMerkleTree_Commit(b *testing.B) { } b.StartTimer() - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) } } @@ -2068,7 +2051,7 @@ func BenchmarkPatriciaMerkleTrie_RootHashAfterChanging30000Nodes(b *testing.B) { _ = tr.Update(key, value) values[i] = key } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -2097,7 +2080,7 @@ func BenchmarkPatriciaMerkleTrie_RootHashAfterChanging30000NodesInBatchesOf200(b _ = tr.Update(key, value) values[i] = key } - _ = tr.Commit() + _ = tr.Commit(hashesCollector.NewDisabledHashesCollector()) b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/trie/rootManager.go b/trie/rootManager.go index 182ef20df87..148b9a523b9 100644 --- a/trie/rootManager.go +++ b/trie/rootManager.go @@ -65,9 +65,7 @@ func (rm *rootManager) GetOldHashes() [][]byte { rm.mutOperation.RLock() defer rm.mutOperation.RUnlock() - oldHashes := make([][]byte, len(rm.oldHashes)) - copy(oldHashes, rm.oldHashes) - return oldHashes + return rm.oldHashes } // GetOldRootHash returns the old root hash diff --git a/update/genesis/import.go b/update/genesis/import.go index 6092a7ceaaa..a2e6c25eff7 100644 --- a/update/genesis/import.go +++ b/update/genesis/import.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "strings" "github.com/multiversx/mx-chain-core-go/core" @@ -352,7 +353,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b } if common.IsEmptyTrie(originalRootHash) { - err = dataTrie.Commit() + err = dataTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return err } @@ -386,7 +387,7 @@ func (si *stateImport) importDataTrie(identifier string, shID uint32, keys [][]b return fmt.Errorf("%w identifier: %s", err, identifier) } - err = dataTrie.Commit() + err = dataTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return err } diff --git a/update/sync/coordinator_test.go b/update/sync/coordinator_test.go index b56b2d8f99a..15d4b629801 100644 --- a/update/sync/coordinator_test.go +++ b/update/sync/coordinator_test.go @@ -125,7 +125,7 @@ func createSyncTrieState(retErr bool) update.EpochStartTriesSyncHandler { RecreateAllTriesCalled: func(rootHash []byte) (map[string]common.Trie, error) { tries := make(map[string]common.Trie) tries[string(rootHash)] = &trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { if retErr { return errors.New("err") } @@ -140,7 +140,7 @@ func createSyncTrieState(retErr bool) update.EpochStartTriesSyncHandler { RecreateAllTriesCalled: func(rootHash []byte) (map[string]common.Trie, error) { tries := make(map[string]common.Trie) tries[string(rootHash)] = &trieMock.TrieStub{ - CommitCalled: func() error { + CommitCalled: func(_ common.TrieHashesCollector) error { if retErr { return errors.New("err") } diff --git a/update/sync/syncAccountsDBs.go b/update/sync/syncAccountsDBs.go index 803460bd914..82e21adc176 100644 --- a/update/sync/syncAccountsDBs.go +++ b/update/sync/syncAccountsDBs.go @@ -10,6 +10,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-go/common" "github.com/multiversx/mx-chain-go/state" + "github.com/multiversx/mx-chain-go/state/hashesCollector" "github.com/multiversx/mx-chain-go/trie/storageMarker" "github.com/multiversx/mx-chain-go/update" "github.com/multiversx/mx-chain-go/update/genesis" @@ -194,7 +195,7 @@ func (st *syncAccountsDBs) tryRecreateTrie(shardId uint32, id string, trieID sta } for _, recreatedTrie := range tries { - err = recreatedTrie.Commit() + err = recreatedTrie.Commit(hashesCollector.NewDisabledHashesCollector()) if err != nil { return false }