From 6bb4c2fa3ab957ce6b88c15aafe80769f7f985a6 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 13 Jun 2024 13:06:55 +0200 Subject: [PATCH 1/4] core/state/pruner: reintreduce improvement in dumpRawTrieDescendants unintetionally removed by 1e2855b24d --- core/state/pruner/pruner.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 3604f6c94c..45e7369e5c 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -385,7 +385,10 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl output.Put(data.CodeHash, nil) } if data.Root != (common.Hash{}) { - storageTr, err := trie.NewStateTrie(trie.StorageTrieID(root, key, data.Root), sdb.TrieDB()) + // note: we are passing data.Root as stateRoot here, to skip the check for stateRoot existence in trie.newTrieReader, + // we already check that when opening state trie and reading the account node + trieID := trie.StorageTrieID(data.Root, key, data.Root) + storageTr, err := trie.NewStateTrie(trieID, sdb.TrieDB()) if err != nil { return err } From b113d158dd78aa4479d72125fba1457f6395d4c2 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 13 Jun 2024 14:18:14 +0200 Subject: [PATCH 2/4] core/state/pruner: add threads config to pruner --- core/state/pruner/pruner.go | 32 +++++++++++++++++--------------- eth/backend.go | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 45e7369e5c..bd4afdcfa3 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -24,7 +24,6 @@ import ( "math" "os" "path/filepath" - "runtime" "sync" "time" @@ -58,6 +57,7 @@ const ( type Config struct { Datadir string // The directory of the state database BloomSize uint64 // The Megabytes of memory allocated to bloom-filter + Threads int // maximum number of threads spawned in dumpRawTrieDescendants and removeOtherRoots } // Pruner is an offline tool to prune the stale state with the @@ -107,6 +107,10 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { if err != nil { return nil, err } + // sanitize threads number, if set too low + if config.Threads <= 0 { + config.Threads = 1 + } return &Pruner{ config: config, chainHeader: headBlock.Header(), @@ -124,7 +128,7 @@ func readStoredChainConfig(db ethdb.Database) *params.ChainConfig { return rawdb.ReadChainConfig(db, block0Hash) } -func removeOtherRoots(db ethdb.Database, rootsList []common.Hash, stateBloom *stateBloom) error { +func removeOtherRoots(db ethdb.Database, rootsList []common.Hash, stateBloom *stateBloom, threads int) error { chainConfig := readStoredChainConfig(db) var genesisBlockNum uint64 if chainConfig != nil { @@ -139,7 +143,6 @@ func removeOtherRoots(db ethdb.Database, rootsList []common.Hash, stateBloom *st return errors.New("failed to load head block") } blockRange := headBlock.NumberU64() - genesisBlockNum - threads := runtime.NumCPU() var wg sync.WaitGroup errors := make(chan error, threads) for thread := 0; thread < threads; thread++ { @@ -207,7 +210,7 @@ func removeOtherRoots(db ethdb.Database, rootsList []common.Hash, stateBloom *st } // Arbitrum: snaptree and root are for the final snapshot kept -func prune(snaptree *snapshot.Tree, allRoots []common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, start time.Time) error { +func prune(snaptree *snapshot.Tree, allRoots []common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, start time.Time, threads int) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered // out. A very small part of stale tries will also be filtered because of @@ -297,7 +300,7 @@ func prune(snaptree *snapshot.Tree, allRoots []common.Hash, maindb ethdb.Databas } // Clean up any false positives that are top-level state roots. - err := removeOtherRoots(maindb, allRoots, stateBloom) + err := removeOtherRoots(maindb, allRoots, stateBloom, threads) if err != nil { return err } @@ -333,7 +336,7 @@ func prune(snaptree *snapshot.Tree, allRoots []common.Hash, maindb ethdb.Databas } // We assume state blooms do not need the value, only the key -func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBloom) error { +func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBloom, threads int) error { sdb := state.NewDatabase(db) tr, err := sdb.OpenTrie(root) if err != nil { @@ -350,7 +353,6 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl // To do so, we create a semaphore out of a channel's buffer. // Before launching a new goroutine, we acquire the semaphore by taking an entry from this channel. // This channel doubles as a mechanism for the background goroutine to report an error on release. - threads := runtime.NumCPU() results := make(chan error, threads) for i := 0; i < threads; i++ { results <- nil @@ -448,7 +450,7 @@ func (p *Pruner) Prune(inputRoots []common.Hash) error { return err } if bloomExists { - return RecoverPruning(p.config.Datadir, p.db) + return RecoverPruning(p.config.Datadir, p.db, p.config.Threads) } // Retrieve all snapshot layers from the current HEAD. // In theory there are 128 difflayers + 1 disk layer present, @@ -514,14 +516,14 @@ func (p *Pruner) Prune(inputRoots []common.Hash) error { return err } } else { - if err := dumpRawTrieDescendants(p.db, root, p.stateBloom); err != nil { + if err := dumpRawTrieDescendants(p.db, root, p.stateBloom, p.config.Threads); err != nil { return err } } } // Traverse the genesis, put all genesis state entries into the // bloom filter too. - if err := extractGenesis(p.db, p.stateBloom); err != nil { + if err := extractGenesis(p.db, p.stateBloom, p.config.Threads); err != nil { return err } @@ -532,7 +534,7 @@ func (p *Pruner) Prune(inputRoots []common.Hash) error { return err } log.Info("State bloom filter committed", "name", filterName, "roots", roots) - return prune(p.snaptree, roots, p.db, p.stateBloom, filterName, start) + return prune(p.snaptree, roots, p.db, p.stateBloom, filterName, start, p.config.Threads) } // RecoverPruning will resume the pruning procedure during the system restart. @@ -542,7 +544,7 @@ func (p *Pruner) Prune(inputRoots []common.Hash) error { // pruning can be resumed. What's more if the bloom filter is constructed, the // pruning **has to be resumed**. Otherwise a lot of dangling nodes may be left // in the disk. -func RecoverPruning(datadir string, db ethdb.Database) error { +func RecoverPruning(datadir string, db ethdb.Database, threads int) error { exists, err := bloomFilterExists(datadir) if err != nil { return err @@ -581,12 +583,12 @@ func RecoverPruning(datadir string, db ethdb.Database) error { } log.Info("Loaded state bloom filter", "path", stateBloomPath, "roots", stateBloomRoots) - return prune(snaptree, stateBloomRoots, db, stateBloom, stateBloomPath, time.Now()) + return prune(snaptree, stateBloomRoots, db, stateBloom, stateBloomPath, time.Now(), threads) } // extractGenesis loads the genesis state and commits all the state entries // into the given bloomfilter. -func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { +func extractGenesis(db ethdb.Database, stateBloom *stateBloom, threads int) error { genesisHash := rawdb.ReadCanonicalHash(db, 0) if genesisHash == (common.Hash{}) { return errors.New("missing genesis hash") @@ -596,7 +598,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return errors.New("missing genesis block") } - return dumpRawTrieDescendants(db, genesis.Root(), stateBloom) + return dumpRawTrieDescendants(db, genesis.Root(), stateBloom, threads) } func bloomFilterPath(datadir string) string { diff --git a/eth/backend.go b/eth/backend.go index a8046dacd7..924aa71de1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -140,7 +140,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } // Try to recover offline state pruning only in hash-based. if scheme == rawdb.HashScheme { - if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { + if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, 1); err != nil { log.Error("Failed to recover state", "error", err) } } From fd4ac69ff22b024ba245ef0ac1f735780004b277 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 13 Jun 2024 17:40:12 +0200 Subject: [PATCH 3/4] core/state/prunner: add extra progress log when traversing a storage trie takes long --- core/state/pruner/pruner.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index bd4afdcfa3..d3327e5dc1 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -357,6 +358,7 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl for i := 0; i < threads; i++ { results <- nil } + var threadsRunning atomic.Int32 for accountIt.Next(true) { accountTrieHash := accountIt.Hash() @@ -399,14 +401,20 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl return err } go func() { + threadsRunning.Add(1) + defer threadsRunning.Add(-1) var err error defer func() { results <- err }() + threadStartedAt := time.Now() + threadLastLog := time.Now() + storageIt, err := storageTr.NodeIterator(nil) if err != nil { return } + var processedNodes uint64 for storageIt.Next(true) { storageTrieHash := storageIt.Hash() if storageTrieHash != (common.Hash{}) { @@ -416,6 +424,13 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl return } } + processedNodes++ + if time.Since(threadLastLog) > 5*time.Minute { + elapsedTotal := time.Since(startedAt) + elapsedThread := time.Since(threadStartedAt) + log.Info("traversing trie database - traversing storage trie taking long", "key", key, "elapsedTotal", elapsedTotal, "elapsedThread", elapsedThread, "processedNodes", processedNodes, "threadsRunning", threadsRunning.Load()) + threadLastLog = time.Now() + } } err = storageIt.Error() if err != nil { From 9793aa9c8f695c3f5361a0ed8cb80b8de0976d26 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 13 Jun 2024 23:40:48 +0200 Subject: [PATCH 4/4] core/state/pruner: use cleans cache when traversing state trie --- core/state/pruner/pruner.go | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index d3327e5dc1..ce53715f27 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/hashdb" ) const ( @@ -56,9 +57,10 @@ const ( // Config includes all the configurations for pruning. type Config struct { - Datadir string // The directory of the state database - BloomSize uint64 // The Megabytes of memory allocated to bloom-filter - Threads int // maximum number of threads spawned in dumpRawTrieDescendants and removeOtherRoots + Datadir string // The directory of the state database + BloomSize uint64 // The Megabytes of memory allocated to bloom-filter + Threads int // The maximum number of threads spawned in dumpRawTrieDescendants and removeOtherRoots + CleanCacheSize int // The Megabytes of clean cache size used in dumpRawTrieDescendants } // Pruner is an offline tool to prune the stale state with the @@ -337,8 +339,16 @@ func prune(snaptree *snapshot.Tree, allRoots []common.Hash, maindb ethdb.Databas } // We assume state blooms do not need the value, only the key -func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBloom, threads int) error { - sdb := state.NewDatabase(db) +func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBloom, config *Config) error { + // Offline pruning is only supported in legacy hash based scheme. + hashConfig := *hashdb.Defaults + hashConfig.CleanCacheSize = config.CleanCacheSize * 1024 * 1024 + trieConfig := &trie.Config{ + Preimages: false, + HashDB: &hashConfig, + } + sdb := state.NewDatabaseWithConfig(db, trieConfig) + defer sdb.TrieDB().Close() tr, err := sdb.OpenTrie(root) if err != nil { return err @@ -354,6 +364,7 @@ func dumpRawTrieDescendants(db ethdb.Database, root common.Hash, output *stateBl // To do so, we create a semaphore out of a channel's buffer. // Before launching a new goroutine, we acquire the semaphore by taking an entry from this channel. // This channel doubles as a mechanism for the background goroutine to report an error on release. + threads := config.Threads results := make(chan error, threads) for i := 0; i < threads; i++ { results <- nil @@ -531,14 +542,14 @@ func (p *Pruner) Prune(inputRoots []common.Hash) error { return err } } else { - if err := dumpRawTrieDescendants(p.db, root, p.stateBloom, p.config.Threads); err != nil { + if err := dumpRawTrieDescendants(p.db, root, p.stateBloom, &p.config); err != nil { return err } } } // Traverse the genesis, put all genesis state entries into the // bloom filter too. - if err := extractGenesis(p.db, p.stateBloom, p.config.Threads); err != nil { + if err := extractGenesis(p.db, p.stateBloom, &p.config); err != nil { return err } @@ -603,7 +614,7 @@ func RecoverPruning(datadir string, db ethdb.Database, threads int) error { // extractGenesis loads the genesis state and commits all the state entries // into the given bloomfilter. -func extractGenesis(db ethdb.Database, stateBloom *stateBloom, threads int) error { +func extractGenesis(db ethdb.Database, stateBloom *stateBloom, config *Config) error { genesisHash := rawdb.ReadCanonicalHash(db, 0) if genesisHash == (common.Hash{}) { return errors.New("missing genesis hash") @@ -613,7 +624,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom, threads int) erro return errors.New("missing genesis block") } - return dumpRawTrieDescendants(db, genesis.Root(), stateBloom, threads) + return dumpRawTrieDescendants(db, genesis.Root(), stateBloom, config) } func bloomFilterPath(datadir string) string {