From fa44e50c84d2ddf4984555da39d58a104975eafb Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Thu, 14 Dec 2023 23:42:22 +0000 Subject: [PATCH 01/32] add prunning test --- cmd/conf/init.go | 57 +++++++ cmd/nitro/init.go | 288 +--------------------------------- cmd/nitro/nitro.go | 6 +- cmd/prunning/prunning.go | 250 +++++++++++++++++++++++++++++ go-ethereum | 2 +- system_tests/common_test.go | 4 +- system_tests/prunning_test.go | 121 ++++++++++++++ 7 files changed, 439 insertions(+), 289 deletions(-) create mode 100644 cmd/conf/init.go create mode 100644 cmd/prunning/prunning.go create mode 100644 system_tests/prunning_test.go diff --git a/cmd/conf/init.go b/cmd/conf/init.go new file mode 100644 index 0000000000..f228891b90 --- /dev/null +++ b/cmd/conf/init.go @@ -0,0 +1,57 @@ +package conf + +import ( + "time" + + "github.com/spf13/pflag" +) + +type InitConfig struct { + Force bool `koanf:"force"` + Url string `koanf:"url"` + DownloadPath string `koanf:"download-path"` + DownloadPoll time.Duration `koanf:"download-poll"` + DevInit bool `koanf:"dev-init"` + DevInitAddress string `koanf:"dev-init-address"` + DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` + Empty bool `koanf:"empty"` + AccountsPerSync uint `koanf:"accounts-per-sync"` + ImportFile string `koanf:"import-file"` + ThenQuit bool `koanf:"then-quit"` + Prune string `koanf:"prune"` + PruneBloomSize uint64 `koanf:"prune-bloom-size"` + ResetToMessage int64 `koanf:"reset-to-message"` +} + +var InitConfigDefault = InitConfig{ + Force: false, + Url: "", + DownloadPath: "/tmp/", + DownloadPoll: time.Minute, + DevInit: false, + DevInitAddress: "", + DevInitBlockNum: 0, + ImportFile: "", + AccountsPerSync: 100000, + ThenQuit: false, + Prune: "", + PruneBloomSize: 2048, + ResetToMessage: -1, +} + +func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { + f.Bool(prefix+".force", InitConfigDefault.Force, "if true: in case database exists init code will be reexecuted and genesis block compared to database") + f.String(prefix+".url", InitConfigDefault.Url, "url to download initializtion data - will poll if download fails") + f.String(prefix+".download-path", InitConfigDefault.DownloadPath, "path to save temp downloaded file") + f.Duration(prefix+".download-poll", InitConfigDefault.DownloadPoll, "how long to wait between polling attempts") + f.Bool(prefix+".dev-init", InitConfigDefault.DevInit, "init with dev data (1 account with balance) instead of file import") + f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") + f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") + f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") + f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") + f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") + f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") + f.String(prefix+".prune", InitConfigDefault.Prune, "pruning for a given use: \"full\" for full nodes serving RPC requests, or \"validator\" for validators") + f.Uint64(prefix+".prune-bloom-size", InitConfigDefault.PruneBloomSize, "the amount of memory in megabytes to use for the pruning bloom filter (higher values prune better)") + f.Int64(prefix+".reset-to-message", InitConfigDefault.ResetToMessage, "forces a reset to an old message height. Also set max-reorg-resequence-depth=0 to force re-reading messages") +} diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 1427ef161e..745e168375 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -10,93 +10,37 @@ import ( "fmt" "math/big" "os" - "reflect" - "regexp" "runtime" "strings" "sync" "time" + "github.com/offchainlabs/nitro/cmd/conf" + "github.com/offchainlabs/nitro/cmd/prunning" "github.com/offchainlabs/nitro/cmd/util" "github.com/cavaliergopher/grab/v3" extract "github.com/codeclysm/extract/v3" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" - "github.com/offchainlabs/nitro/arbnode/dataposter/storage" "github.com/offchainlabs/nitro/arbos/arbosState" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/ipfshelper" "github.com/offchainlabs/nitro/execution/gethexec" - "github.com/offchainlabs/nitro/staker" "github.com/offchainlabs/nitro/statetransfer" - "github.com/spf13/pflag" ) -type InitConfig struct { - Force bool `koanf:"force"` - Url string `koanf:"url"` - DownloadPath string `koanf:"download-path"` - DownloadPoll time.Duration `koanf:"download-poll"` - DevInit bool `koanf:"dev-init"` - DevInitAddress string `koanf:"dev-init-address"` - DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` - Empty bool `koanf:"empty"` - AccountsPerSync uint `koanf:"accounts-per-sync"` - ImportFile string `koanf:"import-file"` - ThenQuit bool `koanf:"then-quit"` - Prune string `koanf:"prune"` - PruneBloomSize uint64 `koanf:"prune-bloom-size"` - ResetToMessage int64 `koanf:"reset-to-message"` -} - -var InitConfigDefault = InitConfig{ - Force: false, - Url: "", - DownloadPath: "/tmp/", - DownloadPoll: time.Minute, - DevInit: false, - DevInitAddress: "", - DevInitBlockNum: 0, - ImportFile: "", - AccountsPerSync: 100000, - ThenQuit: false, - Prune: "", - PruneBloomSize: 2048, - ResetToMessage: -1, -} - -func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { - f.Bool(prefix+".force", InitConfigDefault.Force, "if true: in case database exists init code will be reexecuted and genesis block compared to database") - f.String(prefix+".url", InitConfigDefault.Url, "url to download initializtion data - will poll if download fails") - f.String(prefix+".download-path", InitConfigDefault.DownloadPath, "path to save temp downloaded file") - f.Duration(prefix+".download-poll", InitConfigDefault.DownloadPoll, "how long to wait between polling attempts") - f.Bool(prefix+".dev-init", InitConfigDefault.DevInit, "init with dev data (1 account with balance) instead of file import") - f.String(prefix+".dev-init-address", InitConfigDefault.DevInitAddress, "Address of dev-account. Leave empty to use the dev-wallet.") - f.Uint64(prefix+".dev-init-blocknum", InitConfigDefault.DevInitBlockNum, "Number of preinit blocks. Must exist in ancient database.") - f.Bool(prefix+".empty", InitConfigDefault.Empty, "init with empty state") - f.Bool(prefix+".then-quit", InitConfigDefault.ThenQuit, "quit after init is done") - f.String(prefix+".import-file", InitConfigDefault.ImportFile, "path for json data to import") - f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") - f.String(prefix+".prune", InitConfigDefault.Prune, "pruning for a given use: \"full\" for full nodes serving RPC requests, or \"validator\" for validators") - f.Uint64(prefix+".prune-bloom-size", InitConfigDefault.PruneBloomSize, "the amount of memory in megabytes to use for the pruning bloom filter (higher values prune better)") - f.Int64(prefix+".reset-to-message", InitConfigDefault.ResetToMessage, "forces a reset to an old message height. Also set max-reorg-resequence-depth=0 to force re-reading messages") -} - -func downloadInit(ctx context.Context, initConfig *InitConfig) (string, error) { +func downloadInit(ctx context.Context, initConfig *conf.InitConfig) (string, error) { if initConfig.Url == "" { return "", nil } @@ -215,228 +159,6 @@ func validateBlockChain(blockChain *core.BlockChain, chainConfig *params.ChainCo return nil } -type importantRoots struct { - chainDb ethdb.Database - roots []common.Hash - heights []uint64 -} - -// The minimum block distance between two important roots -const minRootDistance = 2000 - -// Marks a header as important, and records its root and height. -// If overwrite is true, it'll remove any future roots and replace them with this header. -// If overwrite is false, it'll ignore this header if it has future roots. -func (r *importantRoots) addHeader(header *types.Header, overwrite bool) error { - targetBlockNum := header.Number.Uint64() - for { - if header == nil || header.Root == (common.Hash{}) { - log.Error("missing state of pruning target", "blockNum", targetBlockNum) - return nil - } - exists, err := r.chainDb.Has(header.Root.Bytes()) - if err != nil { - return err - } - if exists { - break - } - num := header.Number.Uint64() - if num%3000 == 0 { - log.Info("looking for old block with state to keep", "current", num, "target", targetBlockNum) - } - // An underflow is fine here because it'll just return nil due to not found - header = rawdb.ReadHeader(r.chainDb, header.ParentHash, num-1) - } - height := header.Number.Uint64() - for len(r.heights) > 0 && r.heights[len(r.heights)-1] > height { - if !overwrite { - return nil - } - r.roots = r.roots[:len(r.roots)-1] - r.heights = r.heights[:len(r.heights)-1] - } - if len(r.heights) > 0 && r.heights[len(r.heights)-1]+minRootDistance > height { - return nil - } - r.roots = append(r.roots, header.Root) - r.heights = append(r.heights, height) - return nil -} - -var hashListRegex = regexp.MustCompile("^(0x)?[0-9a-fA-F]{64}(,(0x)?[0-9a-fA-F]{64})*$") - -// Finds important roots to retain while proving -func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, nodeConfig *NodeConfig, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) ([]common.Hash, error) { - initConfig := &nodeConfig.Init - chainConfig := gethexec.TryReadStoredChainConfig(chainDb) - if chainConfig == nil { - return nil, errors.New("database doesn't have a chain config (was this node initialized?)") - } - arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "", true) - if err != nil { - return nil, err - } - defer func() { - err := arbDb.Close() - if err != nil { - log.Warn("failed to close arbitrum database after finding pruning targets", "err", err) - } - }() - roots := importantRoots{ - chainDb: chainDb, - } - genesisNum := chainConfig.ArbitrumChainParams.GenesisBlockNum - genesisHash := rawdb.ReadCanonicalHash(chainDb, genesisNum) - genesisHeader := rawdb.ReadHeader(chainDb, genesisHash, genesisNum) - if genesisHeader == nil { - return nil, errors.New("missing L2 genesis block header") - } - err = roots.addHeader(genesisHeader, false) - if err != nil { - return nil, err - } - if initConfig.Prune == "validator" { - if l1Client == nil || reflect.ValueOf(l1Client).IsNil() { - return nil, errors.New("an L1 connection is required for validator pruning") - } - callOpts := bind.CallOpts{ - Context: ctx, - BlockNumber: big.NewInt(int64(rpc.FinalizedBlockNumber)), - } - rollup, err := staker.NewRollupWatcher(rollupAddrs.Rollup, l1Client, callOpts) - if err != nil { - return nil, err - } - latestConfirmedNum, err := rollup.LatestConfirmed(&callOpts) - if err != nil { - return nil, err - } - latestConfirmedNode, err := rollup.LookupNode(ctx, latestConfirmedNum) - if err != nil { - return nil, err - } - confirmedHash := latestConfirmedNode.Assertion.AfterState.GlobalState.BlockHash - confirmedNumber := rawdb.ReadHeaderNumber(chainDb, confirmedHash) - var confirmedHeader *types.Header - if confirmedNumber != nil { - confirmedHeader = rawdb.ReadHeader(chainDb, confirmedHash, *confirmedNumber) - } - if confirmedHeader != nil { - err = roots.addHeader(confirmedHeader, false) - if err != nil { - return nil, err - } - } else { - log.Warn("missing latest confirmed block", "hash", confirmedHash) - } - - validatorDb := rawdb.NewTable(arbDb, storage.BlockValidatorPrefix) - lastValidated, err := staker.ReadLastValidatedInfo(validatorDb) - if err != nil { - return nil, err - } - if lastValidated != nil { - var lastValidatedHeader *types.Header - headerNum := rawdb.ReadHeaderNumber(chainDb, lastValidated.GlobalState.BlockHash) - if headerNum != nil { - lastValidatedHeader = rawdb.ReadHeader(chainDb, lastValidated.GlobalState.BlockHash, *headerNum) - } - if lastValidatedHeader != nil { - err = roots.addHeader(lastValidatedHeader, false) - if err != nil { - return nil, err - } - } else { - log.Warn("missing latest validated block", "hash", lastValidated.GlobalState.BlockHash) - } - } - } else if initConfig.Prune == "full" { - if nodeConfig.Node.ValidatorRequired() { - return nil, errors.New("refusing to prune to full-node level when validator is enabled (you should prune in validator mode)") - } - } else if hashListRegex.MatchString(initConfig.Prune) { - parts := strings.Split(initConfig.Prune, ",") - roots := []common.Hash{genesisHeader.Root} - for _, part := range parts { - root := common.HexToHash(part) - if root == genesisHeader.Root { - // This was already included in the builtin list - continue - } - roots = append(roots, root) - } - return roots, nil - } else { - return nil, fmt.Errorf("unknown pruning mode: \"%v\"", initConfig.Prune) - } - if l1Client != nil { - // Find the latest finalized block and add it as a pruning target - l1Block, err := l1Client.BlockByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber))) - if err != nil { - return nil, fmt.Errorf("failed to get finalized block: %w", err) - } - l1BlockNum := l1Block.NumberU64() - tracker, err := arbnode.NewInboxTracker(arbDb, nil, nil) - if err != nil { - return nil, err - } - batch, err := tracker.GetBatchCount() - if err != nil { - return nil, err - } - for { - if ctx.Err() != nil { - return nil, ctx.Err() - } - if batch == 0 { - // No batch has been finalized - break - } - batch -= 1 - meta, err := tracker.GetBatchMetadata(batch) - if err != nil { - return nil, err - } - if meta.ParentChainBlock <= l1BlockNum { - signedBlockNum := arbutil.MessageCountToBlockNumber(meta.MessageCount, genesisNum) - blockNum := uint64(signedBlockNum) - l2Hash := rawdb.ReadCanonicalHash(chainDb, blockNum) - l2Header := rawdb.ReadHeader(chainDb, l2Hash, blockNum) - if l2Header == nil { - log.Warn("latest finalized L2 block is unknown", "blockNum", signedBlockNum) - break - } - err = roots.addHeader(l2Header, false) - if err != nil { - return nil, err - } - break - } - } - } - roots.roots = append(roots.roots, common.Hash{}) // the latest snapshot - log.Info("found pruning target blocks", "heights", roots.heights, "roots", roots.roots) - return roots.roots, nil -} - -func pruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, nodeConfig *NodeConfig, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) error { - config := &nodeConfig.Init - if config.Prune == "" { - return pruner.RecoverPruning(stack.InstanceDir(), chainDb) - } - root, err := findImportantRoots(ctx, chainDb, stack, nodeConfig, cacheConfig, l1Client, rollupAddrs) - if err != nil { - return fmt.Errorf("failed to find root to retain for pruning: %w", err) - } - - pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: config.PruneBloomSize}) - if err != nil { - return err - } - return pruner.Prune(root) -} - func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", 0, 0, "", "", true); err == nil { @@ -446,7 +168,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return chainDb, nil, err } - err = pruneChainDb(ctx, chainDb, stack, config, cacheConfig, l1Client, rollupAddrs) + err = prunning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) if err != nil { return chainDb, nil, fmt.Errorf("error pruning: %w", err) } @@ -642,7 +364,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } - err = pruneChainDb(ctx, chainDb, stack, config, cacheConfig, l1Client, rollupAddrs) + err = prunning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) if err != nil { return chainDb, nil, fmt.Errorf("error pruning: %w", err) } diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index 86093a85d9..ca0d3496f0 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -677,7 +677,7 @@ type NodeConfig struct { MetricsServer genericconf.MetricsServerConfig `koanf:"metrics-server"` PProf bool `koanf:"pprof"` PprofCfg genericconf.PProf `koanf:"pprof-cfg"` - Init InitConfig `koanf:"init"` + Init conf.InitConfig `koanf:"init"` Rpc genericconf.RpcConfig `koanf:"rpc"` } @@ -699,7 +699,7 @@ var NodeConfigDefault = NodeConfig{ GraphQL: genericconf.GraphQLConfigDefault, Metrics: false, MetricsServer: genericconf.MetricsServerConfigDefault, - Init: InitConfigDefault, + Init: conf.InitConfigDefault, Rpc: genericconf.DefaultRpcConfig, PProf: false, PprofCfg: genericconf.PProfDefault, @@ -726,7 +726,7 @@ func NodeConfigAddOptions(f *flag.FlagSet) { f.Bool("pprof", NodeConfigDefault.PProf, "enable pprof") genericconf.PProfAddOptions("pprof-cfg", f) - InitConfigAddOptions("init", f) + conf.InitConfigAddOptions("init", f) genericconf.RpcConfigAddOptions("rpc", f) } diff --git a/cmd/prunning/prunning.go b/cmd/prunning/prunning.go new file mode 100644 index 0000000000..7538c68a10 --- /dev/null +++ b/cmd/prunning/prunning.go @@ -0,0 +1,250 @@ +package prunning + +import ( + "context" + "errors" + "fmt" + "math/big" + "reflect" + "regexp" + "strings" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/pruner" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/arbnode" + "github.com/offchainlabs/nitro/arbnode/dataposter/storage" + "github.com/offchainlabs/nitro/arbutil" + "github.com/offchainlabs/nitro/cmd/chaininfo" + "github.com/offchainlabs/nitro/cmd/conf" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/staker" +) + +type importantRoots struct { + chainDb ethdb.Database + roots []common.Hash + heights []uint64 +} + +// The minimum block distance between two important roots +const minRootDistance = 2000 + +// Marks a header as important, and records its root and height. +// If overwrite is true, it'll remove any future roots and replace them with this header. +// If overwrite is false, it'll ignore this header if it has future roots. +func (r *importantRoots) addHeader(header *types.Header, overwrite bool) error { + targetBlockNum := header.Number.Uint64() + for { + if header == nil || header.Root == (common.Hash{}) { + log.Error("missing state of pruning target", "blockNum", targetBlockNum) + return nil + } + exists, err := r.chainDb.Has(header.Root.Bytes()) + if err != nil { + return err + } + if exists { + break + } + num := header.Number.Uint64() + if num%3000 == 0 { + log.Info("looking for old block with state to keep", "current", num, "target", targetBlockNum) + } + // An underflow is fine here because it'll just return nil due to not found + header = rawdb.ReadHeader(r.chainDb, header.ParentHash, num-1) + } + height := header.Number.Uint64() + for len(r.heights) > 0 && r.heights[len(r.heights)-1] > height { + if !overwrite { + return nil + } + r.roots = r.roots[:len(r.roots)-1] + r.heights = r.heights[:len(r.heights)-1] + } + if len(r.heights) > 0 && r.heights[len(r.heights)-1]+minRootDistance > height { + return nil + } + r.roots = append(r.roots, header.Root) + r.heights = append(r.heights, height) + return nil +} + +var hashListRegex = regexp.MustCompile("^(0x)?[0-9a-fA-F]{64}(,(0x)?[0-9a-fA-F]{64})*$") + +// Finds important roots to retain while proving +func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) ([]common.Hash, error) { + chainConfig := gethexec.TryReadStoredChainConfig(chainDb) + if chainConfig == nil { + return nil, errors.New("database doesn't have a chain config (was this node initialized?)") + } + arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "", true) + if err != nil { + return nil, err + } + defer func() { + err := arbDb.Close() + if err != nil { + log.Warn("failed to close arbitrum database after finding pruning targets", "err", err) + } + }() + roots := importantRoots{ + chainDb: chainDb, + } + genesisNum := chainConfig.ArbitrumChainParams.GenesisBlockNum + genesisHash := rawdb.ReadCanonicalHash(chainDb, genesisNum) + genesisHeader := rawdb.ReadHeader(chainDb, genesisHash, genesisNum) + log.Warn("XXX", "genesisNum", genesisNum, "genesisHash", genesisHash) + if genesisHeader == nil { + return nil, errors.New("missing L2 genesis block header") + } + err = roots.addHeader(genesisHeader, false) + if err != nil { + return nil, err + } + if initConfig.Prune == "validator" { + if l1Client == nil || reflect.ValueOf(l1Client).IsNil() { + return nil, errors.New("an L1 connection is required for validator pruning") + } + callOpts := bind.CallOpts{ + Context: ctx, + BlockNumber: big.NewInt(int64(rpc.FinalizedBlockNumber)), + } + rollup, err := staker.NewRollupWatcher(rollupAddrs.Rollup, l1Client, callOpts) + if err != nil { + return nil, err + } + latestConfirmedNum, err := rollup.LatestConfirmed(&callOpts) + if err != nil { + return nil, err + } + latestConfirmedNode, err := rollup.LookupNode(ctx, latestConfirmedNum) + if err != nil { + return nil, err + } + confirmedHash := latestConfirmedNode.Assertion.AfterState.GlobalState.BlockHash + confirmedNumber := rawdb.ReadHeaderNumber(chainDb, confirmedHash) + var confirmedHeader *types.Header + if confirmedNumber != nil { + confirmedHeader = rawdb.ReadHeader(chainDb, confirmedHash, *confirmedNumber) + } + if confirmedHeader != nil { + err = roots.addHeader(confirmedHeader, false) + if err != nil { + return nil, err + } + } else { + log.Warn("missing latest confirmed block", "hash", confirmedHash) + } + + validatorDb := rawdb.NewTable(arbDb, storage.BlockValidatorPrefix) + lastValidated, err := staker.ReadLastValidatedInfo(validatorDb) + if err != nil { + return nil, err + } + if lastValidated != nil { + var lastValidatedHeader *types.Header + headerNum := rawdb.ReadHeaderNumber(chainDb, lastValidated.GlobalState.BlockHash) + if headerNum != nil { + lastValidatedHeader = rawdb.ReadHeader(chainDb, lastValidated.GlobalState.BlockHash, *headerNum) + } + if lastValidatedHeader != nil { + err = roots.addHeader(lastValidatedHeader, false) + if err != nil { + return nil, err + } + } else { + log.Warn("missing latest validated block", "hash", lastValidated.GlobalState.BlockHash) + } + } + } else if initConfig.Prune == "full" { + if validatorRequired { + return nil, errors.New("refusing to prune to full-node level when validator is enabled (you should prune in validator mode)") + } + } else if hashListRegex.MatchString(initConfig.Prune) { + parts := strings.Split(initConfig.Prune, ",") + roots := []common.Hash{genesisHeader.Root} + for _, part := range parts { + root := common.HexToHash(part) + if root == genesisHeader.Root { + // This was already included in the builtin list + continue + } + roots = append(roots, root) + } + return roots, nil + } else { + return nil, fmt.Errorf("unknown pruning mode: \"%v\"", initConfig.Prune) + } + if l1Client != nil { + // Find the latest finalized block and add it as a pruning target + l1Block, err := l1Client.BlockByNumber(ctx, big.NewInt(int64(rpc.FinalizedBlockNumber))) + if err != nil { + return nil, fmt.Errorf("failed to get finalized block: %w", err) + } + l1BlockNum := l1Block.NumberU64() + tracker, err := arbnode.NewInboxTracker(arbDb, nil, nil) + if err != nil { + return nil, err + } + batch, err := tracker.GetBatchCount() + if err != nil { + return nil, err + } + for { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if batch == 0 { + // No batch has been finalized + break + } + batch -= 1 + meta, err := tracker.GetBatchMetadata(batch) + if err != nil { + return nil, err + } + if meta.ParentChainBlock <= l1BlockNum { + signedBlockNum := arbutil.MessageCountToBlockNumber(meta.MessageCount, genesisNum) + blockNum := uint64(signedBlockNum) + l2Hash := rawdb.ReadCanonicalHash(chainDb, blockNum) + l2Header := rawdb.ReadHeader(chainDb, l2Hash, blockNum) + if l2Header == nil { + log.Warn("latest finalized L2 block is unknown", "blockNum", signedBlockNum) + break + } + err = roots.addHeader(l2Header, false) + if err != nil { + return nil, err + } + break + } + } + } + roots.roots = append(roots.roots, common.Hash{}) // the latest snapshot + log.Info("found pruning target blocks", "heights", roots.heights, "roots", roots.roots) + return roots.roots, nil +} + +func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, initConfig *conf.InitConfig, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses, validatorRequired bool) error { + if initConfig.Prune == "" { + return pruner.RecoverPruning(stack.InstanceDir(), chainDb) + } + root, err := findImportantRoots(ctx, chainDb, stack, initConfig, cacheConfig, l1Client, rollupAddrs, validatorRequired) + if err != nil { + return fmt.Errorf("failed to find root to retain for pruning: %w", err) + } + + pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize}) + if err != nil { + return err + } + return pruner.Prune(root) +} diff --git a/go-ethereum b/go-ethereum index b1622e6ac4..1e2855b24d 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit b1622e6ac4bf3762aebde92a585de2889d90823f +Subproject commit 1e2855b24d6555c8cfaf471bd9e2c3d19ab5c32c diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 7752fbd34e..937b8980fc 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -698,7 +698,7 @@ func createL2BlockChainWithStackConfig( chainDb, err := stack.OpenDatabase("chaindb", 0, 0, "", false) Require(t, err) - arbDb, err := stack.OpenDatabase("arbdb", 0, 0, "", false) + arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "", false) Require(t, err) initReader := statetransfer.NewMemoryInitDataReader(&l2info.ArbInitData) @@ -903,7 +903,7 @@ func Create2ndNodeWithConfig( l2chainDb, err := l2stack.OpenDatabase("chaindb", 0, 0, "", false) Require(t, err) - l2arbDb, err := l2stack.OpenDatabase("arbdb", 0, 0, "", false) + l2arbDb, err := l2stack.OpenDatabase("arbitrumdata", 0, 0, "", false) Require(t, err) initReader := statetransfer.NewMemoryInitDataReader(l2InitData) diff --git a/system_tests/prunning_test.go b/system_tests/prunning_test.go new file mode 100644 index 0000000000..aadf3c5e9e --- /dev/null +++ b/system_tests/prunning_test.go @@ -0,0 +1,121 @@ +package arbtest + +import ( + "context" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/node" + "github.com/offchainlabs/nitro/cmd/conf" + "github.com/offchainlabs/nitro/cmd/prunning" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func countDbEntries(db ethdb.Iteratee) int { + entries := 0 + it := db.NewIterator(nil, nil) + for it.Next() { + entries++ + } + it.Release() + return entries +} + +func TestPrunning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var dataDir string + + func() { + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + _ = builder.Build(t) + dataDir = builder.dataDir + l2cleanupDone := false + defer func() { + if !l2cleanupDone { + builder.L2.cleanup() + } + builder.L1.cleanup() + }() + builder.L2Info.GenerateAccount("User2") + var txs []*types.Transaction + for i := uint64(0); i < 1000; i++ { + tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, common.Big1, nil) + txs = append(txs, tx) + err := builder.L2.Client.SendTransaction(ctx, tx) + Require(t, err) + } + for _, tx := range txs { + _, err := builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + } + l2cleanupDone = true + builder.L2.cleanup() + t.Log("stopped l2 node") + + stack, err := node.New(builder.l2StackConfig) + Require(t, err) + defer stack.Close() + chainDb, err := stack.OpenDatabase("chaindb", 0, 0, "", false) + Require(t, err) + defer chainDb.Close() + entriesBeforePrunning := countDbEntries(chainDb) + + prand := testhelpers.NewPseudoRandomDataSource(t, 1) + var testKeys [][]byte + for i := 0; i < 100; i++ { + // generate test keys with length of hash to emulate legacy state trie nodes + testKeys = append(testKeys, prand.GetHash().Bytes()) + } + for _, key := range testKeys { + err = chainDb.Put(key, common.FromHex("0xdeadbeef")) + Require(t, err) + } + for _, key := range testKeys { + if has, _ := chainDb.Has(key); !has { + Fatal(t, "internal test error - failed to check existence of test key") + } + } + + initConfig := conf.InitConfigDefault + initConfig.Prune = "full" + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &builder.execConfig.Caching) + err = prunning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false) + Require(t, err) + + for _, key := range testKeys { + if has, _ := chainDb.Has(key); has { + Fatal(t, "test key hasn't been prunned as expected") + } + } + + entriesAfterPrunning := countDbEntries(chainDb) + t.Log("db entries pre-prunning:", entriesBeforePrunning) + t.Log("db entries post-prunning:", entriesAfterPrunning) + + if entriesAfterPrunning >= entriesBeforePrunning { + Fatal(t, "The db doesn't have less entires after prunning then before. Before:", entriesBeforePrunning, "After:", entriesAfterPrunning) + } + }() + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + builder.dataDir = dataDir + cancel = builder.Build(t) + defer cancel() + + builder.L2Info.GenerateAccount("User2") + var txs []*types.Transaction + for i := uint64(0); i < 10; i++ { + tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, common.Big1, nil) + txs = append(txs, tx) + err := builder.L2.Client.SendTransaction(ctx, tx) + Require(t, err) + } + for _, tx := range txs { + _, err := builder.L2.EnsureTxSucceeded(tx) + Require(t, err) + } +} From 875c558d9b0350d0d1923a1a7028a95c6107940d Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 15 Dec 2023 15:29:08 +0000 Subject: [PATCH 02/32] add missing default value of InitConfig.Empty --- cmd/conf/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/conf/init.go b/cmd/conf/init.go index f228891b90..bebf1955b7 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -31,6 +31,7 @@ var InitConfigDefault = InitConfig{ DevInit: false, DevInitAddress: "", DevInitBlockNum: 0, + Empty: false, ImportFile: "", AccountsPerSync: 100000, ThenQuit: false, From ae5c2f28227b7e9ea9353a8ae9c581232bca5aa7 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 15 Dec 2023 15:43:18 +0000 Subject: [PATCH 03/32] cleanup imports --- cmd/nitro/init.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 745e168375..00012ac62d 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -15,10 +15,6 @@ import ( "sync" "time" - "github.com/offchainlabs/nitro/cmd/conf" - "github.com/offchainlabs/nitro/cmd/prunning" - "github.com/offchainlabs/nitro/cmd/util" - "github.com/cavaliergopher/grab/v3" extract "github.com/codeclysm/extract/v3" "github.com/ethereum/go-ethereum/common" @@ -35,7 +31,10 @@ import ( "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/chaininfo" + "github.com/offchainlabs/nitro/cmd/conf" "github.com/offchainlabs/nitro/cmd/ipfshelper" + "github.com/offchainlabs/nitro/cmd/prunning" + "github.com/offchainlabs/nitro/cmd/util" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/statetransfer" ) From 6ca851673ed9b86bdca4c60097712f041509f6c5 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Tue, 19 Dec 2023 14:33:30 +0000 Subject: [PATCH 04/32] fix arbitrum data db name in das test --- system_tests/das_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index 8c9621d57a..6db339521c 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -179,7 +179,7 @@ func TestDASRekey(t *testing.T) { l2chainDb, err := l2stackA.OpenDatabase("chaindb", 0, 0, "", false) Require(t, err) - l2arbDb, err := l2stackA.OpenDatabase("arbdb", 0, 0, "", false) + l2arbDb, err := l2stackA.OpenDatabase("arbitrumdata", 0, 0, "", false) Require(t, err) l2blockchain, err := gethexec.GetBlockChain(l2chainDb, nil, chainConfig, gethexec.ConfigDefaultTest().TxLookupLimit) From 7a26e97d04ef7a26a9adcd6b53db0afc0bf384d6 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 20 Dec 2023 15:03:39 +0000 Subject: [PATCH 05/32] clean up debug log --- cmd/prunning/prunning.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/prunning/prunning.go b/cmd/prunning/prunning.go index 7538c68a10..5efdf8e643 100644 --- a/cmd/prunning/prunning.go +++ b/cmd/prunning/prunning.go @@ -101,7 +101,6 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node genesisNum := chainConfig.ArbitrumChainParams.GenesisBlockNum genesisHash := rawdb.ReadCanonicalHash(chainDb, genesisNum) genesisHeader := rawdb.ReadHeader(chainDb, genesisHash, genesisNum) - log.Warn("XXX", "genesisNum", genesisNum, "genesisHash", genesisHash) if genesisHeader == nil { return nil, errors.New("missing L2 genesis block header") } From 712bc0ce2658f98b7476375148c72891dd659bb3 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 20 Dec 2023 15:10:26 +0000 Subject: [PATCH 06/32] fix typo --- system_tests/prunning_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_tests/prunning_test.go b/system_tests/prunning_test.go index aadf3c5e9e..cfb1a4bcf7 100644 --- a/system_tests/prunning_test.go +++ b/system_tests/prunning_test.go @@ -98,7 +98,7 @@ func TestPrunning(t *testing.T) { t.Log("db entries post-prunning:", entriesAfterPrunning) if entriesAfterPrunning >= entriesBeforePrunning { - Fatal(t, "The db doesn't have less entires after prunning then before. Before:", entriesBeforePrunning, "After:", entriesAfterPrunning) + Fatal(t, "The db doesn't have less entries after prunning then before. Before:", entriesBeforePrunning, "After:", entriesAfterPrunning) } }() builder := NewNodeBuilder(ctx).DefaultConfig(t, true) From b005bd0c1cce855ac100e3bddb66739dc0e912c0 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Wed, 20 Dec 2023 18:24:38 +0000 Subject: [PATCH 07/32] count only state entries in diskdb in prunning test --- system_tests/prunning_test.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/system_tests/prunning_test.go b/system_tests/prunning_test.go index cfb1a4bcf7..d522458209 100644 --- a/system_tests/prunning_test.go +++ b/system_tests/prunning_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" @@ -14,11 +15,14 @@ import ( "github.com/offchainlabs/nitro/util/testhelpers" ) -func countDbEntries(db ethdb.Iteratee) int { +func countStateEntries(db ethdb.Iteratee) int { entries := 0 it := db.NewIterator(nil, nil) for it.Next() { - entries++ + isCode, _ := rawdb.IsCodeKey(it.Key()) + if len(it.Key()) == common.HashLength || isCode { + entries++ + } } it.Release() return entries @@ -43,7 +47,7 @@ func TestPrunning(t *testing.T) { }() builder.L2Info.GenerateAccount("User2") var txs []*types.Transaction - for i := uint64(0); i < 1000; i++ { + for i := uint64(0); i < 200; i++ { tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, common.Big1, nil) txs = append(txs, tx) err := builder.L2.Client.SendTransaction(ctx, tx) @@ -63,7 +67,7 @@ func TestPrunning(t *testing.T) { chainDb, err := stack.OpenDatabase("chaindb", 0, 0, "", false) Require(t, err) defer chainDb.Close() - entriesBeforePrunning := countDbEntries(chainDb) + chainDbEntriesBeforePrunning := countStateEntries(chainDb) prand := testhelpers.NewPseudoRandomDataSource(t, 1) var testKeys [][]byte @@ -93,12 +97,12 @@ func TestPrunning(t *testing.T) { } } - entriesAfterPrunning := countDbEntries(chainDb) - t.Log("db entries pre-prunning:", entriesBeforePrunning) - t.Log("db entries post-prunning:", entriesAfterPrunning) + chainDbEntriesAfterPrunning := countStateEntries(chainDb) + t.Log("db entries pre-prunning:", chainDbEntriesBeforePrunning) + t.Log("db entries post-prunning:", chainDbEntriesAfterPrunning) - if entriesAfterPrunning >= entriesBeforePrunning { - Fatal(t, "The db doesn't have less entries after prunning then before. Before:", entriesBeforePrunning, "After:", entriesAfterPrunning) + if chainDbEntriesAfterPrunning >= chainDbEntriesBeforePrunning { + Fatal(t, "The db doesn't have less entries after prunning then before. Before:", chainDbEntriesBeforePrunning, "After:", chainDbEntriesAfterPrunning) } }() builder := NewNodeBuilder(ctx).DefaultConfig(t, true) From a733e5a1b23d9e2f1aaa678d2a072b6be6d5388f Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 20 Dec 2023 16:25:47 -0700 Subject: [PATCH 08/32] Only lock redis in the batch poster when posting the batch --- arbnode/batch_poster.go | 30 ++++++++++++++++++++++++++++-- arbnode/redislock/redis.go | 21 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index a54336fd5a..4c8bd36f72 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -16,6 +16,7 @@ import ( "time" "github.com/andybalholm/brotli" + "github.com/go-redis/redis/v8" "github.com/spf13/pflag" "github.com/ethereum/go-ethereum" @@ -117,6 +118,7 @@ type BatchPosterConfig struct { DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` RedisUrl string `koanf:"redis-url"` RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` + EnableRedisLock bool `koanf:"enable-redis-lock"` ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` @@ -165,6 +167,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".gas-refunder-address", DefaultBatchPosterConfig.GasRefunderAddress, "The gas refunder contract address (optional)") f.Uint64(prefix+".extra-batch-gas", DefaultBatchPosterConfig.ExtraBatchGas, "use this much more gas than estimation says is necessary to post batches") f.String(prefix+".redis-url", DefaultBatchPosterConfig.RedisUrl, "if non-empty, the Redis URL to store queued transactions in") + f.Bool(prefix+".enable-redis-lock", DefaultBatchPosterConfig.EnableRedisLock, "if a Redis URL is specified, ensure only one batch poster attempts to post batches at a time") f.String(prefix+".l1-block-bound", DefaultBatchPosterConfig.L1BlockBound, "only post messages to batches when they're within the max future block/timestamp as of this L1 block tag (\"safe\", \"finalized\", \"latest\", or \"ignore\" to ignore this check)") f.Duration(prefix+".l1-block-bound-bypass", DefaultBatchPosterConfig.L1BlockBoundBypass, "post batches even if not within the layer 1 future bounds if we're within this margin of the max delay") redislock.AddConfigOptions(prefix+".redis-lock", f) @@ -190,6 +193,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ L1BlockBound: "", L1BlockBoundBypass: time.Hour, RedisLock: redislock.DefaultCfg, + EnableRedisLock: true, } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -215,6 +219,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ ParentChainWallet: DefaultBatchPosterL1WalletConfig, L1BlockBound: "", L1BlockBoundBypass: time.Hour, + EnableRedisLock: true, } type BatchPosterOpts struct { @@ -254,7 +259,11 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e simpleRedisLockConfig.Key = batchPosterSimpleRedisLockKey return &simpleRedisLockConfig } - redisLock, err := redislock.NewSimple(redisClient, redisLockConfigFetcher, func() bool { return opts.SyncMonitor.Synced() }) + var redisClientForLock redis.UniversalClient + if opts.Config().EnableRedisLock { + redisClientForLock = redisClient + } + redisLock, err := redislock.NewSimple(redisClientForLock, redisLockConfigFetcher, func() bool { return opts.SyncMonitor.Synced() }) if err != nil { return nil, err } @@ -830,6 +839,8 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, const ethPosBlockTime = 12 * time.Second +var errAttemptLockFailed = errors.New("failed to acquire lock; either another batch poster posted a batch or this node fell behind") + func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) { if b.batchReverted.Load() { return false, fmt.Errorf("batch was reverted, not posting any more batches") @@ -1006,6 +1017,10 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) } if b.daWriter != nil { + if !b.redisLock.AttemptLock(ctx) { + return false, errAttemptLockFailed + } + cert, err := b.daWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), []byte{}) // b.daWriter will append signature if enabled if errors.Is(err, das.BatchToDasFailed) { if config.DisableDasFallbackStoreDataOnChain { @@ -1043,6 +1058,9 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if err != nil { return false, err } + if !b.redisLock.AttemptLock(ctx) { + return false, errAttemptLockFailed + } tx, err := b.dataPoster.PostTransaction(ctx, firstMsgTime, nonce, @@ -1147,8 +1165,16 @@ func (b *BatchPoster) Start(ctxIn context.Context) { batchPosterWalletBalance.Update(arbmath.BalancePerEther(walletBalance)) } } - if !b.redisLock.AttemptLock(ctx) { + couldLock, err := b.redisLock.CouldAcquireLock(ctx) + if err != nil { + log.Warn("Error checking if we could acquire redis lock", "err", err) + // Might as well try, worst case we fail to lock + couldLock = true + } + if !couldLock { + log.Debug("Not posting batches right now because another batch poster has the lock or this node is behind") b.building = nil + b.firstEphemeralError = time.Time{} return b.config().PollInterval } posted, err := b.maybePostSequencerBatch(ctx) diff --git a/arbnode/redislock/redis.go b/arbnode/redislock/redis.go index c8252e059f..fe46612527 100644 --- a/arbnode/redislock/redis.go +++ b/arbnode/redislock/redis.go @@ -143,6 +143,27 @@ func (l *Simple) Locked() bool { return time.Now().Before(atomicTimeRead(&l.lockedUntil)) } +// Returns true if a call to AttemptLock will likely succeed +func (l *Simple) CouldAcquireLock(ctx context.Context) (bool, error) { + if l.Locked() { + return true, nil + } + if l.stopping || !l.readyToLock() { + return false, nil + } + // l.client shouldn't be nil here because Locked would've returned true + current, err := l.client.Get(ctx, l.config().Key).Result() + if errors.Is(err, redis.Nil) { + // Lock is free for the taking + return true, nil + } + if err != nil { + return false, err + } + // return true if the lock is free for the taking or is already ours + return current == "" || current == l.myId, nil +} + func (l *Simple) Release(ctx context.Context) { l.mutex.Lock() defer l.mutex.Unlock() From 4e73be72871aa342b918c568c3439a11256278c6 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 21 Dec 2023 00:37:17 -0700 Subject: [PATCH 09/32] Always broadcast feed, even when the node isn't the sequencer --- arbnode/transaction_streamer.go | 12 ++++++------ broadcaster/broadcaster.go | 25 ++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index db0658f923..24ef2a7cc4 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -862,12 +862,6 @@ func (s *TransactionStreamer) WriteMessageFromSequencer(pos arbutil.MessageIndex return err } - if s.broadcastServer != nil { - if err := s.broadcastServer.BroadcastSingle(msgWithMeta, pos); err != nil { - log.Error("failed broadcasting message", "pos", pos, "err", err) - } - } - return nil } @@ -927,6 +921,12 @@ func (s *TransactionStreamer) writeMessages(pos arbutil.MessageIndex, messages [ default: } + if s.broadcastServer != nil { + if err := s.broadcastServer.BroadcastMessages(messages, pos); err != nil { + log.Error("failed broadcasting message", "pos", pos, "err", err) + } + } + return nil } diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 8a70e39810..38ffb0696c 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -5,6 +5,7 @@ package broadcaster import ( "context" + "errors" "net" "github.com/gobwas/ws" @@ -56,10 +57,11 @@ func (b *Broadcaster) NewBroadcastFeedMessage(message arbostypes.MessageWithMeta }, nil } -func (b *Broadcaster) BroadcastSingle(msg arbostypes.MessageWithMetadata, seq arbutil.MessageIndex) error { +func (b *Broadcaster) BroadcastSingle(msg arbostypes.MessageWithMetadata, seq arbutil.MessageIndex) (err error) { defer func() { if r := recover(); r != nil { log.Error("recovered error in BroadcastSingle", "recover", r) + err = errors.New("panic in BroadcastSingle") } }() bfm, err := b.NewBroadcastFeedMessage(msg, seq) @@ -79,6 +81,27 @@ func (b *Broadcaster) BroadcastSingleFeedMessage(bfm *m.BroadcastFeedMessage) { b.BroadcastFeedMessages(broadcastFeedMessages) } +func (b *Broadcaster) BroadcastMessages(messages []arbostypes.MessageWithMetadata, seq arbutil.MessageIndex) (err error) { + defer func() { + if r := recover(); r != nil { + log.Error("recovered error in BroadcastMessages", "recover", r) + err = errors.New("panic in BroadcastMessages") + } + }() + var feedMessages []*m.BroadcastFeedMessage + for _, msg := range messages { + bfm, err := b.NewBroadcastFeedMessage(msg, seq) + if err != nil { + return err + } + feedMessages = append(feedMessages, bfm) + } + + b.BroadcastFeedMessages(feedMessages) + + return nil +} + func (b *Broadcaster) BroadcastFeedMessages(messages []*m.BroadcastFeedMessage) { bm := &m.BroadcastMessage{ From ae54e0852e53e78f6fb7fafd531bdb066dec7369 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 21 Dec 2023 00:42:56 -0700 Subject: [PATCH 10/32] Add comment about PopulateFeedBacklog in seq coordinator being redundant --- arbnode/seq_coordinator.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arbnode/seq_coordinator.go b/arbnode/seq_coordinator.go index cb6f4fe502..ecf38ddf42 100644 --- a/arbnode/seq_coordinator.go +++ b/arbnode/seq_coordinator.go @@ -650,6 +650,8 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration { log.Warn("failed sequencing delayed messages after catching lock", "err", err) } } + // This should be redundant now that even non-primary sequencers broadcast over the feed, + // but the backlog efficiently deduplicates messages, so better safe than sorry. err = c.streamer.PopulateFeedBacklog() if err != nil { log.Warn("failed to populate the feed backlog on lockout acquisition", "err", err) From 29a8c854d4d7bdc13dfcb67299dcd5f200d7faa9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 21 Dec 2023 00:47:13 -0700 Subject: [PATCH 11/32] Don't acquire redis lock in data poster --- arbnode/dataposter/data_poster.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 059e080eea..191fd40708 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -86,6 +86,7 @@ type signerFn func(context.Context, common.Address, *types.Transaction) (*types. type AttemptLocker interface { AttemptLock(context.Context) bool + CouldAcquireLock(context.Context) (bool, error) } func parseReplacementTimes(val string) ([]time.Duration, error) { @@ -745,9 +746,6 @@ func (p *DataPoster) Start(ctxIn context.Context) { p.CallIteratively(func(ctx context.Context) time.Duration { p.mutex.Lock() defer p.mutex.Unlock() - if !p.redisLock.AttemptLock(ctx) { - return minWait - } err := p.updateBalance(ctx) if err != nil { log.Warn("failed to update tx poster balance", "err", err) @@ -777,10 +775,16 @@ func (p *DataPoster) Start(ctxIn context.Context) { log.Error("Failed to fetch tx queue contents", "err", err) return minWait } + tryToReplace, err := p.redisLock.CouldAcquireLock(ctx) + if err != nil { + log.Warn("Error checking if we could acquire redis lock", "err", err) + // Might as well try, worst case we hit contention on redis and fail + tryToReplace = true + } for index, tx := range queueContents { backlogOfBatches := len(queueContents) - index - 1 replacing := false - if now.After(tx.NextReplacement) { + if tryToReplace && now.After(tx.NextReplacement) { replacing = true err := p.replaceTx(ctx, tx, uint64(backlogOfBatches)) p.maybeLogError(err, tx, "failed to replace-by-fee transaction") From 11d76f62213f50540c280253fc0c5dc8efe31b59 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 21 Dec 2023 10:21:31 -0700 Subject: [PATCH 12/32] Fix broadcastSegment Contains and its usage --- broadcaster/backlog/backlog.go | 2 +- wsbroadcastserver/clientconnection.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/broadcaster/backlog/backlog.go b/broadcaster/backlog/backlog.go index 851561f482..0ef32167fe 100644 --- a/broadcaster/backlog/backlog.go +++ b/broadcaster/backlog/backlog.go @@ -379,7 +379,7 @@ func (s *backlogSegment) Contains(i uint64) bool { s.messagesLock.RLock() defer s.messagesLock.RUnlock() start := s.start() - if i < start || i > s.end() { + if i < start || i > s.end() || len(s.messages) == 0 { return false } diff --git a/wsbroadcastserver/clientconnection.go b/wsbroadcastserver/clientconnection.go index 49cd2af7e6..5e8763bd95 100644 --- a/wsbroadcastserver/clientconnection.go +++ b/wsbroadcastserver/clientconnection.go @@ -134,7 +134,10 @@ func (cc *ClientConnection) writeBacklog(ctx context.Context, segment backlog.Ba msgs := prevSegment.Messages() if prevSegment.Contains(uint64(cc.requestedSeqNum)) { requestedIdx := int(cc.requestedSeqNum) - int(prevSegment.Start()) - msgs = msgs[requestedIdx:] + // This might be false if messages were added after we fetched the segment's messages + if len(msgs) > requestedIdx { + msgs = msgs[requestedIdx:] + } } bm := &m.BroadcastMessage{ Version: m.V1, From 4b816bcb16ce04f20bef2a94c1e98dbd1a3cba58 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Thu, 21 Dec 2023 10:23:25 -0700 Subject: [PATCH 13/32] Further improve requestedSeqNum check --- wsbroadcastserver/clientconnection.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wsbroadcastserver/clientconnection.go b/wsbroadcastserver/clientconnection.go index 5e8763bd95..82b9603a0e 100644 --- a/wsbroadcastserver/clientconnection.go +++ b/wsbroadcastserver/clientconnection.go @@ -119,6 +119,7 @@ func (cc *ClientConnection) Remove() { func (cc *ClientConnection) writeBacklog(ctx context.Context, segment backlog.BacklogSegment) error { var prevSegment backlog.BacklogSegment + isFirstSegment := true for !backlog.IsBacklogSegmentNil(segment) { // must get the next segment before the messages to be sent are // retrieved ensures another segment is not added in between calls. @@ -132,13 +133,14 @@ func (cc *ClientConnection) writeBacklog(ctx context.Context, segment backlog.Ba } msgs := prevSegment.Messages() - if prevSegment.Contains(uint64(cc.requestedSeqNum)) { + if isFirstSegment && prevSegment.Contains(uint64(cc.requestedSeqNum)) { requestedIdx := int(cc.requestedSeqNum) - int(prevSegment.Start()) // This might be false if messages were added after we fetched the segment's messages - if len(msgs) > requestedIdx { + if len(msgs) >= requestedIdx { msgs = msgs[requestedIdx:] } } + isFirstSegment = false bm := &m.BroadcastMessage{ Version: m.V1, Messages: msgs, From 603634df0402b76c1c2a19cda2be7585487342a0 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 22 Dec 2023 11:28:45 -0700 Subject: [PATCH 14/32] Only use redis lock for DAS, not for normal batch posting --- arbnode/batch_poster.go | 53 +++++++++++++++++++------------ arbnode/dataposter/data_poster.go | 17 +++------- arbnode/node.go | 9 ------ arbnode/redislock/redis.go | 5 ++- staker/staker.go | 3 -- 5 files changed, 41 insertions(+), 46 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 4c8bd36f72..870df2e0d5 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -16,7 +16,6 @@ import ( "time" "github.com/andybalholm/brotli" - "github.com/go-redis/redis/v8" "github.com/spf13/pflag" "github.com/ethereum/go-ethereum" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/offchainlabs/nitro/arbnode/dataposter" + "github.com/offchainlabs/nitro/arbnode/dataposter/storage" "github.com/offchainlabs/nitro/arbnode/redislock" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbstate" @@ -118,7 +118,6 @@ type BatchPosterConfig struct { DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` RedisUrl string `koanf:"redis-url"` RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` - EnableRedisLock bool `koanf:"enable-redis-lock"` ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"` ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` L1BlockBound string `koanf:"l1-block-bound" reload:"hot"` @@ -167,7 +166,6 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".gas-refunder-address", DefaultBatchPosterConfig.GasRefunderAddress, "The gas refunder contract address (optional)") f.Uint64(prefix+".extra-batch-gas", DefaultBatchPosterConfig.ExtraBatchGas, "use this much more gas than estimation says is necessary to post batches") f.String(prefix+".redis-url", DefaultBatchPosterConfig.RedisUrl, "if non-empty, the Redis URL to store queued transactions in") - f.Bool(prefix+".enable-redis-lock", DefaultBatchPosterConfig.EnableRedisLock, "if a Redis URL is specified, ensure only one batch poster attempts to post batches at a time") f.String(prefix+".l1-block-bound", DefaultBatchPosterConfig.L1BlockBound, "only post messages to batches when they're within the max future block/timestamp as of this L1 block tag (\"safe\", \"finalized\", \"latest\", or \"ignore\" to ignore this check)") f.Duration(prefix+".l1-block-bound-bypass", DefaultBatchPosterConfig.L1BlockBoundBypass, "post batches even if not within the layer 1 future bounds if we're within this margin of the max delay") redislock.AddConfigOptions(prefix+".redis-lock", f) @@ -193,7 +191,6 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ L1BlockBound: "", L1BlockBoundBypass: time.Hour, RedisLock: redislock.DefaultCfg, - EnableRedisLock: true, } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -219,7 +216,6 @@ var TestBatchPosterConfig = BatchPosterConfig{ ParentChainWallet: DefaultBatchPosterL1WalletConfig, L1BlockBound: "", L1BlockBoundBypass: time.Hour, - EnableRedisLock: true, } type BatchPosterOpts struct { @@ -259,11 +255,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e simpleRedisLockConfig.Key = batchPosterSimpleRedisLockKey return &simpleRedisLockConfig } - var redisClientForLock redis.UniversalClient - if opts.Config().EnableRedisLock { - redisClientForLock = redisClient - } - redisLock, err := redislock.NewSimple(redisClientForLock, redisLockConfigFetcher, func() bool { return opts.SyncMonitor.Synced() }) + redisLock, err := redislock.NewSimple(redisClient, redisLockConfigFetcher, func() bool { return opts.SyncMonitor.Synced() }) if err != nil { return nil, err } @@ -295,7 +287,6 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e HeaderReader: opts.L1Reader, Auth: opts.TransactOpts, RedisClient: redisClient, - RedisLock: redisLock, Config: dataPosterConfigFetcher, MetadataRetriever: b.getBatchPosterPosition, ExtraBacklog: b.GetBacklogEstimate, @@ -779,6 +770,8 @@ func (b *BatchPoster) encodeAddBatch(seqNum *big.Int, prevMsgNum arbutil.Message return fullData, nil } +var ErrNormalGasEstimationFailed = errors.New("normal gas estimation failed") + func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, delayedMessages uint64, realData []byte, realNonce uint64, realAccessList types.AccessList) (uint64, error) { config := b.config() useNormalEstimation := b.dataPoster.MaxMempoolTransactions() == 1 @@ -799,7 +792,7 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, AccessList: realAccessList, }) if err != nil { - return 0, err + return 0, fmt.Errorf("%w: %w", ErrNormalGasEstimationFailed, err) } return gas + config.ExtraBatchGas, nil } @@ -1021,6 +1014,14 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) return false, errAttemptLockFailed } + gotNonce, gotMeta, err := b.dataPoster.GetNextNonceAndMeta(ctx) + if err != nil { + return false, err + } + if nonce != gotNonce || !bytes.Equal(batchPositionBytes, gotMeta) { + return false, fmt.Errorf("%w: nonce changed from %d to %d while creating batch", storage.ErrStorageRace, nonce, gotNonce) + } + cert, err := b.daWriter.Store(ctx, sequencerMsg, uint64(time.Now().Add(config.DASRetentionPeriod).Unix()), []byte{}) // b.daWriter will append signature if enabled if errors.Is(err, das.BatchToDasFailed) { if config.DisableDasFallbackStoreDataOnChain { @@ -1058,9 +1059,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) if err != nil { return false, err } - if !b.redisLock.AttemptLock(ctx) { - return false, errAttemptLockFailed - } tx, err := b.dataPoster.PostTransaction(ctx, firstMsgTime, nonce, @@ -1184,14 +1182,27 @@ func (b *BatchPoster) Start(ctxIn context.Context) { if err != nil { b.building = nil logLevel := log.Error - // Likely the inbox tracker just isn't caught up. - // Let's see if this error disappears naturally. if b.firstEphemeralError == (time.Time{}) { b.firstEphemeralError = time.Now() - logLevel = log.Warn - } else if time.Since(b.firstEphemeralError) < time.Minute { - logLevel = log.Warn - } else if time.Since(b.firstEphemeralError) < time.Minute*5 && strings.Contains(err.Error(), "will exceed max mempool size") { + } + // Likely the inbox tracker just isn't caught up, or there's some other ephemeral error. + // Let's see if this error disappears naturally. + sinceFirstEphemeralError := time.Since(b.firstEphemeralError) + // If the error matches one of these, it's only logged at debug for the first minute, + // then at warn for the next 4 minutes, then at error. If the error isn't one of these, + // it'll be logged at warn for the first minute, then at error. + ignoreAtFirst := errors.Is(err, dataposter.ErrExceedsMaxMempoolSize) || + errors.Is(err, storage.ErrStorageRace) || + errors.Is(err, ErrNormalGasEstimationFailed) || + errors.Is(err, AccumulatorNotFoundErr) || + errors.Is(err, context.Canceled) + if sinceFirstEphemeralError < time.Minute { + if ignoreAtFirst { + logLevel = log.Debug + } else { + logLevel = log.Warn + } + } else if sinceFirstEphemeralError < time.Minute*5 && ignoreAtFirst { logLevel = log.Warn } logLevel("error posting batch", "err", err) diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 191fd40708..673fd317ee 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -57,7 +57,6 @@ type DataPoster struct { client arbutil.L1Interface auth *bind.TransactOpts signer signerFn - redisLock AttemptLocker config ConfigFetcher usingNoOpStorage bool replacementTimes []time.Duration @@ -115,7 +114,6 @@ type DataPosterOpts struct { HeaderReader *headerreader.HeaderReader Auth *bind.TransactOpts RedisClient redis.UniversalClient - RedisLock AttemptLocker Config ConfigFetcher MetadataRetriever func(ctx context.Context, blockNum *big.Int) ([]byte, error) ExtraBacklog func() uint64 @@ -176,7 +174,6 @@ func NewDataPoster(ctx context.Context, opts *DataPosterOpts) (*DataPoster, erro replacementTimes: replacementTimes, metadataRetriever: opts.MetadataRetriever, queue: queue, - redisLock: opts.RedisLock, errorCount: make(map[uint64]int), maxFeeCapExpression: expression, extraBacklog: opts.ExtraBacklog, @@ -289,6 +286,8 @@ func (p *DataPoster) MaxMempoolTransactions() uint64 { return p.config().MaxMempoolTransactions } +var ErrExceedsMaxMempoolSize = errors.New("posting this transaction will exceed max mempool size") + // Does basic check whether posting transaction with specified nonce would // result in exceeding maximum queue length or maximum transactions in mempool. func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64) error { @@ -311,7 +310,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64) err return fmt.Errorf("getting nonce of a dataposter sender: %w", err) } if nextNonce >= cfg.MaxMempoolTransactions+unconfirmedNonce { - return fmt.Errorf("posting a transaction with nonce: %d will exceed max mempool size: %d, unconfirmed nonce: %d", nextNonce, cfg.MaxMempoolTransactions, unconfirmedNonce) + return fmt.Errorf("%w: transaction nonce: %d, unconfirmed nonce: %d, max mempool size: %d", ErrExceedsMaxMempoolSize, nextNonce, unconfirmedNonce, cfg.MaxMempoolTransactions) } } return nil @@ -534,7 +533,7 @@ func (p *DataPoster) PostTransaction(ctx context.Context, dataCreatedAt time.Tim return nil, err } if nonce != expectedNonce { - return nil, fmt.Errorf("data poster expected next transaction to have nonce %v but was requested to post transaction with nonce %v", expectedNonce, nonce) + return nil, fmt.Errorf("%w: data poster expected next transaction to have nonce %v but was requested to post transaction with nonce %v", storage.ErrStorageRace, expectedNonce, nonce) } err = p.updateBalance(ctx) @@ -775,16 +774,10 @@ func (p *DataPoster) Start(ctxIn context.Context) { log.Error("Failed to fetch tx queue contents", "err", err) return minWait } - tryToReplace, err := p.redisLock.CouldAcquireLock(ctx) - if err != nil { - log.Warn("Error checking if we could acquire redis lock", "err", err) - // Might as well try, worst case we hit contention on redis and fail - tryToReplace = true - } for index, tx := range queueContents { backlogOfBatches := len(queueContents) - index - 1 replacing := false - if tryToReplace && now.After(tx.NextReplacement) { + if now.After(tx.NextReplacement) { replacing = true err := p.replaceTx(ctx, tx, uint64(backlogOfBatches)) p.maybeLogError(err, tx, "failed to replace-by-fee transaction") diff --git a/arbnode/node.go b/arbnode/node.go index 8be2a982cf..accca10e47 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode/dataposter" "github.com/offchainlabs/nitro/arbnode/dataposter/storage" - "github.com/offchainlabs/nitro/arbnode/redislock" "github.com/offchainlabs/nitro/arbnode/resourcemanager" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/broadcastclient" @@ -313,13 +312,6 @@ func StakerDataposter( if err != nil { return nil, fmt.Errorf("creating redis client from url: %w", err) } - lockCfgFetcher := func() *redislock.SimpleCfg { - return &cfg.Staker.RedisLock - } - redisLock, err := redislock.NewSimple(redisC, lockCfgFetcher, func() bool { return syncMonitor.Synced() }) - if err != nil { - return nil, err - } dpCfg := func() *dataposter.DataPosterConfig { return &cfg.Staker.DataPoster } @@ -335,7 +327,6 @@ func StakerDataposter( HeaderReader: l1Reader, Auth: transactOpts, RedisClient: redisC, - RedisLock: redisLock, Config: dpCfg, MetadataRetriever: mdRetriever, RedisKey: sender + ".staker-data-poster.queue", diff --git a/arbnode/redislock/redis.go b/arbnode/redislock/redis.go index fe46612527..09296e3c79 100644 --- a/arbnode/redislock/redis.go +++ b/arbnode/redislock/redis.go @@ -29,6 +29,7 @@ type Simple struct { } type SimpleCfg struct { + Enable bool `koanf:"enable"` MyId string `koanf:"my-id"` LockoutDuration time.Duration `koanf:"lockout-duration" reload:"hot"` RefreshDuration time.Duration `koanf:"refresh-duration" reload:"hot"` @@ -39,6 +40,7 @@ type SimpleCfg struct { type SimpleCfgFetcher func() *SimpleCfg func AddConfigOptions(prefix string, f *flag.FlagSet) { + f.Bool(prefix+".enable", DefaultCfg.Enable, "if false, always treat this as locked and don't write the lock to redis") f.String(prefix+".my-id", "", "this node's id prefix when acquiring the lock (optional)") f.Duration(prefix+".lockout-duration", DefaultCfg.LockoutDuration, "how long lock is held") f.Duration(prefix+".refresh-duration", DefaultCfg.RefreshDuration, "how long between consecutive calls to redis") @@ -60,6 +62,7 @@ func NewSimple(client redis.UniversalClient, config SimpleCfgFetcher, readyToLoc } var DefaultCfg = SimpleCfg{ + Enable: true, LockoutDuration: time.Minute, RefreshDuration: time.Second * 10, Key: "", @@ -137,7 +140,7 @@ func (l *Simple) AttemptLock(ctx context.Context) bool { } func (l *Simple) Locked() bool { - if l.client == nil { + if l.client == nil || !l.config().Enable { return true } return time.Now().Before(atomicTimeRead(&l.lockedUntil)) diff --git a/staker/staker.go b/staker/staker.go index 4f35c1bc9a..5b930dde58 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -87,7 +87,6 @@ type L1ValidatorConfig struct { GasRefunderAddress string `koanf:"gas-refunder-address"` DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"` RedisUrl string `koanf:"redis-url"` - RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"` ExtraGas uint64 `koanf:"extra-gas" reload:"hot"` Dangerous DangerousConfig `koanf:"dangerous"` ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"` @@ -154,7 +153,6 @@ var DefaultL1ValidatorConfig = L1ValidatorConfig{ GasRefunderAddress: "", DataPoster: dataposter.DefaultDataPosterConfigForValidator, RedisUrl: "", - RedisLock: redislock.DefaultCfg, ExtraGas: 50000, Dangerous: DefaultDangerousConfig, ParentChainWallet: DefaultValidatorL1WalletConfig, @@ -175,7 +173,6 @@ var TestL1ValidatorConfig = L1ValidatorConfig{ GasRefunderAddress: "", DataPoster: dataposter.TestDataPosterConfigForValidator, RedisUrl: "", - RedisLock: redislock.DefaultCfg, ExtraGas: 50000, Dangerous: DefaultDangerousConfig, ParentChainWallet: DefaultValidatorL1WalletConfig, From 063164e6be44f29f7f562129c415d9e47f8515de Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 22 Dec 2023 11:47:09 -0700 Subject: [PATCH 15/32] Fix redis lock removal from staker config --- staker/staker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/staker/staker.go b/staker/staker.go index 5b930dde58..5fa6c400c3 100644 --- a/staker/staker.go +++ b/staker/staker.go @@ -21,7 +21,6 @@ import ( flag "github.com/spf13/pflag" "github.com/offchainlabs/nitro/arbnode/dataposter" - "github.com/offchainlabs/nitro/arbnode/redislock" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/cmd/genericconf" "github.com/offchainlabs/nitro/staker/txbuilder" @@ -202,7 +201,6 @@ func L1ValidatorConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".redis-url", DefaultL1ValidatorConfig.RedisUrl, "redis url for L1 validator") f.Uint64(prefix+".extra-gas", DefaultL1ValidatorConfig.ExtraGas, "use this much more gas than estimation says is necessary to post transactions") dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfigForValidator) - redislock.AddConfigOptions(prefix+".redis-lock", f) DangerousConfigAddOptions(prefix+".dangerous", f) genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultL1ValidatorConfig.ParentChainWallet.Pathname) } From 8f9bd0e3f93fa0bbf94173177e8a0d9f44164d96 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 22 Dec 2023 12:06:10 -0700 Subject: [PATCH 16/32] Fix simple redis lock test --- arbnode/simple_redis_lock_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/arbnode/simple_redis_lock_test.go b/arbnode/simple_redis_lock_test.go index b7506145c3..c9dd576749 100644 --- a/arbnode/simple_redis_lock_test.go +++ b/arbnode/simple_redis_lock_test.go @@ -48,6 +48,7 @@ func simpleRedisLockTest(t *testing.T, redisKeySuffix string, chosen int, backgo Require(t, redisClient.Del(ctx, redisKey).Err()) conf := &redislock.SimpleCfg{ + Enable: true, LockoutDuration: test_delay * test_attempts * 10, RefreshDuration: test_delay * 2, Key: redisKey, From 32883f183ceb4073bffec6d6394ddd87aa36281a Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 22 Dec 2023 20:27:51 +0000 Subject: [PATCH 17/32] fix typo --- cmd/nitro/init.go | 6 +++--- .../prunning.go => pruning/pruning.go} | 2 +- .../{prunning_test.go => pruning_test.go} | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) rename cmd/{prunning/prunning.go => pruning/pruning.go} (99%) rename system_tests/{prunning_test.go => pruning_test.go} (79%) diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 00012ac62d..ada195b5c4 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -33,7 +33,7 @@ import ( "github.com/offchainlabs/nitro/cmd/chaininfo" "github.com/offchainlabs/nitro/cmd/conf" "github.com/offchainlabs/nitro/cmd/ipfshelper" - "github.com/offchainlabs/nitro/cmd/prunning" + "github.com/offchainlabs/nitro/cmd/pruning" "github.com/offchainlabs/nitro/cmd/util" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/statetransfer" @@ -167,7 +167,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err != nil { return chainDb, nil, err } - err = prunning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) + err = pruning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) if err != nil { return chainDb, nil, fmt.Errorf("error pruning: %w", err) } @@ -363,7 +363,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo return chainDb, l2BlockChain, err } - err = prunning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) + err = pruning.PruneChainDb(ctx, chainDb, stack, &config.Init, cacheConfig, l1Client, rollupAddrs, config.Node.ValidatorRequired()) if err != nil { return chainDb, nil, fmt.Errorf("error pruning: %w", err) } diff --git a/cmd/prunning/prunning.go b/cmd/pruning/pruning.go similarity index 99% rename from cmd/prunning/prunning.go rename to cmd/pruning/pruning.go index 5efdf8e643..68d89302f0 100644 --- a/cmd/prunning/prunning.go +++ b/cmd/pruning/pruning.go @@ -1,4 +1,4 @@ -package prunning +package pruning import ( "context" diff --git a/system_tests/prunning_test.go b/system_tests/pruning_test.go similarity index 79% rename from system_tests/prunning_test.go rename to system_tests/pruning_test.go index d522458209..e6eefdabd5 100644 --- a/system_tests/prunning_test.go +++ b/system_tests/pruning_test.go @@ -10,7 +10,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/offchainlabs/nitro/cmd/conf" - "github.com/offchainlabs/nitro/cmd/prunning" + "github.com/offchainlabs/nitro/cmd/pruning" "github.com/offchainlabs/nitro/execution/gethexec" "github.com/offchainlabs/nitro/util/testhelpers" ) @@ -28,7 +28,7 @@ func countStateEntries(db ethdb.Iteratee) int { return entries } -func TestPrunning(t *testing.T) { +func TestPruning(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -67,7 +67,7 @@ func TestPrunning(t *testing.T) { chainDb, err := stack.OpenDatabase("chaindb", 0, 0, "", false) Require(t, err) defer chainDb.Close() - chainDbEntriesBeforePrunning := countStateEntries(chainDb) + chainDbEntriesBeforePruning := countStateEntries(chainDb) prand := testhelpers.NewPseudoRandomDataSource(t, 1) var testKeys [][]byte @@ -88,21 +88,21 @@ func TestPrunning(t *testing.T) { initConfig := conf.InitConfigDefault initConfig.Prune = "full" coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &builder.execConfig.Caching) - err = prunning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false) + err = pruning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false) Require(t, err) for _, key := range testKeys { if has, _ := chainDb.Has(key); has { - Fatal(t, "test key hasn't been prunned as expected") + Fatal(t, "test key hasn't been pruned as expected") } } - chainDbEntriesAfterPrunning := countStateEntries(chainDb) - t.Log("db entries pre-prunning:", chainDbEntriesBeforePrunning) - t.Log("db entries post-prunning:", chainDbEntriesAfterPrunning) + chainDbEntriesAfterPruning := countStateEntries(chainDb) + t.Log("db entries pre-pruning:", chainDbEntriesBeforePruning) + t.Log("db entries post-pruning:", chainDbEntriesAfterPruning) - if chainDbEntriesAfterPrunning >= chainDbEntriesBeforePrunning { - Fatal(t, "The db doesn't have less entries after prunning then before. Before:", chainDbEntriesBeforePrunning, "After:", chainDbEntriesAfterPrunning) + if chainDbEntriesAfterPruning >= chainDbEntriesBeforePruning { + Fatal(t, "The db doesn't have less entries after pruning then before. Before:", chainDbEntriesBeforePruning, "After:", chainDbEntriesAfterPruning) } }() builder := NewNodeBuilder(ctx).DefaultConfig(t, true) From cb4d3c5374be0d9ed155fe6ed2514b27747f7366 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik Date: Fri, 22 Dec 2023 20:30:11 +0000 Subject: [PATCH 18/32] make only one test tx after pruning --- system_tests/pruning_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/system_tests/pruning_test.go b/system_tests/pruning_test.go index e6eefdabd5..ef82c0466e 100644 --- a/system_tests/pruning_test.go +++ b/system_tests/pruning_test.go @@ -111,15 +111,9 @@ func TestPruning(t *testing.T) { defer cancel() builder.L2Info.GenerateAccount("User2") - var txs []*types.Transaction - for i := uint64(0); i < 10; i++ { - tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, common.Big1, nil) - txs = append(txs, tx) - err := builder.L2.Client.SendTransaction(ctx, tx) - Require(t, err) - } - for _, tx := range txs { - _, err := builder.L2.EnsureTxSucceeded(tx) - Require(t, err) - } + tx := builder.L2Info.PrepareTx("Owner", "User2", builder.L2Info.TransferGas, common.Big1, nil) + err := builder.L2.Client.SendTransaction(ctx, tx) + Require(t, err) + _, err = builder.L2.EnsureTxSucceeded(tx) + Require(t, err) } From 51e9b557371048362623878df2939c3c5ffbcacd Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 22 Dec 2023 16:23:15 -0700 Subject: [PATCH 19/32] Clean up code a bit to address two comments --- arbnode/batch_poster.go | 7 +++++-- arbnode/dataposter/data_poster.go | 5 ----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 870df2e0d5..8c0fdd332a 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -1180,6 +1180,10 @@ func (b *BatchPoster) Start(ctxIn context.Context) { b.firstEphemeralError = time.Time{} } if err != nil { + if ctx.Err() != nil { + // Shutting down. No need to print the context canceled error. + return 0 + } b.building = nil logLevel := log.Error if b.firstEphemeralError == (time.Time{}) { @@ -1194,8 +1198,7 @@ func (b *BatchPoster) Start(ctxIn context.Context) { ignoreAtFirst := errors.Is(err, dataposter.ErrExceedsMaxMempoolSize) || errors.Is(err, storage.ErrStorageRace) || errors.Is(err, ErrNormalGasEstimationFailed) || - errors.Is(err, AccumulatorNotFoundErr) || - errors.Is(err, context.Canceled) + errors.Is(err, AccumulatorNotFoundErr) if sinceFirstEphemeralError < time.Minute { if ignoreAtFirst { logLevel = log.Debug diff --git a/arbnode/dataposter/data_poster.go b/arbnode/dataposter/data_poster.go index 673fd317ee..ff5a945a17 100644 --- a/arbnode/dataposter/data_poster.go +++ b/arbnode/dataposter/data_poster.go @@ -83,11 +83,6 @@ type DataPoster struct { // This can be local or external, hence the context parameter. type signerFn func(context.Context, common.Address, *types.Transaction) (*types.Transaction, error) -type AttemptLocker interface { - AttemptLock(context.Context) bool - CouldAcquireLock(context.Context) (bool, error) -} - func parseReplacementTimes(val string) ([]time.Duration, error) { var res []time.Duration var lastReplacementTime time.Duration From ac8dbc95784209c3419855bc55db0dbfaf30f150 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Fri, 22 Dec 2023 22:15:07 -0700 Subject: [PATCH 20/32] Log invalid txs in the STF when the chain is in debug mode --- arbos/block_processor.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/arbos/block_processor.go b/arbos/block_processor.go index 4864a31255..f1838132a8 100644 --- a/arbos/block_processor.go +++ b/arbos/block_processor.go @@ -42,6 +42,21 @@ var EmitReedeemScheduledEvent func(*vm.EVM, uint64, uint64, [32]byte, [32]byte, var EmitTicketCreatedEvent func(*vm.EVM, [32]byte) error var gasUsedSinceStartupCounter = metrics.NewRegisteredCounter("arb/gas_used", nil) +// A helper struct that implements String() by marshalling to JSON. +// This is useful for logging because it's lazy, so if the log level is too high to print the transaction, +// it doesn't waste compute marshalling the transaction when the result wouldn't be used. +type printTxAsJson struct { + tx *types.Transaction +} + +func (p printTxAsJson) String() string { + json, err := p.tx.MarshalJSON() + if err != nil { + return fmt.Sprintf("[error marshalling tx: %v]", err) + } + return string(json) +} + type L1Info struct { poster common.Address l1BlockNumber uint64 @@ -357,7 +372,11 @@ func ProduceBlockAdvanced( hooks.TxErrors = append(hooks.TxErrors, err) if err != nil { - log.Debug("error applying transaction", "tx", tx, "err", err) + logLevel := log.Debug + if chainConfig.DebugMode() { + logLevel = log.Warn + } + logLevel("error applying transaction", "tx", printTxAsJson{tx}, "err", err) if !hooks.DiscardInvalidTxsEarly { // we'll still deduct a TxGas's worth from the block-local rate limiter even if the tx was invalid blockGasLeft = arbmath.SaturatingUSub(blockGasLeft, params.TxGas) From 279dea45f9f0edd123f2e5e7155f7275d903307a Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sat, 23 Dec 2023 11:37:06 -0700 Subject: [PATCH 21/32] Fix enabling pprof on the relay --- cmd/relay/relay.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/relay/relay.go b/cmd/relay/relay.go index b25aadf57b..40f4f26eec 100644 --- a/cmd/relay/relay.go +++ b/cmd/relay/relay.go @@ -6,7 +6,6 @@ package main import ( "context" "fmt" - "net/http" "os" "os/signal" "syscall" @@ -22,10 +21,6 @@ import ( "github.com/offchainlabs/nitro/relay" ) -func init() { - http.DefaultServeMux = http.NewServeMux() -} - func main() { if err := startup(); err != nil { log.Error("Error running relay", "err", err) From e0f35dd40eaefa6fff4e6077ebef58cf5d0e2e74 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sat, 23 Dec 2023 13:02:03 -0700 Subject: [PATCH 22/32] Fix memory leak in relay --- broadcaster/backlog/backlog.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/broadcaster/backlog/backlog.go b/broadcaster/backlog/backlog.go index 0ef32167fe..461ec3a6f3 100644 --- a/broadcaster/backlog/backlog.go +++ b/broadcaster/backlog/backlog.go @@ -77,7 +77,6 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { nextSegment := newBacklogSegment() segment.nextSegment.Store(nextSegment) prevMsgIdx = segment.End() - nextSegment.previousSegment.Store(segment) segment = nextSegment b.tail.Store(segment) } @@ -253,10 +252,9 @@ type BacklogSegment interface { // backlogSegment stores messages up to a limit defined by the backlog. It also // points to the next backlogSegment in the list. type backlogSegment struct { - messagesLock sync.RWMutex - messages []*m.BroadcastFeedMessage - nextSegment atomic.Pointer[backlogSegment] - previousSegment atomic.Pointer[backlogSegment] + messagesLock sync.RWMutex + messages []*m.BroadcastFeedMessage + nextSegment atomic.Pointer[backlogSegment] } // newBacklogSegment creates a backlogSegment object with an empty slice of From fd5edcb829a9c440edffcfc7961e4668c4ab9c5c Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sat, 23 Dec 2023 13:21:31 -0700 Subject: [PATCH 23/32] Fix race conditions in backlog.reset() --- broadcaster/backlog/backlog.go | 26 ++++++++++++++------------ broadcaster/backlog/backlog_test.go | 8 ++++---- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/broadcaster/backlog/backlog.go b/broadcaster/backlog/backlog.go index 461ec3a6f3..549a38ff86 100644 --- a/broadcaster/backlog/backlog.go +++ b/broadcaster/backlog/backlog.go @@ -35,18 +35,18 @@ type Backlog interface { type backlog struct { head atomic.Pointer[backlogSegment] tail atomic.Pointer[backlogSegment] - lookupByIndex *containers.SyncMap[uint64, *backlogSegment] + lookupByIndex atomic.Pointer[containers.SyncMap[uint64, *backlogSegment]] config ConfigFetcher messageCount atomic.Uint64 } // NewBacklog creates a backlog. func NewBacklog(c ConfigFetcher) Backlog { - lookup := &containers.SyncMap[uint64, *backlogSegment]{} - return &backlog{ - lookupByIndex: lookup, - config: c, + b := &backlog{ + config: c, } + b.lookupByIndex.Store(&containers.SyncMap[uint64, *backlogSegment]{}) + return b } // Head return the head backlogSegment within the backlog. @@ -63,6 +63,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { b.delete(uint64(bm.ConfirmedSequenceNumberMessage.SequenceNumber)) } + lookupByIndex := b.lookupByIndex.Load() for _, msg := range bm.Messages { segment := b.tail.Load() if segment == nil { @@ -95,7 +96,7 @@ func (b *backlog) Append(bm *m.BroadcastMessage) error { } else if err != nil { return err } - b.lookupByIndex.Store(uint64(msg.SequenceNumber), segment) + lookupByIndex.Store(uint64(msg.SequenceNumber), segment) b.messageCount.Add(1) } @@ -160,7 +161,7 @@ func (b *backlog) delete(confirmed uint64) { } if confirmed > tail.End() { - log.Error("confirmed sequence number is past the end of stored messages", "confirmed sequence number", confirmed, "last stored sequence number", tail.End()) + log.Warn("confirmed sequence number is past the end of stored messages", "confirmed sequence number", confirmed, "last stored sequence number", tail.End()) b.reset() return } @@ -211,14 +212,15 @@ func (b *backlog) delete(confirmed uint64) { // removeFromLookup removes all entries from the head segment's start index to // the given confirmed index. func (b *backlog) removeFromLookup(start, end uint64) { + lookupByIndex := b.lookupByIndex.Load() for i := start; i <= end; i++ { - b.lookupByIndex.Delete(i) + lookupByIndex.Delete(i) } } // Lookup attempts to find the backlogSegment storing the given message index. func (b *backlog) Lookup(i uint64) (BacklogSegment, error) { - segment, ok := b.lookupByIndex.Load(i) + segment, ok := b.lookupByIndex.Load().Load(i) if !ok { return nil, fmt.Errorf("error finding backlog segment containing message with SequenceNumber %d", i) } @@ -233,9 +235,9 @@ func (s *backlog) Count() uint64 { // reset removes all segments from the backlog. func (b *backlog) reset() { - b.head = atomic.Pointer[backlogSegment]{} - b.tail = atomic.Pointer[backlogSegment]{} - b.lookupByIndex = &containers.SyncMap[uint64, *backlogSegment]{} + b.head.Store(nil) + b.tail.Store(nil) + b.lookupByIndex.Store(&containers.SyncMap[uint64, *backlogSegment]{}) b.messageCount.Store(0) } diff --git a/broadcaster/backlog/backlog_test.go b/broadcaster/backlog/backlog_test.go index ab25a523f7..ee712de9ed 100644 --- a/broadcaster/backlog/backlog_test.go +++ b/broadcaster/backlog/backlog_test.go @@ -57,9 +57,9 @@ func validateBroadcastMessage(t *testing.T, bm *m.BroadcastMessage, expectedCoun func createDummyBacklog(indexes []arbutil.MessageIndex) (*backlog, error) { b := &backlog{ - lookupByIndex: &containers.SyncMap[uint64, *backlogSegment]{}, - config: func() *Config { return &DefaultTestConfig }, + config: func() *Config { return &DefaultTestConfig }, } + b.lookupByIndex.Store(&containers.SyncMap[uint64, *backlogSegment]{}) bm := &m.BroadcastMessage{Messages: m.CreateDummyBroadcastMessages(indexes)} err := b.Append(bm) return b, err @@ -161,9 +161,9 @@ func TestDeleteInvalidBacklog(t *testing.T) { lookup := &containers.SyncMap[uint64, *backlogSegment]{} lookup.Store(40, s) b := &backlog{ - lookupByIndex: lookup, - config: func() *Config { return &DefaultTestConfig }, + config: func() *Config { return &DefaultTestConfig }, } + b.lookupByIndex.Store(lookup) b.messageCount.Store(2) b.head.Store(s) b.tail.Store(s) From d7b8d02dac1914643c17fff4ba127c425df3b718 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 24 Dec 2023 10:28:46 -0700 Subject: [PATCH 24/32] Fix Broadcaster BroadcastMessages sequence numbers --- arbnode/node.go | 2 +- broadcaster/broadcaster.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index accca10e47..951403244f 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -764,7 +764,7 @@ func (n *Node) Start(ctx context.Context) error { return fmt.Errorf("error initializing feed broadcast server: %w", err) } } - if n.InboxTracker != nil && n.BroadcastServer != nil && config.Sequencer { + if n.InboxTracker != nil && n.BroadcastServer != nil { // Even if the sequencer coordinator will populate this backlog, // we want to make sure it's populated before any clients connect. err = n.InboxTracker.PopulateFeedBacklog(n.BroadcastServer) diff --git a/broadcaster/broadcaster.go b/broadcaster/broadcaster.go index 38ffb0696c..ed3088ca2e 100644 --- a/broadcaster/broadcaster.go +++ b/broadcaster/broadcaster.go @@ -89,8 +89,8 @@ func (b *Broadcaster) BroadcastMessages(messages []arbostypes.MessageWithMetadat } }() var feedMessages []*m.BroadcastFeedMessage - for _, msg := range messages { - bfm, err := b.NewBroadcastFeedMessage(msg, seq) + for i, msg := range messages { + bfm, err := b.NewBroadcastFeedMessage(msg, seq+arbutil.MessageIndex(i)) if err != nil { return err } From 11cd4efb596c76d5f15091cf1be167bbf9880969 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 24 Dec 2023 10:42:17 -0700 Subject: [PATCH 25/32] Fix unused variable --- arbnode/node.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/arbnode/node.go b/arbnode/node.go index 951403244f..45ef7206e1 100644 --- a/arbnode/node.go +++ b/arbnode/node.go @@ -731,8 +731,6 @@ func CreateNode( } func (n *Node) Start(ctx context.Context) error { - // config is the static config at start, not a dynamic config - config := n.configFetcher.Get() execClient, ok := n.Execution.(*gethexec.ExecutionNode) if !ok { execClient = nil From 96fe66249283f57acfa8cf55ebfd5c485b311d81 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 24 Dec 2023 11:49:03 -0700 Subject: [PATCH 26/32] Revert "Merge pull request #2018 from OffchainLabs/linters" This reverts commit 1786f770eacb3d1ceb90fc019b37f69ba198e45c, reversing changes made to fd8d0b4295e638157b102e3cff91cf41a389c35c. --- Makefile | 31 --------------------- arbos/l1pricing/l1pricing.go | 6 ++-- go.mod | 2 -- go.sum | 16 ----------- linter/comparesame/comparesame.go | 12 -------- linter/cryptorand/cryptorand.go | 13 --------- linter/errcheck/errcheck.go | 12 -------- linter/featureconfig/featureconfig.go | 12 -------- linter/gocognit/gocognit.go | 13 --------- linter/ineffassign/ineffassign.go | 12 -------- linter/interfacechecker/interfacechecker.go | 12 -------- linter/logruswitherror/logruswitherror.go | 13 --------- linter/recursivelock/recursivelock.go | 1 - linter/shadowpredecl/shadowpredecl.go | 13 --------- linter/slicedirect/slicedirect.go | 13 --------- linter/uintcast/uintcast.go | 13 --------- 16 files changed, 3 insertions(+), 191 deletions(-) delete mode 100644 linter/comparesame/comparesame.go delete mode 100644 linter/cryptorand/cryptorand.go delete mode 100644 linter/errcheck/errcheck.go delete mode 100644 linter/featureconfig/featureconfig.go delete mode 100644 linter/gocognit/gocognit.go delete mode 100644 linter/ineffassign/ineffassign.go delete mode 100644 linter/interfacechecker/interfacechecker.go delete mode 100644 linter/logruswitherror/logruswitherror.go delete mode 100644 linter/shadowpredecl/shadowpredecl.go delete mode 100644 linter/slicedirect/slicedirect.go delete mode 100644 linter/uintcast/uintcast.go diff --git a/Makefile b/Makefile index d7f4599db3..42fd1e3595 100644 --- a/Makefile +++ b/Makefile @@ -308,37 +308,6 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(arbitrator_pro .make/lint: $(DEP_PREDICATE) build-node-deps $(ORDER_ONLY_PREDICATE) .make go run ./linter/recursivelock ./... - go run ./linter/comparesame ./... - - # Disabled since we have a lot of use of math/rand package. - # We should probably move to crypto/rand at some point even though most of - # our uses doesn't seem to be security sensitive. - # TODO fix this and enable. - # go run ./linter/cryptorand ./... - - # This yields lot of legitimate warnings, most of which in practice would - # probably never happen. - # # TODO fix this and enable. - # go run ./linter/errcheck ./... - - go run ./linter/featureconfig ./... - - # Disabled since we have high cognitive complexity several places. - # TODO fix this and enable. - # go run ./linter/gocognit ./... - - go run ./linter/ineffassign ./... - go run ./linter/interfacechecker ./... - go run ./linter/logruswitherror ./... - - go run ./linter/shadowpredecl ./... - go run ./linter/slicedirect ./... - - # Disabled since it fails many places, although ones I looked into seem - # to be false positives logically. - # TODO fix this and enable and mark false positives with lint ignore. - # go run ./linter/uintcast ./... - go run ./linter/koanf ./... go run ./linter/pointercheck ./... golangci-lint run --fix diff --git a/arbos/l1pricing/l1pricing.go b/arbos/l1pricing/l1pricing.go index be5540b601..27ecae8b85 100644 --- a/arbos/l1pricing/l1pricing.go +++ b/arbos/l1pricing/l1pricing.go @@ -259,11 +259,11 @@ func (ps *L1PricingState) AddToL1FeesAvailable(delta *big.Int) (*big.Int, error) if err != nil { return nil, err } - newFee := new(big.Int).Add(old, delta) - if err := ps.SetL1FeesAvailable(newFee); err != nil { + new := new(big.Int).Add(old, delta) + if err := ps.SetL1FeesAvailable(new); err != nil { return nil, err } - return newFee, nil + return new, nil } func (ps *L1PricingState) TransferFromL1FeesAvailable( diff --git a/go.mod b/go.mod index f87887e18c..e504542a2f 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,6 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b // indirect github.com/gorilla/mux v1.8.0 // indirect - github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/graph-gophers/graphql-go v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/h2non/filetype v1.0.6 // indirect @@ -254,7 +253,6 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/urfave/cli/v2 v2.25.7 // indirect - github.com/uudashr/gocognit v1.0.5 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect diff --git a/go.sum b/go.sum index 4e3f0f62b4..0a9649c434 100644 --- a/go.sum +++ b/go.sum @@ -581,9 +581,6 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gostaticanalysis/comment v1.4.2 h1:hlnx5+S2fY9Zo9ePo4AhgYsYHbM2+eAv8m/s1JiCd6Q= -github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= -github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -628,7 +625,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -1424,11 +1420,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.4.0 h1:CtfRrOVZtbDj8rt1WXjklw0kqqJQwICrCKmlfUuBUUw= github.com/openzipkin/zipkin-go v0.4.0/go.mod h1:4c3sLeE8xjNqehmF5RpAFLPLJxXscc0R4l6Zg0P1tTQ= -github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -1637,8 +1628,6 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= -github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -1669,8 +1658,6 @@ github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6f github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/uudashr/gocognit v1.0.5 h1:rrSex7oHr3/pPLQ0xoWq108XMU8s678FJcQ+aSfOHa4= -github.com/uudashr/gocognit v1.0.5/go.mod h1:wgYz0mitoKOTysqxTDMOUXg+Jb5SvtihkfmugIZYpEA= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= @@ -1876,7 +1863,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= @@ -2153,10 +2139,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= diff --git a/linter/comparesame/comparesame.go b/linter/comparesame/comparesame.go deleted file mode 100644 index 1d91592e99..0000000000 --- a/linter/comparesame/comparesame.go +++ /dev/null @@ -1,12 +0,0 @@ -// Static Analyzer to ensure code does not contain comparisons of identical expressions. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/comparesame" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(comparesame.Analyzer) -} diff --git a/linter/cryptorand/cryptorand.go b/linter/cryptorand/cryptorand.go deleted file mode 100644 index a7cd1ca65a..0000000000 --- a/linter/cryptorand/cryptorand.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static Analyzer to ensure the crypto/rand package is used for randomness -// throughout the codebase. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/cryptorand" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(cryptorand.Analyzer) -} diff --git a/linter/errcheck/errcheck.go b/linter/errcheck/errcheck.go deleted file mode 100644 index 8275b2bab1..0000000000 --- a/linter/errcheck/errcheck.go +++ /dev/null @@ -1,12 +0,0 @@ -// Static analyzer to ensure that errors are handled in go code. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/errcheck" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(errcheck.Analyzer) -} diff --git a/linter/featureconfig/featureconfig.go b/linter/featureconfig/featureconfig.go deleted file mode 100644 index bb9c2dd3a1..0000000000 --- a/linter/featureconfig/featureconfig.go +++ /dev/null @@ -1,12 +0,0 @@ -// Static analyzer to prevent leaking globals in tests. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/featureconfig" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(featureconfig.Analyzer) -} diff --git a/linter/gocognit/gocognit.go b/linter/gocognit/gocognit.go deleted file mode 100644 index a50d0ac08d..0000000000 --- a/linter/gocognit/gocognit.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static analyzer that checks for high cognitive complexity and complains when -// it's too high. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/gocognit" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(gocognit.Analyzer) -} diff --git a/linter/ineffassign/ineffassign.go b/linter/ineffassign/ineffassign.go deleted file mode 100644 index 697e104822..0000000000 --- a/linter/ineffassign/ineffassign.go +++ /dev/null @@ -1,12 +0,0 @@ -// Static analyzer for detecting ineffectual assignments. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/ineffassign" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(ineffassign.Analyzer) -} diff --git a/linter/interfacechecker/interfacechecker.go b/linter/interfacechecker/interfacechecker.go deleted file mode 100644 index 50bb56f749..0000000000 --- a/linter/interfacechecker/interfacechecker.go +++ /dev/null @@ -1,12 +0,0 @@ -// Static analyzer to prevent conditional checks on select interfaces. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/interfacechecker" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(interfacechecker.Analyzer) -} diff --git a/linter/logruswitherror/logruswitherror.go b/linter/logruswitherror/logruswitherror.go deleted file mode 100644 index d0da1fcb12..0000000000 --- a/linter/logruswitherror/logruswitherror.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static analyzer to ensure that log statements do not use errors in -// templated log statements. Authors should use logrus.WithError(). -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/logruswitherror" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(logruswitherror.Analyzer) -} diff --git a/linter/recursivelock/recursivelock.go b/linter/recursivelock/recursivelock.go index 8c8caff382..9349759cf3 100644 --- a/linter/recursivelock/recursivelock.go +++ b/linter/recursivelock/recursivelock.go @@ -1,4 +1,3 @@ -// Static Analyzer for detecting nested or recursive mutex read lock statements. package main import ( diff --git a/linter/shadowpredecl/shadowpredecl.go b/linter/shadowpredecl/shadowpredecl.go deleted file mode 100644 index e51ff0da83..0000000000 --- a/linter/shadowpredecl/shadowpredecl.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static analyzer which disallows declaring constructs that shadow predeclared -// Go identifiers by having the same name. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/shadowpredecl" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(shadowpredecl.Analyzer) -} diff --git a/linter/slicedirect/slicedirect.go b/linter/slicedirect/slicedirect.go deleted file mode 100644 index b31404013d..0000000000 --- a/linter/slicedirect/slicedirect.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static analyzer to ensure that code does not contain applications of [:] -// on expressions which are already slices. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/slicedirect" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(slicedirect.Analyzer) -} diff --git a/linter/uintcast/uintcast.go b/linter/uintcast/uintcast.go deleted file mode 100644 index 1f0eb78575..0000000000 --- a/linter/uintcast/uintcast.go +++ /dev/null @@ -1,13 +0,0 @@ -// Static analyzer for detecting unsafe uint to int casts. -// Use `lint:ignore uintcast` with proper justification to ignore this check. -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/uintcast" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(uintcast.Analyzer) -} From 68917ae115f3a777420d7874e73dd8e7c97feddd Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Sun, 24 Dec 2023 11:49:17 -0700 Subject: [PATCH 27/32] Revert "Merge pull request #2016 from OffchainLabs/recursivelock-lint" This reverts commit 459a63be17fc52566c26c50af75f6f5617ae2c1d, reversing changes made to a0e63172ca4c6ed08612622cb1605da02b599c6a. --- Makefile | 1 - go.mod | 32 +++++++++--------- go.sum | 47 --------------------------- linter/recursivelock/recursivelock.go | 11 ------- 4 files changed, 15 insertions(+), 76 deletions(-) delete mode 100644 linter/recursivelock/recursivelock.go diff --git a/Makefile b/Makefile index 42fd1e3595..4221100961 100644 --- a/Makefile +++ b/Makefile @@ -307,7 +307,6 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(arbitrator_pro # strategic rules to minimize dependency building .make/lint: $(DEP_PREDICATE) build-node-deps $(ORDER_ONLY_PREDICATE) .make - go run ./linter/recursivelock ./... go run ./linter/koanf ./... go run ./linter/pointercheck ./... golangci-lint run --fix diff --git a/go.mod b/go.mod index e504542a2f..bdda6a61a1 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,11 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.10 github.com/aws/aws-sdk-go-v2/service/s3 v1.26.9 github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 + github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 github.com/codeclysm/extract/v3 v3.0.2 github.com/dgraph-io/badger/v3 v3.2103.2 github.com/enescakir/emoji v1.0.0 - github.com/ethereum/go-ethereum v1.13.1 + github.com/ethereum/go-ethereum v1.10.26 github.com/fatih/structtag v1.2.0 github.com/gdamore/tcell/v2 v2.6.0 github.com/google/go-cmp v0.5.9 @@ -73,7 +73,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect @@ -97,7 +97,7 @@ require ( github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 // indirect + github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -131,7 +131,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 // indirect - github.com/huin/goupnp v1.3.0 // indirect + github.com/huin/goupnp v1.1.0 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect @@ -195,7 +195,7 @@ require ( github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.21.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect - github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect + github.com/libp2p/go-libp2p-pubsub v0.9.0 // indirect github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.6.2 // indirect @@ -238,7 +238,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/prysmaticlabs/prysm/v4 v4.1.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.3 // indirect github.com/quic-go/qtls-go1-20 v0.2.3 // indirect @@ -246,13 +245,13 @@ require ( github.com/quic-go/webtransport-go v0.5.2 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rhnvrm/simples3 v0.6.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.3 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/samber/lo v1.36.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.11 // indirect - github.com/urfave/cli/v2 v2.25.7 // indirect + github.com/urfave/cli/v2 v2.24.1 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect @@ -262,7 +261,6 @@ require ( github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.7.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.7.0 // indirect @@ -310,12 +308,12 @@ require ( require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/VictoriaMetrics/fastcache v1.12.0 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect - github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect + github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-redis/redis/v8 v8.11.4 github.com/go-stack/stack v1.8.1 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect @@ -332,11 +330,11 @@ require ( github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/rs/cors v1.7.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/status-im/keycard-go v0.2.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect github.com/tyler-smith/go-bip39 v1.1.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect ) diff --git a/go.sum b/go.sum index 0a9649c434..bf8b4b826d 100644 --- a/go.sum +++ b/go.sum @@ -173,8 +173,6 @@ github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13P github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= -github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= -github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -200,7 +198,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2oZiOeReJRdU= github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= -github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -228,7 +225,6 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= @@ -236,8 +232,6 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -313,8 +307,6 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 h1:qwcF+vdFrvPSEUDSX5RVoRccG8a5DhOdWdQ4zN62zzo= -github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -368,7 +360,6 @@ github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gammazero/deque v0.2.1 h1:qSdsbG6pgp6nL7A0+K/B7s12mcCY/5l5SIUpMOl+dC0= @@ -376,8 +367,6 @@ github.com/gammazero/deque v0.2.1/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZ github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.6.0 h1:OKbluoP9VYmJwZwq/iLb4BxwKcwGthaa1YNBJIyCySg= @@ -414,8 +403,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -549,7 +536,6 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b h1:Qcx5LM0fSiks9uCyFZwDBUasd3lxd1RM0GYpL+Li5o4= github.com/google/pprof v0.0.0-20230405160723-4a4c7d95572b/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= @@ -654,12 +640,9 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.1.0 h1:gEe0Dp/lZmPZiDFzJJaOfUpOvv2MKUkoBX8lDrn9vKU= github.com/huin/goupnp v1.1.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= -github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -1075,8 +1058,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuD github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-pubsub v0.9.0 h1:mcLb4WzwhUG4OKb0rp1/bYMd/DYhvMyzJheQH3LMd1s= github.com/libp2p/go-libp2p-pubsub v0.9.0/go.mod h1:OEsj0Cc/BpkqikXRTrVspWU/Hx7bMZwHP+6vNMd+c7I= -github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= -github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= github.com/libp2p/go-libp2p-pubsub-router v0.6.0 h1:D30iKdlqDt5ZmLEYhHELCMRj8b4sFAqrUcshIUvVP/s= github.com/libp2p/go-libp2p-pubsub-router v0.6.0/go.mod h1:FY/q0/RBTKsLA7l4vqC2cbRbOvyDotg8PJQ7j8FDudE= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= @@ -1391,7 +1372,6 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1401,8 +1381,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= @@ -1486,8 +1464,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prysmaticlabs/prysm/v4 v4.1.1 h1:sbBkgfPzo/SGTJ5IimtsZSGECoRlhbowR1rEhTOdvHo= -github.com/prysmaticlabs/prysm/v4 v4.1.1/go.mod h1:+o907dc4mwEE0wJkQ8RrzCroC+q2WCzdCLtikwonw8c= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-19 v0.3.3 h1:wznEHvJwd+2X3PqftRha0SUKmGsnb6dfArMhy9PeJVE= @@ -1512,8 +1488,6 @@ github.com/rivo/tview v0.0.0-20230814110005-ccc2c8119703/go.mod h1:nVwGv4MP47T0j github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1539,8 +1513,6 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -1616,7 +1588,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= @@ -1625,18 +1596,12 @@ github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= -github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= @@ -1655,8 +1620,6 @@ github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60Nt github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -1721,8 +1684,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -1925,8 +1886,6 @@ golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -1989,7 +1948,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2042,10 +2000,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2053,8 +2009,6 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2149,7 +2103,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= diff --git a/linter/recursivelock/recursivelock.go b/linter/recursivelock/recursivelock.go deleted file mode 100644 index 9349759cf3..0000000000 --- a/linter/recursivelock/recursivelock.go +++ /dev/null @@ -1,11 +0,0 @@ -package main - -import ( - "github.com/prysmaticlabs/prysm/v4/tools/analyzers/recursivelock" - - "golang.org/x/tools/go/analysis/singlechecker" -) - -func main() { - singlechecker.Main(recursivelock.Analyzer) -} From 1fac15d8bd2b0c3152c937cd444ba89d4abdedd2 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 27 Dec 2023 11:38:27 -0700 Subject: [PATCH 28/32] Fix sequencer forwarder panic on lack of targets --- execution/gethexec/forwarder.go | 2 ++ execution/gethexec/sequencer.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/execution/gethexec/forwarder.go b/execution/gethexec/forwarder.go index dc3420f8de..7fec46c635 100644 --- a/execution/gethexec/forwarder.go +++ b/execution/gethexec/forwarder.go @@ -205,6 +205,8 @@ func (f *TxForwarder) Initialize(inctx context.Context) error { f.targets = targets if len(f.rpcClients) > 0 { f.enabled.Store(true) + } else if lastError == nil { + return errors.New("no non-empty forwarding targets specified") } else { return lastError } diff --git a/execution/gethexec/sequencer.go b/execution/gethexec/sequencer.go index 1c60702c85..b21a274527 100644 --- a/execution/gethexec/sequencer.go +++ b/execution/gethexec/sequencer.go @@ -487,7 +487,7 @@ func (s *Sequencer) CheckHealth(ctx context.Context) error { func (s *Sequencer) ForwardTarget() string { s.activeMutex.Lock() defer s.activeMutex.Unlock() - if s.forwarder == nil { + if s.forwarder == nil || len(s.forwarder.targets) == 0 { return "" } return s.forwarder.targets[0] @@ -497,7 +497,7 @@ func (s *Sequencer) ForwardTo(url string) error { s.activeMutex.Lock() defer s.activeMutex.Unlock() if s.forwarder != nil { - if s.forwarder.targets[0] == url { + if len(s.forwarder.targets) > 0 && s.forwarder.targets[0] == url { log.Warn("attempted to update sequencer forward target with existing target", "url", url) return nil } From 966a1724ac5bcf0eb94e9c245c44c6a33a6ba692 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 27 Dec 2023 12:04:48 -0700 Subject: [PATCH 29/32] Fix broken NodeInterface methods --- execution/gethexec/node.go | 2 +- system_tests/nodeinterface_test.go | 37 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index 7dd6e301fe..00337cc355 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -248,7 +248,7 @@ func CreateExecutionNode( } func (n *ExecutionNode) Initialize(ctx context.Context, arbnode interface{}, sync arbitrum.SyncProgressBackend) error { - n.ArbInterface.Initialize(n) + n.ArbInterface.Initialize(arbnode) err := n.Backend.Start() if err != nil { return fmt.Errorf("error starting geth backend: %w", err) diff --git a/system_tests/nodeinterface_test.go b/system_tests/nodeinterface_test.go index 40953a449d..3424a58e9e 100644 --- a/system_tests/nodeinterface_test.go +++ b/system_tests/nodeinterface_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/offchainlabs/nitro/arbos/util" "github.com/offchainlabs/nitro/solgen/go/node_interfacegen" @@ -73,3 +74,39 @@ func TestL2BlockRangeForL1(t *testing.T) { t.Fatalf("GetL2BlockRangeForL1 didn't fail for an invalid input") } } + +func TestGetL1Confirmations(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + builder := NewNodeBuilder(ctx).DefaultConfig(t, true) + cleanup := builder.Build(t) + defer cleanup() + + nodeInterface, err := node_interfacegen.NewNodeInterface(types.NodeInterfaceAddress, builder.L2.Client) + Require(t, err) + + genesisBlock, err := builder.L2.Client.BlockByNumber(ctx, big.NewInt(0)) + Require(t, err) + l1Confs, err := nodeInterface.GetL1Confirmations(&bind.CallOpts{}, genesisBlock.Hash()) + Require(t, err) + + numTransactions := 200 + + if l1Confs >= uint64(numTransactions) { + t.Fatalf("L1Confirmations for latest block %v is already %v (over %v)", genesisBlock.Number(), l1Confs, numTransactions) + } + + for i := 0; i < numTransactions; i++ { + builder.L1.TransferBalance(t, "User", "User", common.Big0, builder.L1Info) + } + + l1Confs, err = nodeInterface.GetL1Confirmations(&bind.CallOpts{}, genesisBlock.Hash()) + Require(t, err) + + // Allow a gap of 10 for asynchronicity, just in case + if l1Confs+10 < uint64(numTransactions) { + t.Fatalf("L1Confirmations for latest block %v is only %v (did not hit expected %v)", genesisBlock.Number(), l1Confs, numTransactions) + } +} From c75e6049a787889d5bbcb20787b49d47573254f9 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 27 Dec 2023 12:22:20 -0700 Subject: [PATCH 30/32] Allow the TxForwarder to be empty --- execution/gethexec/forwarder.go | 16 ++++++++++++---- execution/gethexec/sequencer.go | 6 +++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/execution/gethexec/forwarder.go b/execution/gethexec/forwarder.go index 7fec46c635..9783c604a1 100644 --- a/execution/gethexec/forwarder.go +++ b/execution/gethexec/forwarder.go @@ -151,7 +151,7 @@ func (f *TxForwarder) PublishTransaction(inctx context.Context, tx *types.Transa if err == nil || !f.tryNewForwarderErrors.MatchString(err.Error()) { return err } - log.Info("error forwarding transaction to a backup target", "target", f.targets[pos], "err", err) + log.Warn("error forwarding transaction to a backup target", "target", f.targets[pos], "err", err) } return errors.New("failed to publish transaction to any of the forwarding targets") } @@ -161,7 +161,9 @@ const maxHealthTimeout = 10 * time.Second // CheckHealth returns health of the highest priority forwarding target func (f *TxForwarder) CheckHealth(inctx context.Context) error { - if !f.enabled.Load() { + // If f.enabled is false, len(f.rpcClients) should always be greater than zero, + // but better safe than sorry. + if !f.enabled.Load() || len(f.rpcClients) == 0 { return ErrNoSequencer } f.healthMutex.Lock() @@ -205,8 +207,6 @@ func (f *TxForwarder) Initialize(inctx context.Context) error { f.targets = targets if len(f.rpcClients) > 0 { f.enabled.Store(true) - } else if lastError == nil { - return errors.New("no non-empty forwarding targets specified") } else { return lastError } @@ -232,6 +232,14 @@ func (f *TxForwarder) Started() bool { return true } +// Returns the URL of the first forwarding target, or an empty string if none are set. +func (f *TxForwarder) PrimaryTarget() string { + if len(f.targets) == 0 { + return "" + } + return f.targets[0] +} + type TxDropper struct{} func NewTxDropper() *TxDropper { diff --git a/execution/gethexec/sequencer.go b/execution/gethexec/sequencer.go index b21a274527..5db38cbb4d 100644 --- a/execution/gethexec/sequencer.go +++ b/execution/gethexec/sequencer.go @@ -487,17 +487,17 @@ func (s *Sequencer) CheckHealth(ctx context.Context) error { func (s *Sequencer) ForwardTarget() string { s.activeMutex.Lock() defer s.activeMutex.Unlock() - if s.forwarder == nil || len(s.forwarder.targets) == 0 { + if s.forwarder == nil { return "" } - return s.forwarder.targets[0] + return s.forwarder.PrimaryTarget() } func (s *Sequencer) ForwardTo(url string) error { s.activeMutex.Lock() defer s.activeMutex.Unlock() if s.forwarder != nil { - if len(s.forwarder.targets) > 0 && s.forwarder.targets[0] == url { + if s.forwarder.PrimaryTarget() == url { log.Warn("attempted to update sequencer forward target with existing target", "url", url) return nil } From 02bfe08a2da0ce08522c52f0507d5fc49927b841 Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Wed, 27 Dec 2023 12:26:01 -0700 Subject: [PATCH 31/32] Fix comment --- execution/gethexec/forwarder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/execution/gethexec/forwarder.go b/execution/gethexec/forwarder.go index 9783c604a1..984c7224e8 100644 --- a/execution/gethexec/forwarder.go +++ b/execution/gethexec/forwarder.go @@ -161,7 +161,7 @@ const maxHealthTimeout = 10 * time.Second // CheckHealth returns health of the highest priority forwarding target func (f *TxForwarder) CheckHealth(inctx context.Context) error { - // If f.enabled is false, len(f.rpcClients) should always be greater than zero, + // If f.enabled is true, len(f.rpcClients) should always be greater than zero, // but better safe than sorry. if !f.enabled.Load() || len(f.rpcClients) == 0 { return ErrNoSequencer From e2086163542bf28ee70b1dcfb2d302e4ed5b8cbf Mon Sep 17 00:00:00 2001 From: Lee Bousfield Date: Mon, 1 Jan 2024 21:58:48 -0600 Subject: [PATCH 32/32] Hide error when getting latest confirmed block --- util/headerreader/header_reader.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/util/headerreader/header_reader.go b/util/headerreader/header_reader.go index a5100f74fa..c5b8fa23f1 100644 --- a/util/headerreader/header_reader.go +++ b/util/headerreader/header_reader.go @@ -53,6 +53,7 @@ type HeaderReader struct { type cachedHeader struct { mutex sync.Mutex + blockTag string // "safe" or "finalized" rpcBlockNum *big.Int headWhenCached *types.Header header *types.Header @@ -135,8 +136,8 @@ func New(ctx context.Context, client arbutil.L1Interface, config ConfigFetcher, arbSys: arbSys, outChannels: make(map[chan<- *types.Header]struct{}), outChannelsBehind: make(map[chan<- *types.Header]struct{}), - safe: cachedHeader{rpcBlockNum: big.NewInt(rpc.SafeBlockNumber.Int64())}, - finalized: cachedHeader{rpcBlockNum: big.NewInt(rpc.FinalizedBlockNumber.Int64())}, + safe: cachedHeader{blockTag: "safe", rpcBlockNum: big.NewInt(rpc.SafeBlockNumber.Int64())}, + finalized: cachedHeader{blockTag: "finalized", rpcBlockNum: big.NewInt(rpc.FinalizedBlockNumber.Int64())}, }, nil } @@ -470,6 +471,11 @@ func (s *HeaderReader) getCached(ctx context.Context, c *cachedHeader) (*types.H } header, err := s.client.HeaderByNumber(ctx, c.rpcBlockNum) if err != nil { + if !errors.Is(err, context.Canceled) { + log.Warn("Failed to get latest confirmed block", "blockTag", c.blockTag, "err", err) + // Hide error to caller to avoid exposing potentially sensitive L1 information. + err = fmt.Errorf("failed to get latest %v block", c.blockTag) + } return nil, err } c.header = header