Merge branch 'master' into forcedelayed_on_init

Dockerfile

@@ -163,6 +163,7 @@ COPY ./scripts/download-machine.sh .
 RUN ./download-machine.sh consensus-v10 0x6b94a7fc388fd8ef3def759297828dc311761e88d8179c7ee8d3887dc554f3c3
 RUN ./download-machine.sh consensus-v10.1 0xda4e3ad5e7feacb817c21c8d0220da7650fe9051ece68a3f0b1c5d38bbb27b21
 RUN ./download-machine.sh consensus-v10.2 0x0754e09320c381566cc0449904c377a52bd34a6b9404432e80afd573b67f7b17
+RUN ./download-machine.sh consensus-v10.3 0xf559b6d4fa869472dabce70fe1c15221bdda837533dfd891916836975b434dec
 
 FROM golang:1.20-bullseye as node-builder
 WORKDIR /workspace

arbnode/batch_poster.go

@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 
 	"github.com/offchainlabs/nitro/arbnode/dataposter"
-	"github.com/offchainlabs/nitro/arbnode/dataposter/storage"
 	"github.com/offchainlabs/nitro/arbnode/redislock"
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbstate"
@@ -1109,22 +1108,21 @@ func (b *BatchPoster) Start(ctxIn context.Context) {
 			return b.config().PollInterval
 		}
 		posted, err := b.maybePostSequencerBatch(ctx)
-		ephemeralError := errors.Is(err, AccumulatorNotFoundErr) || errors.Is(err, storage.ErrStorageRace)
-		if !ephemeralError {
+		if err == nil {
 			b.firstEphemeralError = time.Time{}
 		}
 		if err != nil {
 			b.building = nil
 			logLevel := log.Error
-			if ephemeralError {
-				// Likely the inbox tracker just isn't caught up.
-				// Let's see if this error disappears naturally.
-				if b.firstEphemeralError == (time.Time{}) {
-					b.firstEphemeralError = time.Now()
-					logLevel = log.Debug
-				} else if time.Since(b.firstEphemeralError) < time.Minute {
-					logLevel = log.Debug
-				}
+			// Likely the inbox tracker just isn't caught up.
+			// Let's see if this error disappears naturally.
+			if b.firstEphemeralError == (time.Time{}) {
+				b.firstEphemeralError = time.Now()
+				logLevel = log.Warn
+			} else if time.Since(b.firstEphemeralError) < time.Minute {
+				logLevel = log.Warn
+			} else if time.Since(b.firstEphemeralError) < time.Minute*5 && strings.Contains(err.Error(), "will exceed max mempool size") {
+				logLevel = log.Warn
 			}
 			logLevel("error posting batch", "err", err)
 			return b.config().ErrorDelay

arbnode/dataposter/data_poster.go

@@ -823,7 +823,7 @@ var DefaultDataPosterConfig = DataPosterConfig{
 	AllocateMempoolBalance: true,
 	UseDBStorage:           true,
 	UseNoOpStorage:         false,
-	LegacyStorageEncoding:  true,
+	LegacyStorageEncoding:  false,
 	Dangerous:              DangerousConfig{ClearDBStorage: false},
 	ExternalSigner:         ExternalSignerCfg{Method: "eth_signTransaction"},
 }
@@ -847,6 +847,7 @@ var TestDataPosterConfig = DataPosterConfig{
 	AllocateMempoolBalance: true,
 	UseDBStorage:           false,
 	UseNoOpStorage:         false,
+	LegacyStorageEncoding:  false,
 	ExternalSigner:         ExternalSignerCfg{Method: "eth_signTransaction"},
 }
 

arbnode/delayed_sequencer.go

@@ -60,7 +60,7 @@ var TestDelayedSequencerConfig = DelayedSequencerConfig{
 	Enable:              true,
 	FinalizeDistance:    20,
 	RequireFullFinality: false,
-	UseMergeFinality:    true,
+	UseMergeFinality:    false,
 }
 
 func NewDelayedSequencer(l1Reader *headerreader.HeaderReader, reader *InboxReader, exec execution.ExecutionSequencer, coordinator *SeqCoordinator, config DelayedSequencerConfigFetcher) (*DelayedSequencer, error) {

arbnode/transaction_streamer.go

@@ -696,6 +696,7 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil
 	var oldMsg *arbostypes.MessageWithMetadata
 	var lastDelayedRead uint64
 	var hasNewConfirmedMessages bool
+	var cacheClearLen int
 
 	messagesAfterPos := messageStartPos + arbutil.MessageIndex(len(messages))
 	broadcastStartPos := arbutil.MessageIndex(atomic.LoadUint64(&s.broadcasterQueuedMessagesPos))
@@ -724,10 +725,13 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil
 		// Or no active broadcast reorg and broadcast messages start before or immediately after last L1 message
 		if messagesAfterPos >= broadcastStartPos {
 			broadcastSliceIndex := int(messagesAfterPos - broadcastStartPos)
+			messagesOldLen := len(messages)
 			if broadcastSliceIndex < len(s.broadcasterQueuedMessages) {
 				// Some cached feed messages can be used
 				messages = append(messages, s.broadcasterQueuedMessages[broadcastSliceIndex:]...)
 			}
+			// This calculation gives the exact length of cache which was appended to messages
+			cacheClearLen = broadcastSliceIndex + len(messages) - messagesOldLen
 		}
 
 		// L1 used or replaced broadcast cache items
@@ -800,8 +804,14 @@ func (s *TransactionStreamer) addMessagesAndEndBatchImpl(messageStartPos arbutil
 	}
 
 	if clearQueueOnSuccess {
-		s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[:0]
-		atomic.StoreUint64(&s.broadcasterQueuedMessagesPos, 0)
+		// Check if new messages were added at the end of cache, if they were, then dont remove those particular messages
+		if len(s.broadcasterQueuedMessages) > cacheClearLen {
+			s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[cacheClearLen:]
+			atomic.StoreUint64(&s.broadcasterQueuedMessagesPos, uint64(broadcastStartPos)+uint64(cacheClearLen))
+		} else {
+			s.broadcasterQueuedMessages = s.broadcasterQueuedMessages[:0]
+			atomic.StoreUint64(&s.broadcasterQueuedMessagesPos, 0)
+		}
 		s.broadcasterQueuedMessagesActiveReorg = false
 	}
 

arbos/arbosState/initialize.go

@@ -58,7 +58,7 @@ func InitializeArbosInDatabase(db ethdb.Database, initData statetransfer.InitDat
 	}
 
 	commit := func() (common.Hash, error) {
-		root, err := statedb.Commit(true)
+		root, err := statedb.Commit(chainConfig.ArbitrumChainParams.GenesisBlockNum, true)
 		if err != nil {
 			return common.Hash{}, err
 		}

arbos/parse_l2.go

@@ -166,8 +166,9 @@ func parseL2Message(rd io.Reader, poster common.Address, timestamp uint64, reque
 		if err := newTx.UnmarshalBinary(readBytes); err != nil {
 			return nil, err
 		}
-		if newTx.Type() >= types.ArbitrumDepositTxType {
-			// Should be unreachable due to UnmarshalBinary not accepting Arbitrum internal txs
+		if newTx.Type() >= types.ArbitrumDepositTxType || newTx.Type() == types.BlobTxType {
+			// Should be unreachable for Arbitrum types due to UnmarshalBinary not accepting Arbitrum internal txs
+			// and we want to disallow BlobTxType since Arbitrum doesn't support EIP-4844 txs yet.
 			return nil, types.ErrTxTypeNotSupported
 		}
 		return types.Transactions{newTx}, nil

broadcastclients/broadcastclients.go

@@ -166,7 +166,9 @@ func (bcs *BroadcastClients) Start(ctx context.Context) {
 					continue
 				}
 				lastConfirmed = cs
-				bcs.primaryRouter.forwardConfirmationChan <- cs
+				if bcs.primaryRouter.forwardConfirmationChan != nil {
+					bcs.primaryRouter.forwardConfirmationChan <- cs
+				}
 
 			// Secondary Feeds
 			case msg := <-bcs.secondaryRouter.messageChan:
@@ -187,7 +189,9 @@ func (bcs *BroadcastClients) Start(ctx context.Context) {
 					continue
 				}
 				lastConfirmed = cs
-				bcs.secondaryRouter.forwardConfirmationChan <- cs
+				if bcs.secondaryRouter.forwardConfirmationChan != nil {
+					bcs.secondaryRouter.forwardConfirmationChan <- cs
+				}
 
 			// Cycle buckets to get rid of old entries
 			case <-recentFeedItemsCleanup.C:

cmd/datool/datool.go

@@ -102,7 +102,6 @@ func parseClientStoreConfig(args []string) (*ClientStoreConfig, error) {
 	f.String("signing-wallet", "", "wallet containing ecdsa key to sign the message with")
 	f.String("signing-wallet-password", genericconf.PASSWORD_NOT_SET, "password to unlock the wallet, if not specified the user is prompted for the password")
 	f.Duration("das-retention-period", 24*time.Hour, "The period which DASes are requested to retain the stored batches.")
-	genericconf.ConfConfigAddOptions("conf", f)
 
 	k, err := confighelpers.BeginCommonParse(f, args)
 	if err != nil {
@@ -204,8 +203,6 @@ func parseRESTClientGetByHashConfig(args []string) (*RESTClientGetByHashConfig,
 	f.String("url", "http://localhost:9877", "URL of DAS server to connect to.")
 	f.String("data-hash", "", "hash of the message to retrieve, if starts with '0x' it's treated as hex encoded, otherwise base64 encoded")
 
-	genericconf.ConfConfigAddOptions("conf", f)
-
 	k, err := confighelpers.BeginCommonParse(f, args)
 	if err != nil {
 		return nil, err
@@ -267,7 +264,6 @@ func parseKeyGenConfig(args []string) (*KeyGenConfig, error) {
 	f.String("dir", "", "the directory to generate the keys in")
 	f.Bool("ecdsa", false, "generate an ECDSA keypair instead of BLS")
 	f.Bool("wallet", false, "generate the ECDSA keypair in a wallet file")
-	genericconf.ConfConfigAddOptions("conf", f)
 
 	k, err := confighelpers.BeginCommonParse(f, args)
 	if err != nil {

cmd/genericconf/config.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/node"
 	flag "github.com/spf13/pflag"
 )
 
@@ -107,16 +107,20 @@ func FileLoggingConfigAddOptions(prefix string, f *flag.FlagSet) {
 
 type RpcConfig struct {
 	MaxBatchResponseSize int `koanf:"max-batch-response-size"`
+	BatchRequestLimit    int `koanf:"batch-request-limit"`
 }
 
 var DefaultRpcConfig = RpcConfig{
 	MaxBatchResponseSize: 10_000_000, // 10MB
+	BatchRequestLimit:    node.DefaultConfig.BatchRequestLimit,
 }
 
-func (c *RpcConfig) Apply() {
-	rpc.MaxBatchResponseSize = c.MaxBatchResponseSize
+func (c *RpcConfig) Apply(stackConf *node.Config) {
+	stackConf.BatchResponseMaxSize = c.MaxBatchResponseSize
+	stackConf.BatchRequestLimit = c.BatchRequestLimit
 }
 
 func RpcConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Int(prefix+".max-batch-response-size", DefaultRpcConfig.MaxBatchResponseSize, "the maximum response size for a JSON-RPC request measured in bytes (-1 means no limit)")
+	f.Int(prefix+".batch-request-limit", DefaultRpcConfig.BatchRequestLimit, "the maximum number of requests in a batch")
 }

cmd/nitro/init.go

@@ -421,17 +421,16 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node
 }
 
 func pruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, nodeConfig *NodeConfig, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) error {
-	trieCachePath := cacheConfig.TrieCleanJournal
 	config := &nodeConfig.Init
 	if config.Prune == "" {
-		return pruner.RecoverPruning(stack.InstanceDir(), chainDb, trieCachePath)
+		return pruner.RecoverPruning(stack.InstanceDir(), chainDb)
 	}
 	root, err := findImportantRoots(ctx, chainDb, stack, nodeConfig, cacheConfig, l1Client, rollupAddrs)
 	if err != nil {
 		return fmt.Errorf("failed to find root to retain for pruning: %w", err)
 	}
 
-	pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), Cachedir: trieCachePath, BloomSize: config.PruneBloomSize})
+	pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: config.PruneBloomSize})
 	if err != nil {
 		return err
 	}

cmd/nitro/nitro.go

@@ -166,6 +166,7 @@ func mainImpl() int {
 	stackConf := node.DefaultConfig
 	stackConf.DataDir = nodeConfig.Persistent.Chain
 	stackConf.DBEngine = nodeConfig.Persistent.DBEngine
+	nodeConfig.Rpc.Apply(&stackConf)
 	nodeConfig.HTTP.Apply(&stackConf)
 	nodeConfig.WS.Apply(&stackConf)
 	nodeConfig.Auth.Apply(&stackConf)
@@ -779,7 +780,6 @@ func ParseNode(ctx context.Context, args []string) (*NodeConfig, *genericconf.Wa
 	if err != nil {
 		return nil, nil, nil, err
 	}
-	nodeConfig.Rpc.Apply()
 	return &nodeConfig, &l1Wallet, &l2DevWallet, nil
 }
 

execution/gethexec/blockchain.go

@@ -75,8 +75,6 @@ func DefaultCacheConfigFor(stack *node.Node, cachingConfig *CachingConfig) *core
 
 	return &core.CacheConfig{
 		TrieCleanLimit:                     cachingConfig.TrieCleanCache,
-		TrieCleanJournal:                   stack.ResolvePath(baseConf.TrieCleanCacheJournal),
-		TrieCleanRejournal:                 baseConf.TrieCleanCacheRejournal,
 		TrieCleanNoPrefetch:                baseConf.NoPrefetch,
 		TrieDirtyLimit:                     cachingConfig.TrieDirtyCache,
 		TrieDirtyDisabled:                  cachingConfig.Archive,

execution/gethexec/sequencer.go

@@ -386,8 +386,9 @@ func (s *Sequencer) PublishTransaction(parentCtx context.Context, tx *types.Tran
 			return errors.New("transaction sender is not on the whitelist")
 		}
 	}
-	if tx.Type() >= types.ArbitrumDepositTxType {
-		// Should be unreachable due to UnmarshalBinary not accepting Arbitrum internal txs
+	if tx.Type() >= types.ArbitrumDepositTxType || tx.Type() == types.BlobTxType {
+		// Should be unreachable for Arbitrum types due to UnmarshalBinary not accepting Arbitrum internal txs
+		// and we want to disallow BlobTxType since Arbitrum doesn't support EIP-4844 txs yet.
 		return types.ErrTxTypeNotSupported
 	}
 

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/gdamore/tcell/v2 v2.6.0
 	github.com/google/go-cmp v0.5.9
 	github.com/hashicorp/golang-lru/v2 v2.0.2
+	github.com/holiman/uint256 v1.2.3
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-libipfs v0.6.2
 	github.com/ipfs/interface-go-ipfs-core v0.11.0
@@ -37,8 +38,10 @@ require (
 	github.com/rivo/tview v0.0.0-20230814110005-ccc2c8119703
 	github.com/spf13/pflag v1.0.5
 	github.com/wealdtech/go-merkletree v1.0.0
+	golang.org/x/crypto v0.14.0
+	golang.org/x/sys v0.13.0
 	golang.org/x/term v0.13.0
-	golang.org/x/tools v0.7.0
+	golang.org/x/tools v0.9.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
@@ -66,6 +69,7 @@ require (
 	github.com/aws/smithy-go v1.11.2 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.7.0 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
@@ -76,10 +80,13 @@ require (
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.10.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+	github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect
 	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
@@ -88,10 +95,11 @@ require (
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 // indirect
+	github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/ethereum/c-kzg-4844 v0.3.0 // indirect
 	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
 	github.com/flynn/noise v1.0.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
@@ -120,6 +128,8 @@ require (
 	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 // indirect
+	github.com/huin/goupnp v1.1.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-block-format v0.1.1 // indirect
@@ -205,6 +215,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
@@ -237,7 +248,8 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/samber/lo v1.36.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa // indirect
+	github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b // indirect
+	github.com/urfave/cli/v2 v2.24.1 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
@@ -265,8 +277,12 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect
@@ -276,6 +292,7 @@ require (
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	lukechampine.com/blake3 v1.1.7 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
 require (
@@ -303,8 +320,6 @@ require (
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c
-	github.com/huin/goupnp v1.1.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
@@ -319,11 +334,5 @@ require (
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.13.0
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )