merge master and resolve conflicts

arbnode/batch_poster.go

@@ -216,9 +216,8 @@ var DefaultBatchPosterConfig = BatchPosterConfig{
 	Enable:                             false,
 	DisableDapFallbackStoreDataOnChain: false,
 	// This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go
-	MaxSize: 100000,
-	// TODO: is 1000 bytes an appropriate margin for error vs blob space efficiency?
-	Max4844BatchSize:               blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
+	MaxSize:                        100000,
+	Max4844BatchSize:               blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 2000,
 	PollInterval:                   time.Second * 10,
 	ErrorDelay:                     time.Second * 10,
 	MaxDelay:                       time.Hour,
@@ -1270,7 +1269,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		return false, err
 	}
 	if len(kzgBlobs)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock {
-		return false, fmt.Errorf("produced %v blobs for batch but a block can only hold %v", len(kzgBlobs), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
+		return false, fmt.Errorf("produced %v blobs for batch but a block can only hold %v (compressed batch was %v bytes long)", len(kzgBlobs), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob, len(sequencerMsg))
 	}
 	accessList := b.accessList(int(batchPosition.NextSeqNum), int(b.building.segments.delayedMsg))
 	// On restart, we may be trying to estimate gas for a batch whose successor has

arbnode/dataposter/data_poster.go

@@ -832,6 +832,37 @@ func (p *DataPoster) sendTx(ctx context.Context, prevTx *storage.QueuedTransacti
 	if err := p.saveTx(ctx, prevTx, newTx); err != nil {
 		return err
 	}
+
+	// The following check is to avoid sending transactions of a different type (eg DynamicFeeTxType vs BlobTxType)
+	// to the previous tx if the previous tx is not yet included in a reorg resistant block, in order to avoid issues
+	// where eventual consistency of parent chain mempools causes a tx with higher nonce blocking a tx of a
+	// different type with a lower nonce.
+	// If we decide not to send this tx yet, just leave it queued and with Sent set to false.
+	// The resending/repricing loop in DataPoster.Start will keep trying.
+	if !newTx.Sent && newTx.FullTx.Nonce() > 0 {
+		precedingTx, err := p.queue.Get(ctx, arbmath.SaturatingUSub(newTx.FullTx.Nonce(), 1))
+		if err != nil {
+			return fmt.Errorf("couldn't get preceding tx in DataPoster to check if should send tx with nonce %d: %w", newTx.FullTx.Nonce(), err)
+		}
+		if precedingTx != nil && // precedingTx == nil -> the actual preceding tx was already confirmed
+			precedingTx.FullTx.Type() != newTx.FullTx.Type() {
+			latestBlockNumber, err := p.client.BlockNumber(ctx)
+			if err != nil {
+				return fmt.Errorf("couldn't get block number in DataPoster to check if should send tx with nonce %d: %w", newTx.FullTx.Nonce(), err)
+			}
+			prevBlockNumber := arbmath.SaturatingUSub(latestBlockNumber, 1)
+			reorgResistantNonce, err := p.client.NonceAt(ctx, p.Sender(), new(big.Int).SetUint64(prevBlockNumber))
+			if err != nil {
+				return fmt.Errorf("couldn't determine reorg resistant nonce in DataPoster to check if should send tx with nonce %d: %w", newTx.FullTx.Nonce(), err)
+			}
+
+			if precedingTx.FullTx.Nonce() > reorgResistantNonce {
+				log.Info("DataPoster is holding off on sending a transaction of different type to the previous transaction until the previous transaction has been included in a reorg resistant block (it remains queued and will be retried)", "nonce", newTx.FullTx.Nonce(), "prevType", precedingTx.FullTx.Type(), "type", newTx.FullTx.Type())
+				return nil
+			}
+		}
+	}
+
 	if err := p.client.SendTransaction(ctx, newTx.FullTx); err != nil {
 		if !rpcclient.IsAlreadyKnownError(err) && !strings.Contains(err.Error(), "nonce too low") {
 			log.Warn("DataPoster failed to send transaction", "err", err, "nonce", newTx.FullTx.Nonce(), "feeCap", newTx.FullTx.GasFeeCap(), "tipCap", newTx.FullTx.GasTipCap(), "blobFeeCap", newTx.FullTx.BlobGasFeeCap(), "gas", newTx.FullTx.Gas())
@@ -1072,19 +1103,21 @@ func (p *DataPoster) Start(ctxIn context.Context) {
 			latestNonce = latestQueued.FullTx.Nonce()
 		}
 		for _, tx := range queueContents {
-			replacing := false
+			previouslyUnsent := !tx.Sent
+			sendAttempted := false
 			if now.After(tx.NextReplacement) {
-				replacing = true
 				nonceBacklog := arbmath.SaturatingUSub(latestNonce, tx.FullTx.Nonce())
 				weightBacklog := arbmath.SaturatingUSub(latestCumulativeWeight, tx.CumulativeWeight())
 				err := p.replaceTx(ctx, tx, arbmath.MaxInt(nonceBacklog, weightBacklog))
+				sendAttempted = true
 				p.maybeLogError(err, tx, "failed to replace-by-fee transaction")
 			}
 			if nextCheck.After(tx.NextReplacement) {
 				nextCheck = tx.NextReplacement
 			}
-			if !replacing && !tx.Sent {
+			if !sendAttempted && previouslyUnsent {
 				err := p.sendTx(ctx, tx, tx)
+				sendAttempted = true
 				p.maybeLogError(err, tx, "failed to re-send transaction")
 				if err != nil {
 					nextSend := time.Now().Add(time.Minute)
@@ -1093,6 +1126,12 @@ func (p *DataPoster) Start(ctxIn context.Context) {
 					}
 				}
 			}
+			if previouslyUnsent && sendAttempted {
+				// Don't try to send more than 1 unsent transaction, to play nicely with parent chain mempools.
+				// Transactions will be unsent if there was some error when originally sending them,
+				// or if transaction type changes and the prior tx is not yet reorg resistant.
+				break
+			}
 		}
 		wait := time.Until(nextCheck)
 		if wait < minWait {

arbnode/node.go

@@ -551,7 +551,7 @@ func createNodeImpl(
 	txStreamer.SetInboxReaders(inboxReader, delayedBridge)
 
 	var statelessBlockValidator *staker.StatelessBlockValidator
-	if config.BlockValidator.ValidationServerConfigs[0].URL != "" {
+	if config.BlockValidator.RedisValidationClientConfig.Enabled() || config.BlockValidator.ValidationServerConfigs[0].URL != "" {
 		statelessBlockValidator, err = staker.NewStatelessBlockValidator(
 			inboxReader,
 			inboxTracker,

arbos/arbosState/arbosstate.go

@@ -36,24 +36,26 @@ import (
 // persisted beyond the end of the test.)
 
 type ArbosState struct {
-	arbosVersion           uint64                      // version of the ArbOS storage format and semantics
-	upgradeVersion         storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade
-	upgradeTimestamp       storage.StorageBackedUint64 // when to do the planned upgrade
-	networkFeeAccount      storage.StorageBackedAddress
-	l1PricingState         *l1pricing.L1PricingState
-	l2PricingState         *l2pricing.L2PricingState
-	retryableState         *retryables.RetryableState
-	addressTable           *addressTable.AddressTable
-	chainOwners            *addressSet.AddressSet
-	sendMerkle             *merkleAccumulator.MerkleAccumulator
-	blockhashes            *blockhash.Blockhashes
-	chainId                storage.StorageBackedBigInt
-	chainConfig            storage.StorageBackedBytes
-	genesisBlockNum        storage.StorageBackedUint64
-	infraFeeAccount        storage.StorageBackedAddress
-	brotliCompressionLevel storage.StorageBackedUint64 // brotli compression level used for pricing
-	backingStorage         *storage.Storage
-	Burner                 burn.Burner
+	arbosVersion                  uint64                      // version of the ArbOS storage format and semantics
+	maxArbosVersionSupported      uint64                      // maximum ArbOS version supported by this code
+	maxDebugArbosVersionSupported uint64                      // maximum ArbOS version supported by this code in debug mode
+	upgradeVersion                storage.StorageBackedUint64 // version we're planning to upgrade to, or 0 if not planning to upgrade
+	upgradeTimestamp              storage.StorageBackedUint64 // when to do the planned upgrade
+	networkFeeAccount             storage.StorageBackedAddress
+	l1PricingState                *l1pricing.L1PricingState
+	l2PricingState                *l2pricing.L2PricingState
+	retryableState                *retryables.RetryableState
+	addressTable                  *addressTable.AddressTable
+	chainOwners                   *addressSet.AddressSet
+	sendMerkle                    *merkleAccumulator.MerkleAccumulator
+	blockhashes                   *blockhash.Blockhashes
+	chainId                       storage.StorageBackedBigInt
+	chainConfig                   storage.StorageBackedBytes
+	genesisBlockNum               storage.StorageBackedUint64
+	infraFeeAccount               storage.StorageBackedAddress
+	brotliCompressionLevel        storage.StorageBackedUint64 // brotli compression level used for pricing
+	backingStorage                *storage.Storage
+	Burner                        burn.Burner
 }
 
 var ErrUninitializedArbOS = errors.New("ArbOS uninitialized")
@@ -70,6 +72,8 @@ func OpenArbosState(stateDB vm.StateDB, burner burn.Burner) (*ArbosState, error)
 	}
 	return &ArbosState{
 		arbosVersion,
+		20,
+		20,
 		backingStorage.OpenStorageBackedUint64(uint64(upgradeVersionOffset)),
 		backingStorage.OpenStorageBackedUint64(uint64(upgradeTimestampOffset)),
 		backingStorage.OpenStorageBackedAddress(uint64(networkFeeAccountOffset)),
@@ -400,6 +404,14 @@ func (state *ArbosState) RetryableState() *retryables.RetryableState {
 	return state.retryableState
 }
 
+func (state *ArbosState) MaxArbosVersionSupported() uint64 {
+	return state.maxArbosVersionSupported
+}
+
+func (state *ArbosState) MaxDebugArbosVersionSupported() uint64 {
+	return state.maxDebugArbosVersionSupported
+}
+
 func (state *ArbosState) L1PricingState() *l1pricing.L1PricingState {
 	return state.l1PricingState
 }

cmd/nitro/init.go

@@ -155,19 +155,30 @@ func validateBlockChain(blockChain *core.BlockChain, chainConfig *params.ChainCo
 			return fmt.Errorf("invalid chain config, not compatible with previous: %w", err)
 		}
 	}
+	// Make sure we don't allow accidentally downgrading ArbOS
+	if chainConfig.DebugMode() {
+		if currentArbosState.ArbOSVersion() > currentArbosState.MaxDebugArbosVersionSupported() {
+			return fmt.Errorf("attempted to launch node in debug mode with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxDebugArbosVersionSupported(), currentArbosState.ArbOSVersion())
+		}
+	} else {
+		if currentArbosState.ArbOSVersion() > currentArbosState.MaxArbosVersionSupported() {
+			return fmt.Errorf("attempted to launch node with ArbOS version %v on ArbOS state with version %v", currentArbosState.MaxArbosVersionSupported(), currentArbosState.ArbOSVersion())
+		}
+
+	}
 
 	return nil
 }
 
 func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) {
 	if !config.Init.Force {
-		if readOnlyDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", 0, 0, "", "", true); err == nil {
+		if readOnlyDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", 0, 0, "", "l2chaindata/", true); err == nil {
 			if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil {
 				readOnlyDb.Close()
 				if !arbmath.BigEquals(chainConfig.ChainID, chainId) {
 					return nil, nil, fmt.Errorf("database has chain ID %v but config has chain ID %v (are you sure this database is for the right chain?)", chainConfig.ChainID, chainId)
 				}
-				chainDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", config.Execution.Caching.DatabaseCache, config.Persistent.Handles, config.Persistent.Ancient, "", false)
+				chainDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", config.Execution.Caching.DatabaseCache, config.Persistent.Handles, config.Persistent.Ancient, "l2chaindata/", false)
 				if err != nil {
 					return chainDb, nil, err
 				}
@@ -219,7 +230,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo
 
 	var initDataReader statetransfer.InitDataReader = nil
 
-	chainDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", config.Execution.Caching.DatabaseCache, config.Persistent.Handles, config.Persistent.Ancient, "", false)
+	chainDb, err := stack.OpenDatabaseWithFreezer("l2chaindata", config.Execution.Caching.DatabaseCache, config.Persistent.Handles, config.Persistent.Ancient, "l2chaindata/", false)
 	if err != nil {
 		return chainDb, nil, err
 	}

cmd/nitro/nitro.go

@@ -487,7 +487,7 @@ func mainImpl() int {
 		return 1
 	}
 
-	arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "", false)
+	arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "arbitrumdata/", false)
 	deferFuncs = append(deferFuncs, func() { closeDb(arbDb, "arbDb") })
 	if err != nil {
 		log.Error("failed to open database", "err", err)

cmd/pruning/pruning.go

@@ -85,7 +85,7 @@ func findImportantRoots(ctx context.Context, chainDb ethdb.Database, stack *node
 	if chainConfig == nil {
 		return nil, errors.New("database doesn't have a chain config (was this node initialized?)")
 	}
-	arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "", true)
+	arbDb, err := stack.OpenDatabase("arbitrumdata", 0, 0, "arbitrumdata/", true)
 	if err != nil {
 		return nil, err
 	}

execution/gethexec/node.go

@@ -216,7 +216,7 @@ func CreateExecutionNode(
 	var classicOutbox *ClassicOutboxRetriever
 
 	if l2BlockChain.Config().ArbitrumChainParams.GenesisBlockNum > 0 {
-		classicMsgDb, err := stack.OpenDatabase("classic-msg", 0, 0, "", true)
+		classicMsgDb, err := stack.OpenDatabase("classic-msg", 0, 0, "classicmsg/", true)
 		if err != nil {
 			log.Warn("Classic Msg Database not found", "err", err)
 			classicOutbox = nil

execution/gethexec/sync_monitor.go

@@ -38,16 +38,23 @@ func NewSyncMonitor(config *SyncMonitorConfig, exec *ExecutionEngine) *SyncMonit
 
 func (s *SyncMonitor) FullSyncProgressMap() map[string]interface{} {
 	res := s.consensus.FullSyncProgressMap()
-	consensusSyncTarget := s.consensus.SyncTargetMessageCount()
 
-	built, err := s.exec.HeadMessageNumber()
+	res["consensusSyncTarget"] = s.consensus.SyncTargetMessageCount()
+
+	header, err := s.exec.getCurrentHeader()
 	if err != nil {
-		res["headMsgNumberError"] = err
+		res["currentHeaderError"] = err
+	} else {
+		blockNum := header.Number.Uint64()
+		res["blockNum"] = blockNum
+		messageNum, err := s.exec.BlockNumberToMessageIndex(blockNum)
+		if err != nil {
+			res["messageOfLastBlockError"] = err
+		} else {
+			res["messageOfLastBlock"] = messageNum
+		}
 	}
 
-	res["builtBlock"] = built
-	res["consensusSyncTarget"] = consensusSyncTarget
-
 	return res
 }
 

gethhook/geth-hook.go

@@ -58,9 +58,7 @@ func init() {
 	precompileErrors := make(map[[4]byte]abi.Error)
 	for addr, precompile := range precompiles.Precompiles() {
 		for _, errABI := range precompile.Precompile().GetErrorABIs() {
-			var id [4]byte
-			copy(id[:], errABI.ID[:4])
-			precompileErrors[id] = errABI
+			precompileErrors[[4]byte(errABI.ID.Bytes())] = errABI
 		}
 		var wrapped vm.AdvancedPrecompile = ArbosPrecompileWrapper{precompile}
 		vm.PrecompiledContractsArbitrum[addr] = wrapped