Merge branch 'master' into sepolia-tooling-merge-redis

.golangci.yml

@@ -16,6 +16,7 @@ linters:
   enable:
     - asciicheck  # check for non-ascii characters
     - errorlint   # enure error wrapping is safely done
+    - gci         # keep imports sorted deterministically
     - gocritic    # check for certain simplifications
     - gofmt       # ensure code is formatted
     - gosec       # check for security concerns
@@ -30,6 +31,13 @@ linters-settings:
     #
     check-type-assertions: true
 
+  gci:
+    sections:
+      - standard
+      - default
+      - prefix(github.com/ethereum/go-ethereum)
+      - prefix(github.com/offchainlabs)
+
   gocritic:
     disabled-tags:
       - experimental

Dockerfile

@@ -88,7 +88,6 @@ COPY ./safe-smart-account ./safe-smart-account
 COPY ./solgen/gen.go ./solgen/
 COPY ./fastcache ./fastcache
 COPY ./go-ethereum ./go-ethereum
-COPY ./bold ./bold
 COPY --from=brotli-wasm-export / target/
 COPY --from=contracts-builder workspace/contracts/build/contracts/src/precompiles/ contracts/build/contracts/src/precompiles/
 COPY --from=contracts-builder workspace/contracts/node_modules/@offchainlabs/upgrade-executor/build/contracts/src/UpgradeExecutor.sol/UpgradeExecutor.json contracts/

LICENSE.md

@@ -19,7 +19,7 @@ Additional Use Grant: You may use the Licensed Work in a production environment
                       validating the correctness of the posted chain state, or to deploy
                       and operate (x) a blockchain that settles to a Covered Arbitrum Chain
                       or (y) a blockchain in accordance with, and subject to, the [Arbitrum
-                      Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this
+                      Expansion Program Term of Use](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf). For purposes of this
                       Additional Use Grant, the "Covered Arbitrum Chains" are 
                       (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), 
                       Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro

README.md

@@ -47,7 +47,7 @@ Nitro is currently licensed under a [Business Source License](./LICENSE.md), sim
 
 The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova.
 
-For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
+For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
 
 ## Contact
 

arbcompress/native.go

@@ -12,6 +12,7 @@ package arbcompress
 #include "arbitrator.h"
 */
 import "C"
+
 import (
 	"errors"
 	"fmt"

arbitrator/bench/src/bin.rs

@@ -18,13 +18,13 @@ use prover::prepare::prepare_machine;
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Args {
-    /// Path to a preimages text file
+    /// Path to a preimages json file
     #[arg(short, long)]
-    preimages_path: PathBuf,
+    json_inputs: PathBuf,
 
     /// Path to a machine.wavm.br
     #[arg(short, long)]
-    machine_path: PathBuf,
+    binary: PathBuf,
 }
 
 fn main() -> eyre::Result<()> {
@@ -33,7 +33,7 @@ fn main() -> eyre::Result<()> {
 
     println!("Running benchmark with always merkleize feature on");
     for step_size in step_sizes {
-        let mut machine = prepare_machine(args.preimages_path.clone(), args.machine_path.clone())?;
+        let mut machine = prepare_machine(args.json_inputs.clone(), args.binary.clone())?;
         let _ = machine.hash();
         let mut hash_times = vec![];
         let mut step_times = vec![];

arbitrator/prover/src/lib.rs

@@ -127,6 +127,12 @@ pub unsafe extern "C" fn arbitrator_load_wavm_binary(binary_path: *const c_char)
     }
 }
 
+#[no_mangle]
+#[cfg(feature = "native")]
+pub unsafe extern "C" fn arbitrator_new_finished(gs: GlobalState) -> *mut Machine {
+    Box::into_raw(Box::new(Machine::new_finished(gs)))
+}
+
 unsafe fn cstr_to_string(c_str: *const c_char) -> String {
     CStr::from_ptr(c_str).to_string_lossy().into_owned()
 }

arbitrator/prover/src/machine.rs

@@ -1565,6 +1565,36 @@ impl Machine {
         Ok(mach)
     }
 
+    // new_finished returns a Machine in the Finished state at step 0.
+    //
+    // This allows the Mahine to be set up to model the final state of the
+    // machine at the end of the execution of a block.
+    pub fn new_finished(gs: GlobalState) -> Machine {
+        Machine {
+            steps: 0,
+            status: MachineStatus::Finished,
+            global_state: gs,
+            // The machine is in the Finished state, so nothing else really matters.
+            // values_stacks and frame_stacks cannot be empty for proof serialization,
+            // but everything else can just be entirely blank.
+            thread_state: ThreadState::Main,
+            value_stacks: vec![Vec::new()],
+            frame_stacks: vec![Vec::new()],
+            internal_stack: Default::default(),
+            modules: Default::default(),
+            modules_merkle: Default::default(),
+            pc: Default::default(),
+            stdio_output: Default::default(),
+            inbox_contents: Default::default(),
+            first_too_far: Default::default(),
+            preimage_resolver: PreimageResolverWrapper::new(Arc::new(|_, _, _| None)),
+            stylus_modules: Default::default(),
+            initial_hash: Default::default(),
+            context: Default::default(),
+            debug_info: Default::default(),
+        }
+    }
+
     pub fn new_from_wavm(wavm_binary: &Path) -> Result<Machine> {
         let mut modules: Vec<Module> = {
             let compressed = std::fs::read(wavm_binary)?;
@@ -2867,6 +2897,15 @@ impl Machine {
         let mod_merkle = self.get_modules_merkle();
         out!(mod_merkle.root());
 
+        if self.is_halted() {
+            // If the machine is halted, instead of serializing the module,
+            // serialize the global state and return.
+            // This is for the "kickstart" BoLD proof, but it's backwards compatible
+            // with the old OSP behavior which reads no further.
+            out!(self.global_state.serialize());
+            return data;
+        }
+
         // End machine serialization, serialize module
 
         let module = &self.modules[self.pc.module()];

arbitrator/prover/test-cases/dynamic.wat

@@ -12,7 +12,7 @@
 
     ;; WAVM Module hash
     (data (i32.const 0x000)
-        "\a1\49\cf\81\13\ff\9c\95\f2\c8\c2\a1\42\35\75\36\7d\e8\6d\d4\22\d8\71\14\bb\9e\a4\7b\af\53\5d\d7") ;; user
+        "\ae\87\91\cf\6a\c4\55\ff\28\06\b9\55\d5\a7\36\e8\1b\c7\91\f7\93\8a\22\a4\08\23\25\16\37\01\48\25") ;; user
     (func $start (local $user i32) (local $internals i32)
         ;; link in user.wat
         i32.const 0

arbitrator/prover/test-cases/go/main.go

@@ -73,7 +73,7 @@ const BYTES_PER_FIELD_ELEMENT = 32
 
 var BLS_MODULUS, _ = new(big.Int).SetString("52435875175126190479447740508185965837690552500527637822603658699938581184513", 10)
 
-var stylusModuleHash = common.HexToHash("a149cf8113ff9c95f2c8c2a1423575367de86dd422d87114bb9ea47baf535dd7") // user.wat
+var stylusModuleHash = common.HexToHash("ae8791cf6ac455ff2806b955d5a736e81bc791f7938a22a40823251637014825") // user.wat
 
 func callStylusProgram(recurse int) {
 	evmData := programs.EvmData{}

arbitrator/prover/test-cases/link.wat

@@ -30,7 +30,7 @@
     (data (i32.const 0x140)
         "\47\f7\4f\9c\21\51\4f\52\24\ea\d3\37\5c\bf\a9\1b\1a\5f\ef\22\a5\2a\60\30\c5\52\18\90\6b\b1\51\e5") ;; iops
     (data (i32.const 0x160)
-        "\a1\49\cf\81\13\ff\9c\95\f2\c8\c2\a1\42\35\75\36\7d\e8\6d\d4\22\d8\71\14\bb\9e\a4\7b\af\53\5d\d7") ;; user
+        "\ae\87\91\cf\6a\c4\55\ff\28\06\b9\55\d5\a7\36\e8\1b\c7\91\f7\93\8a\22\a4\08\23\25\16\37\01\48\25") ;; user
     (data (i32.const 0x180)
         "\ee\47\08\f6\47\b2\10\88\1f\89\86\e7\e3\79\6b\b2\77\43\f1\4e\ee\cf\45\4a\9b\7c\d7\c4\5b\63\b6\d7") ;; return
 

arbitrator/prover/test-cases/user.wat

@@ -22,6 +22,12 @@
         i32.const 0xFFFFFF
         i32.load
     )
+    (func $infinite_loop (result i32)
+        (loop $loop
+            br $loop
+        )
+        unreachable
+    )
     (func (export "user_entrypoint") (param $args_len i32) (result i32)
         ;; this func uses $args_len to select which func to call
 
@@ -43,6 +49,12 @@
             (then (call $out_of_bounds) (return))
         )
 
+        ;; reverts due to an out-of-gas error
+        (i32.eq (local.get $args_len) (i32.const 4))
+        (if
+            (then (call $infinite_loop) (return))
+        )
+
         (i32.eq (local.get $args_len) (i32.const 32))
         (if
             (then (call $storage_load) (return))

arbitrator/stylus/tests/hostio-test/src/main.rs

@@ -204,4 +204,37 @@ impl HostioTest {
     fn tx_origin() -> Result<Address> {
         Ok(tx::origin())
     }
+
+    fn storage_cache_bytes32() {
+        let key = B256::ZERO;
+        let val = B256::ZERO;
+        unsafe {
+            hostio::storage_cache_bytes32(key.as_ptr(), val.as_ptr());
+        }
+    }
+
+    fn pay_for_memory_grow(pages: U256) {
+        let pages: u16 = pages.try_into().unwrap();
+        unsafe {
+            hostio::pay_for_memory_grow(pages);
+        }
+    }
+
+    fn write_result_empty() {
+    }
+
+    fn write_result(size: U256) -> Result<Vec<u32>> {
+        let size: usize = size.try_into().unwrap();
+        let data = vec![0; size];
+        Ok(data)
+    }
+
+    fn read_args_no_args() {
+    }
+
+    fn read_args_one_arg(_arg1: U256) {
+    }
+
+    fn read_args_three_args(_arg1: U256, _arg2: U256, _arg3: U256) {
+    }
 }

arbitrator/wasm-libraries/user-host-trait/src/lib.rs

@@ -936,7 +936,7 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
     fn pay_for_memory_grow(&mut self, pages: u16) -> Result<(), Self::Err> {
         if pages == 0 {
             self.buy_ink(HOSTIO_INK)?;
-            return Ok(());
+            return trace!("pay_for_memory_grow", self, be!(pages), &[]);
         }
         let gas_cost = self.evm_api().add_pages(pages); // no sentry needed since the work happens after the hostio
         self.buy_gas(gas_cost)?;

arbnode/api.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/ethdb"
+
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/validator"

arbnode/batch_poster.go

@@ -203,11 +203,15 @@ func (c *BatchPosterConfig) Validate() error {
 
 type BatchPosterConfigFetcher func() *BatchPosterConfig
 
+func DangerousBatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
+	f.Bool(prefix+".allow-posting-first-batch-when-sequencer-message-count-mismatch", DefaultBatchPosterConfig.Dangerous.AllowPostingFirstBatchWhenSequencerMessageCountMismatch, "allow posting the first batch even if sequence number doesn't match chain (useful after force-inclusion)")
+}
+
 func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	f.Bool(prefix+".enable", DefaultBatchPosterConfig.Enable, "enable posting batches to l1")
 	f.Bool(prefix+".disable-dap-fallback-store-data-on-chain", DefaultBatchPosterConfig.DisableDapFallbackStoreDataOnChain, "If unable to batch to DA provider, disable fallback storing data on chain")
-	f.Int(prefix+".max-size", DefaultBatchPosterConfig.MaxSize, "maximum batch size")
-	f.Int(prefix+".max-4844-batch-size", DefaultBatchPosterConfig.Max4844BatchSize, "maximum 4844 blob enabled batch size")
+	f.Int(prefix+".max-size", DefaultBatchPosterConfig.MaxSize, "maximum estimated compressed batch size")
+	f.Int(prefix+".max-4844-batch-size", DefaultBatchPosterConfig.Max4844BatchSize, "maximum estimated compressed 4844 blob enabled batch size")
 	f.Duration(prefix+".max-delay", DefaultBatchPosterConfig.MaxDelay, "maximum batch posting delay")
 	f.Bool(prefix+".wait-for-max-delay", DefaultBatchPosterConfig.WaitForMaxDelay, "wait for the max batch delay, even if the batch is full")
 	f.Duration(prefix+".poll-interval", DefaultBatchPosterConfig.PollInterval, "how long to wait after no batches are ready to be posted before checking again")
@@ -229,6 +233,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	redislock.AddConfigOptions(prefix+".redis-lock", f)
 	dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig)
 	genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname)
+	DangerousBatchPosterConfigAddOptions(prefix+".dangerous", f)
 }
 
 var DefaultBatchPosterConfig = BatchPosterConfig{
@@ -280,7 +285,7 @@ var TestBatchPosterConfig = BatchPosterConfig{
 	DASRetentionPeriod:             daprovider.DefaultDASRetentionPeriod,
 	GasRefunderAddress:             "",
 	ExtraBatchGas:                  10_000,
-	Post4844Blobs:                  true,
+	Post4844Blobs:                  false,
 	IgnoreBlobPrice:                false,
 	DataPoster:                     dataposter.TestDataPosterConfig,
 	ParentChainWallet:              DefaultBatchPosterL1WalletConfig,

arbnode/dataposter/data_poster.go

@@ -20,10 +20,15 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
+	"github.com/holiman/uint256"
+	"github.com/redis/go-redis/v9"
+	"github.com/spf13/pflag"
+
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
+	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -34,9 +39,10 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/signer/core/apitypes"
-	"github.com/holiman/uint256"
+
 	"github.com/offchainlabs/nitro/arbnode/dataposter/dbstorage"
 	"github.com/offchainlabs/nitro/arbnode/dataposter/noop"
+	redisstorage "github.com/offchainlabs/nitro/arbnode/dataposter/redis"
 	"github.com/offchainlabs/nitro/arbnode/dataposter/slice"
 	"github.com/offchainlabs/nitro/arbnode/dataposter/storage"
 	"github.com/offchainlabs/nitro/util/arbmath"
@@ -45,10 +51,6 @@ import (
 	"github.com/offchainlabs/nitro/util/rpcclient"
 	"github.com/offchainlabs/nitro/util/signature"
 	"github.com/offchainlabs/nitro/util/stopwaiter"
-	"github.com/redis/go-redis/v9"
-	"github.com/spf13/pflag"
-
-	redisstorage "github.com/offchainlabs/nitro/arbnode/dataposter/redis"
 )
 
 var (
@@ -710,7 +712,7 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u
 	return newBaseFeeCap, newTipCap, newBlobFeeCap, nil
 }
 
-func (p *DataPoster) PostSimpleTransactionAutoNonce(ctx context.Context, to common.Address, calldata []byte, gasLimit uint64, value *big.Int) (*types.Transaction, error) {
+func (p *DataPoster) PostSimpleTransaction(ctx context.Context, to common.Address, calldata []byte, gasLimit uint64, value *big.Int) (*types.Transaction, error) {
 	p.mutex.Lock()
 	defer p.mutex.Unlock()
 	nonce, _, _, _, err := p.getNextNonceAndMaybeMeta(ctx, 1)
@@ -720,10 +722,6 @@ func (p *DataPoster) PostSimpleTransactionAutoNonce(ctx context.Context, to comm
 	return p.postTransactionWithMutex(ctx, time.Now(), nonce, nil, to, calldata, gasLimit, value, nil, nil)
 }
 
-func (p *DataPoster) PostSimpleTransaction(ctx context.Context, nonce uint64, to common.Address, calldata []byte, gasLimit uint64, value *big.Int) (*types.Transaction, error) {
-	return p.PostTransaction(ctx, time.Now(), nonce, nil, to, calldata, gasLimit, value, nil, nil)
-}
-
 func (p *DataPoster) PostTransaction(ctx context.Context, dataCreatedAt time.Time, nonce uint64, meta []byte, to common.Address, calldata []byte, gasLimit uint64, value *big.Int, kzgBlobs []kzg4844.Blob, accessList types.AccessList) (*types.Transaction, error) {
 	p.mutex.Lock()
 	defer p.mutex.Unlock()
@@ -1101,7 +1099,7 @@ func (p *DataPoster) updateBalance(ctx context.Context) error {
 	return nil
 }
 
-const maxConsecutiveIntermittentErrors = 10
+const maxConsecutiveIntermittentErrors = 20
 
 func (p *DataPoster) maybeLogError(err error, tx *storage.QueuedTransaction, msg string) {
 	nonce := tx.FullTx.Nonce()
@@ -1110,10 +1108,17 @@ func (p *DataPoster) maybeLogError(err error, tx *storage.QueuedTransaction, msg
 		return
 	}
 	logLevel := log.Error
-	if errors.Is(err, storage.ErrStorageRace) {
+	isStorageRace := errors.Is(err, storage.ErrStorageRace)
+	if isStorageRace || strings.Contains(err.Error(), txpool.ErrFutureReplacePending.Error()) {
 		p.errorCount[nonce]++
 		if p.errorCount[nonce] <= maxConsecutiveIntermittentErrors {
-			logLevel = log.Debug
+			if isStorageRace {
+				logLevel = log.Debug
+			} else {
+				logLevel = log.Info
+			}
+		} else if isStorageRace {
+			logLevel = log.Warn
 		}
 	} else {
 		delete(p.errorCount, nonce)

arbnode/dataposter/dataposter_test.go

@@ -9,6 +9,9 @@ import (
 	"time"
 
 	"github.com/Knetic/govaluate"
+	"github.com/google/go-cmp/cmp"
+	"github.com/holiman/uint256"
+
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -17,8 +20,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/google/go-cmp/cmp"
-	"github.com/holiman/uint256"
+
 	"github.com/offchainlabs/nitro/arbnode/dataposter/externalsignertest"
 	"github.com/offchainlabs/nitro/util/arbmath"
 )