Merge branch 'master' into merge-wasmer-4.3.7

.dockerignore

@@ -36,6 +36,7 @@ arbitrator/tools/wasmer/target/
 arbitrator/tools/wasm-tools/
 arbitrator/tools/pricers/
 arbitrator/tools/module_roots/
+arbitrator/tools/stylus_benchmark
 arbitrator/langs/rust/target/
 arbitrator/langs/bf/target/
 

.github/buildspec.yml

@@ -0,0 +1,36 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - git submodule update --init
+      - echo Logging in to Dockerhub....
+      - docker login --username $DOCKERHUB_USERNAME --password $DOCKERHUB_PASSWORD
+      - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $REPOSITORY_URI
+      - COMMIT_HASH=$(git rev-parse --short=7 HEAD || echo "latest")
+      - VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' |grep -v '^grafted\|HEAD\|master\|main$' || echo "dev")
+      - NITRO_VERSION=${VERSION_TAG}-${COMMIT_HASH}
+      - IMAGE_TAG=${NITRO_VERSION}
+      - NITRO_DATETIME=$(git show -s --date=iso-strict --format=%cd)
+      - NITRO_MODIFIED="false"
+      - echo ${NITRO_VERSION} > ./.nitro-tag.txt
+  build:
+    commands:
+      - echo Build started on `date` 
+      - echo Building the Docker image ${NITRO_VERSION}...
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-slim --target nitro-node-slim --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node --target nitro-node --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-dev --target nitro-node-dev --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - DOCKER_BUILDKIT=1 docker build . -t nitro-node-validator --target nitro-node-validator --build-arg version=$NITRO_VERSION --build-arg datetime=$NITRO_DATETIME --build-arg modified=$NITRO_MODIFIED
+      - docker tag nitro-node:latest $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG
+      - docker tag nitro-node-slim:latest $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG
+      - docker tag nitro-node-dev:latest $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG
+      - docker tag nitro-node-validator:latest $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG
+  post_build: 
+    commands:
+      - echo Build completed on `date` 
+      - echo pushing to repo
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-slim-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-dev-$ARCH_TAG
+      - docker push $REPOSITORY_URI:$IMAGE_TAG-validator-$ARCH_TAG

.github/workflows/arbitrator-ci.yml

@@ -171,6 +171,9 @@ jobs:
       - name: Rustfmt - langs/rust
         run: cargo fmt --all --manifest-path arbitrator/langs/rust/Cargo.toml -- --check
 
+      - name: Rustfmt - tools/stylus_benchmark
+        run: cargo fmt --all --manifest-path arbitrator/tools/stylus_benchmark/Cargo.toml -- --check
+
       - name: Make proofs from test cases
         run: make -j test-gen-proofs
 

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       redis:
         image: redis
         ports:
-            - 6379:6379
+          - 6379:6379
 
     strategy:
       fail-fast: false
@@ -192,7 +192,7 @@ jobs:
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'
-        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover
+        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --timeout 60m --cover
 
       - name: run stylus tests
         if: matrix.test-mode == 'stylus'

.github/workflows/submodule-pin-check.yml

@@ -25,9 +25,9 @@ jobs:
         run: |
           status_state="pending"
           declare -Ar exceptions=(
-            [contracts]=origin/develop
+            [contracts]=origin/pre-bold
             [nitro-testnode]=origin/master
-            
+
             #TODO Rachel to check these are the intended branches.
             [arbitrator/langs/c]=origin/vm-storage-cache
             [arbitrator/tools/wasmer]=origin/adopt-v4.2.8
@@ -38,7 +38,7 @@ jobs:
             if [[ -v exceptions[$mod] ]]; then
               branch=${exceptions[$mod]}
             fi
-            
+
             if ! git -C $mod merge-base --is-ancestor HEAD $branch; then
               echo $mod diverges from $branch
               divergent=1

.gitmodules

@@ -23,6 +23,9 @@
 [submodule "nitro-testnode"]
 	path = nitro-testnode
 	url = https://github.com/OffchainLabs/nitro-testnode.git
+[submodule "bold"]
+	path = bold
+	url = https://github.com/OffchainLabs/bold.git
 [submodule "arbitrator/langs/rust"]
 	path = arbitrator/langs/rust
 	url = https://github.com/OffchainLabs/stylus-sdk-rs.git

.golangci.yml

@@ -16,6 +16,7 @@ linters:
   enable:
     - asciicheck  # check for non-ascii characters
     - errorlint   # enure error wrapping is safely done
+    - gci         # keep imports sorted deterministically
     - gocritic    # check for certain simplifications
     - gofmt       # ensure code is formatted
     - gosec       # check for security concerns
@@ -30,6 +31,13 @@ linters-settings:
     #
     check-type-assertions: true
 
+  gci:
+    sections:
+      - standard
+      - default
+      - prefix(github.com/ethereum/go-ethereum)
+      - prefix(github.com/offchainlabs)
+
   gocritic:
     disabled-tags:
       - experimental

Dockerfile

@@ -234,6 +234,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 COPY go.mod go.sum ./
 COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
 COPY fastcache/go.mod fastcache/go.sum fastcache/
+COPY bold/go.mod bold/go.sum bold/
 RUN go mod download
 COPY . ./
 COPY --from=contracts-builder workspace/contracts/build/ contracts/build/

LICENSE.md

@@ -19,7 +19,7 @@ Additional Use Grant: You may use the Licensed Work in a production environment
                       validating the correctness of the posted chain state, or to deploy
                       and operate (x) a blockchain that settles to a Covered Arbitrum Chain
                       or (y) a blockchain in accordance with, and subject to, the [Arbitrum
-                      Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this
+                      Expansion Program Term of Use](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf). For purposes of this
                       Additional Use Grant, the "Covered Arbitrum Chains" are 
                       (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), 
                       Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro

Makefile

@@ -155,6 +155,8 @@ stylus_test_hostio-test_src       = $(call get_stylus_test_rust,hostio-test)
 stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm)
 stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms)
 
+CBROTLI_WASM_BUILD_ARGS ?=-d
+
 # user targets
 
 .PHONY: push
@@ -579,9 +581,9 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(prover_bin)
 	@touch $@
 
 .make/cbrotli-wasm: $(DEP_PREDICATE) $(ORDER_ONLY_PREDICATE) .make
-	test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w -d
-	test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w -d
-	test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w -d
+	test -f target/lib-wasm/libbrotlicommon-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlienc-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
+	test -f target/lib-wasm/libbrotlidec-static.a || ./scripts/build-brotli.sh -w $(CBROTLI_WASM_BUILD_ARGS)
 	@touch $@
 
 .make/wasm-lib: $(DEP_PREDICATE) arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/softfloat.a  $(ORDER_ONLY_PREDICATE) .make

README.md

@@ -47,7 +47,7 @@ Nitro is currently licensed under a [Business Source License](./LICENSE.md), sim
 
 The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova.
 
-For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
+For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/aep/ArbitrumExpansionProgramTerms.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
 
 ## Contact
 

arbcompress/native.go

@@ -12,6 +12,7 @@ package arbcompress
 #include "arbitrator.h"
 */
 import "C"
+
 import (
 	"errors"
 	"fmt"

arbitrator/Cargo.toml

@@ -12,6 +12,7 @@ members = [
 exclude = [
         "stylus/tests/",
         "tools/wasmer/",
+        "tools/stylus_benchmark",
 ]
 resolver = "2"
 

arbitrator/arbutil/src/benchmark.rs

@@ -0,0 +1,14 @@
+// Copyright 2024, Offchain Labs, Inc.
+// For license information, see https://github.com/nitro/blob/master/LICENSE
+
+use crate::evm::api::Ink;
+use std::time::{Duration, Instant};
+
+// Benchmark is used to track the performance of blocks of code in stylus
+#[derive(Clone, Copy, Debug, Default)]
+pub struct Benchmark {
+    pub timer: Option<Instant>,
+    pub elapsed_total: Duration,
+    pub ink_start: Option<Ink>,
+    pub ink_total: Ink,
+}

arbitrator/arbutil/src/lib.rs

@@ -1,6 +1,7 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
 
+pub mod benchmark;
 /// cbindgen:ignore
 pub mod color;
 pub mod crypto;

arbitrator/bench/src/bin.rs

@@ -18,13 +18,13 @@ use prover::prepare::prepare_machine;
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Args {
-    /// Path to a preimages text file
+    /// Path to a preimages json file
     #[arg(short, long)]
-    preimages_path: PathBuf,
+    json_inputs: PathBuf,
 
     /// Path to a machine.wavm.br
     #[arg(short, long)]
-    machine_path: PathBuf,
+    binary: PathBuf,
 }
 
 fn main() -> eyre::Result<()> {
@@ -33,7 +33,7 @@ fn main() -> eyre::Result<()> {
 
     println!("Running benchmark with always merkleize feature on");
     for step_size in step_sizes {
-        let mut machine = prepare_machine(args.preimages_path.clone(), args.machine_path.clone())?;
+        let mut machine = prepare_machine(args.json_inputs.clone(), args.binary.clone())?;
         let _ = machine.hash();
         let mut hash_times = vec![];
         let mut step_times = vec![];

arbitrator/jit/src/lib.rs

@@ -0,0 +1,51 @@
+// Copyright 2021-2024, Offchain Labs, Inc.
+// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
+
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+mod arbcompress;
+mod caller_env;
+pub mod machine;
+mod prepare;
+pub mod program;
+mod socket;
+pub mod stylus_backend;
+mod test;
+mod wasip1_stub;
+mod wavmio;
+
+#[derive(StructOpt)]
+#[structopt(name = "jit-prover")]
+pub struct Opts {
+    #[structopt(short, long)]
+    binary: PathBuf,
+    #[structopt(long, default_value = "0")]
+    inbox_position: u64,
+    #[structopt(long, default_value = "0")]
+    delayed_inbox_position: u64,
+    #[structopt(long, default_value = "0")]
+    position_within_message: u64,
+    #[structopt(long)]
+    last_block_hash: Option<String>,
+    #[structopt(long)]
+    last_send_root: Option<String>,
+    #[structopt(long)]
+    inbox: Vec<PathBuf>,
+    #[structopt(long)]
+    delayed_inbox: Vec<PathBuf>,
+    #[structopt(long)]
+    preimages: Option<PathBuf>,
+    #[structopt(long)]
+    cranelift: bool,
+    #[structopt(long)]
+    forks: bool,
+    #[structopt(long)]
+    pub debug: bool,
+    #[structopt(long)]
+    pub require_success: bool,
+    // JSON inputs supercede any of the command-line inputs which could
+    // be specified in the JSON file.
+    #[structopt(long)]
+    json_inputs: Option<PathBuf>,
+}

arbitrator/jit/src/main.rs

@@ -1,58 +1,13 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/nitro/blob/master/LICENSE
 
-use crate::machine::{Escape, WasmEnv};
 use arbutil::{color, Color};
 use eyre::Result;
-use std::path::PathBuf;
+use jit::machine;
+use jit::machine::{Escape, WasmEnv};
+use jit::Opts;
 use structopt::StructOpt;
 
-mod arbcompress;
-mod caller_env;
-mod machine;
-mod prepare;
-mod program;
-mod socket;
-mod stylus_backend;
-mod test;
-mod wasip1_stub;
-mod wavmio;
-
-#[derive(StructOpt)]
-#[structopt(name = "jit-prover")]
-pub struct Opts {
-    #[structopt(short, long)]
-    binary: PathBuf,
-    #[structopt(long, default_value = "0")]
-    inbox_position: u64,
-    #[structopt(long, default_value = "0")]
-    delayed_inbox_position: u64,
-    #[structopt(long, default_value = "0")]
-    position_within_message: u64,
-    #[structopt(long)]
-    last_block_hash: Option<String>,
-    #[structopt(long)]
-    last_send_root: Option<String>,
-    #[structopt(long)]
-    inbox: Vec<PathBuf>,
-    #[structopt(long)]
-    delayed_inbox: Vec<PathBuf>,
-    #[structopt(long)]
-    preimages: Option<PathBuf>,
-    #[structopt(long)]
-    cranelift: bool,
-    #[structopt(long)]
-    forks: bool,
-    #[structopt(long)]
-    debug: bool,
-    #[structopt(long)]
-    require_success: bool,
-    // JSON inputs supercede any of the command-line inputs which could
-    // be specified in the JSON file.
-    #[structopt(long)]
-    json_inputs: Option<PathBuf>,
-}
-
 fn main() -> Result<()> {
     let opts = Opts::from_args();
     let env = match WasmEnv::cli(&opts) {

arbitrator/jit/src/prepare.rs

@@ -1,7 +1,7 @@
 // Copyright 2022-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/nitro/blob/master/LICENSE
 
-use crate::WasmEnv;
+use crate::machine::WasmEnv;
 use arbutil::{Bytes32, PreimageType};
 use eyre::Ok;
 use prover::parse_input::FileData;