Merge branch 'master' into sepolia-tooling-merge

Dockerfile

@@ -41,7 +41,8 @@ RUN apt-get update && apt-get install -y curl build-essential=12.9
 
 FROM wasm-base as wasm-libs-builder
 	# clang / lld used by soft-float wasm
-RUN apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1 wabt
+RUN apt-get update && \
+    apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1 wabt
     # pinned rust 1.75.0
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.75.0 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
 COPY ./Makefile ./
@@ -204,6 +205,7 @@ COPY ./scripts/download-machine.sh .
 #RUN ./download-machine.sh consensus-v11 0xf4389b835497a910d7ba3ebfb77aa93da985634f3c052de1290360635be40c4a
 #RUN ./download-machine.sh consensus-v11.1 0x68e4fe5023f792d4ef584796c84d710303a5e12ea02d6e37e2b5e9c4332507c4
 #RUN ./download-machine.sh consensus-v20 0x8b104a2e80ac6165dc58b9048de12f301d70b02a0ab51396c22b4b4b802a16a4
+RUN ./download-machine.sh consensus-v30-rc.2 0xb0de9cb89e4d944ae6023a3b62276e54804c242fd8c4c2d8e6cc4450f5fa8b1b
 
 FROM golang:1.21-bookworm as node-builder
 WORKDIR /workspace
@@ -270,11 +272,15 @@ USER user
 WORKDIR /home/user/
 ENTRYPOINT [ "/usr/local/bin/nitro" ]
 
+FROM offchainlabs/nitro-node:v2.3.4-rc.5-b4cc111 as nitro-legacy
+
 FROM nitro-node-slim as nitro-node
 USER root
 COPY --from=prover-export /bin/jit                        /usr/local/bin/
 COPY --from=node-builder  /workspace/target/bin/daserver  /usr/local/bin/
 COPY --from=node-builder  /workspace/target/bin/datool    /usr/local/bin/
+COPY --from=nitro-legacy /home/user/target/machines /home/user/nitro-legacy/machines
+RUN rm -rf /workspace/target/legacy-machines/latest
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y \
@@ -284,10 +290,23 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/ldconfig/aux-cache /usr/lib/python3.9/__pycache__/ /usr/lib/python3.9/*/__pycache__/ /var/log/* && \
     nitro --version
+ENTRYPOINT [ "/usr/local/bin/nitro" , "--validation.wasm.allowed-wasm-module-roots", "/home/user/nitro-legacy/machines,/home/user/target/machines"]
+
+USER user
 
+FROM nitro-node as nitro-node-validator
+USER root
+COPY --from=nitro-legacy /usr/local/bin/nitro-val /home/user/nitro-legacy/bin/nitro-val
+COPY --from=nitro-legacy /usr/local/bin/jit /home/user/nitro-legacy/bin/jit
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y xxd netcat-traditional && \
+    rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/ldconfig/aux-cache /usr/lib/python3.9/__pycache__/ /usr/lib/python3.9/*/__pycache__/ /var/log/*
+COPY scripts/split-val-entry.sh /usr/local/bin
+ENTRYPOINT [ "/usr/local/bin/split-val-entry.sh" ]
 USER user
 
-FROM nitro-node as nitro-node-dev-base
+FROM nitro-node-validator as nitro-node-dev
 USER root
 # Copy in latest WASM module root
 RUN rm -f /home/user/target/machines/latest
@@ -312,22 +331,5 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
 
 USER user
 
-FROM offchainlabs/nitro-node:v2.3.4-rc.5-b4cc111 as nitro-legacy
-
-FROM nitro-node-dev-base as nitro-node-dev
-USER root
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update && \
-    apt-get install -y xxd netcat-traditional && \
-    rm -rf /var/lib/apt/lists/* /usr/share/doc/* /var/cache/ldconfig/aux-cache /usr/lib/python3.9/__pycache__/ /usr/lib/python3.9/*/__pycache__/ /var/log/*
-COPY scripts/split-val-entry.sh /usr/local/bin
-COPY --from=nitro-legacy /home/user/target/machines /home/user/nitro-legacy/machines
-RUN rm -rf /workspace/target/legacy-machines/latest
-COPY --from=nitro-legacy /usr/local/bin/nitro-val /home/user/nitro-legacy/bin/nitro-val
-COPY --from=nitro-legacy /usr/local/bin/jit /home/user/nitro-legacy/bin/jit
-ENTRYPOINT [ "/usr/local/bin/split-val-entry.sh" ]
-USER user
-
 FROM nitro-node as nitro-node-default
 # Just to ensure nitro-node-dist is default

Makefile

@@ -292,7 +292,7 @@ $(arbitrator_jit): $(DEP_PREDICATE) $(jit_files)
 $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm: $(arbitrator_cases)/rust/src/bin/%.rs $(arbitrator_cases)/rust/src/lib.rs
 	cargo build --manifest-path $(arbitrator_cases)/rust/Cargo.toml --release --target wasm32-wasi --bin $(patsubst $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm,%, $@)
 
-$(arbitrator_cases)/go/testcase.wasm: $(arbitrator_cases)/go/*.go
+$(arbitrator_cases)/go/testcase.wasm: $(arbitrator_cases)/go/*.go .make/solgen
 	cd $(arbitrator_cases)/go && GOOS=wasip1 GOARCH=wasm go build -o testcase.wasm
 
 $(arbitrator_generated_header): $(DEP_PREDICATE) $(stylus_files)
@@ -442,8 +442,12 @@ target/testdata/preimages.bin:
 contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin
 	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin
 
-contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps)
-	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -i 50000000 --require-success --preimages target/testdata/preimages.bin
+contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm
+	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin  --stylus-modules $(arbitrator_cases)/user.wasm
+
+# avoid testing user.wasm in onestepproofs. It can only run as stylus program.
+contracts/test/prover/proofs/user.json:
+	echo "[]" > $@
 
 # avoid testing read-inboxmsg-10 in onestepproofs. It's used for go challenge testing.
 contracts/test/prover/proofs/read-inboxmsg-10.json:

arbitrator/prover/src/main.rs

@@ -263,7 +263,10 @@ fn main() -> Result<()> {
 
         if opts.proving_backoff {
             let mut extra_data = 0;
-            if matches!(next_opcode, Opcode::ReadInboxMessage | Opcode::ReadPreImage) {
+            if matches!(
+                next_opcode,
+                Opcode::ReadInboxMessage | Opcode::ReadPreImage | Opcode::SwitchThread
+            ) {
                 extra_data = next_inst.argument_data;
             }
             let count_entry = proving_backoff

arbitrator/prover/test-cases/dynamic.wat

@@ -12,8 +12,7 @@
 
     ;; WAVM Module hash
     (data (i32.const 0x000)
-        "\87\12\6b\19\8a\ce\0c\ba\00\6a\ab\9b\b7\45\bb\0a\ac\48\4d\6b\b8\b5\f9\03\a2\99\8f\64\00\9f\e2\04") ;; user
-
+        "\a1\49\cf\81\13\ff\9c\95\f2\c8\c2\a1\42\35\75\36\7d\e8\6d\d4\22\d8\71\14\bb\9e\a4\7b\af\53\5d\d7") ;; user
     (func $start (local $user i32) (local $internals i32)
         ;; link in user.wat
         i32.const 0

arbitrator/prover/test-cases/go/main.go

@@ -1,6 +1,9 @@
 // Copyright 2021-2024, Offchain Labs, Inc.
 // For license information, see https://github.com/nitro/blob/master/LICENSE
 
+//go:build wasm
+// +build wasm
+
 package main
 
 import (
@@ -19,6 +22,7 @@ import (
 	merkletree "github.com/wealdtech/go-merkletree"
 
 	"github.com/offchainlabs/nitro/arbcompress"
+	"github.com/offchainlabs/nitro/arbos/programs"
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/wavmio"
 )
@@ -69,11 +73,51 @@ const BYTES_PER_FIELD_ELEMENT = 32
 
 var BLS_MODULUS, _ = new(big.Int).SetString("52435875175126190479447740508185965837690552500527637822603658699938581184513", 10)
 
+var stylusModuleHash = common.HexToHash("a149cf8113ff9c95f2c8c2a1423575367de86dd422d87114bb9ea47baf535dd7") // user.wat
+
+func callStylusProgram(recurse int) {
+	evmData := programs.EvmData{}
+	progParams := programs.ProgParams{
+		MaxDepth:  10000,
+		InkPrice:  1,
+		DebugMode: true,
+	}
+	reqHandler := func(req programs.RequestType, input []byte) ([]byte, []byte, uint64) {
+		fmt.Printf("got request type %d req %v\n", req, input)
+		if req == programs.GetBytes32 {
+			if recurse > 0 {
+				callStylusProgram(recurse - 1)
+			}
+			answer := common.Hash{}
+			return answer[:], nil, 1
+		}
+
+		panic("unsupported call")
+	}
+	calldata := common.Hash{}.Bytes()
+	_, _, err := programs.CallProgramLoop(
+		stylusModuleHash,
+		calldata,
+		160000000,
+		&evmData,
+		&progParams,
+		reqHandler)
+	if err != nil {
+		panic(err)
+	}
+}
+
 func main() {
 	fmt.Printf("starting executable with %v arg(s): %v\n", len(os.Args), os.Args)
 	runtime.GC()
 	time.Sleep(time.Second)
 
+	fmt.Printf("Stylus test\n")
+
+	callStylusProgram(5)
+
+	fmt.Printf("Stylus test done!\n")
+
 	// Data for the tree
 	data := [][]byte{
 		[]byte("Foo"),

arbitrator/prover/test-cases/link.wat

@@ -30,7 +30,7 @@
     (data (i32.const 0x140)
         "\47\f7\4f\9c\21\51\4f\52\24\ea\d3\37\5c\bf\a9\1b\1a\5f\ef\22\a5\2a\60\30\c5\52\18\90\6b\b1\51\e5") ;; iops
     (data (i32.const 0x160)
-        "\87\12\6b\19\8a\ce\0c\ba\00\6a\ab\9b\b7\45\bb\0a\ac\48\4d\6b\b8\b5\f9\03\a2\99\8f\64\00\9f\e2\04") ;; user
+        "\a1\49\cf\81\13\ff\9c\95\f2\c8\c2\a1\42\35\75\36\7d\e8\6d\d4\22\d8\71\14\bb\9e\a4\7b\af\53\5d\d7") ;; user
     (data (i32.const 0x180)
         "\ee\47\08\f6\47\b2\10\88\1f\89\86\e7\e3\79\6b\b2\77\43\f1\4e\ee\cf\45\4a\9b\7c\d7\c4\5b\63\b6\d7") ;; return
 

arbitrator/prover/test-cases/user.wat

@@ -2,6 +2,14 @@
 ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
 
 (module
+    (import "vm_hooks" "storage_load_bytes32" (func $storage_load_bytes32 (param i32 i32)))
+
+    (func $storage_load (result i32)
+        i32.const 0
+        i32.const 32
+        call $storage_load_bytes32
+        i32.const 0
+    )
     (func $safe (result i32)
         i32.const 5
     )
@@ -35,6 +43,11 @@
             (then (call $out_of_bounds) (return))
         )
 
+        (i32.eq (local.get $args_len) (i32.const 32))
+        (if
+            (then (call $storage_load) (return))
+        )
+
         unreachable
     )
     (memory (export "memory") 1 1))

arbitrator/stylus/src/cache.rs

@@ -21,7 +21,7 @@ macro_rules! cache {
 }
 
 pub struct InitCache {
-    arbos: HashMap<CacheKey, CacheItem>,
+    long_term: HashMap<CacheKey, CacheItem>,
     lru: LruCache<CacheKey, CacheItem>,
 }
 
@@ -59,20 +59,31 @@ impl CacheItem {
 }
 
 impl InitCache {
+    // current implementation only has one tag that stores to the long_term
+    // future implementations might have more, but 0 is a reserved tag
+    // that will never modify long_term state
+    const ARBOS_TAG: u32 = 1;
+
     fn new(size: usize) -> Self {
         Self {
-            arbos: HashMap::new(),
+            long_term: HashMap::new(),
             lru: LruCache::new(NonZeroUsize::new(size).unwrap()),
         }
     }
 
+    pub fn set_lru_size(size: u32) {
+        cache!()
+            .lru
+            .resize(NonZeroUsize::new(size.try_into().unwrap()).unwrap())
+    }
+
     /// Retrieves a cached value, updating items as necessary.
     pub fn get(module_hash: Bytes32, version: u16, debug: bool) -> Option<(Module, Store)> {
         let mut cache = cache!();
         let key = CacheKey::new(module_hash, version, debug);
 
         // See if the item is in the long term cache
-        if let Some(item) = cache.arbos.get(&key) {
+        if let Some(item) = cache.long_term.get(&key) {
             return Some(item.data());
         }
 
@@ -84,18 +95,27 @@ impl InitCache {
     }
 
     /// Inserts an item into the long term cache, cloning from the LRU cache if able.
+    /// If long_term_tag is 0 will only insert to LRU
     pub fn insert(
         module_hash: Bytes32,
         module: &[u8],
         version: u16,
+        long_term_tag: u32,
         debug: bool,
     ) -> Result<(Module, Store)> {
         let key = CacheKey::new(module_hash, version, debug);
 
         // if in LRU, add to ArbOS
         let mut cache = cache!();
+        if let Some(item) = cache.long_term.get(&key) {
+            return Ok(item.data());
+        }
         if let Some(item) = cache.lru.peek(&key).cloned() {
-            cache.arbos.insert(key, item.clone());
+            if long_term_tag == Self::ARBOS_TAG {
+                cache.long_term.insert(key, item.clone());
+            } else {
+                cache.lru.promote(&key)
+            }
             return Ok(item.data());
         }
         drop(cache);
@@ -105,37 +125,34 @@ impl InitCache {
 
         let item = CacheItem::new(module, engine);
         let data = item.data();
-        cache!().arbos.insert(key, item);
+        let mut cache = cache!();
+        if long_term_tag != Self::ARBOS_TAG {
+            cache.lru.put(key, item);
+        } else {
+            cache.long_term.insert(key, item);
+        }
         Ok(data)
     }
 
-    /// Inserts an item into the short-lived LRU cache.
-    pub fn insert_lru(
-        module_hash: Bytes32,
-        module: &[u8],
-        version: u16,
-        debug: bool,
-    ) -> Result<(Module, Store)> {
-        let engine = CompileConfig::version(version, debug).engine();
-        let module = unsafe { Module::deserialize_unchecked(&engine, module)? };
-
-        let key = CacheKey::new(module_hash, version, debug);
-        let item = CacheItem::new(module, engine);
-        cache!().lru.put(key, item.clone());
-        Ok(item.data())
-    }
-
     /// Evicts an item in the long-term cache.
-    pub fn evict(module_hash: Bytes32, version: u16, debug: bool) {
+    pub fn evict(module_hash: Bytes32, version: u16, long_term_tag: u32, debug: bool) {
+        if long_term_tag != Self::ARBOS_TAG {
+            return;
+        }
         let key = CacheKey::new(module_hash, version, debug);
-        cache!().arbos.remove(&key);
+        let mut cache = cache!();
+        if let Some(item) = cache.long_term.remove(&key) {
+            cache.lru.put(key, item);
+        }
     }
 
-    /// Modifies the cache for reorg, dropping the long-term cache.
-    pub fn reorg(_block: u64) {
+    pub fn clear_long_term(long_term_tag: u32) {
+        if long_term_tag != Self::ARBOS_TAG {
+            return;
+        }
         let mut cache = cache!();
         let cache = &mut *cache;
-        for (key, item) in cache.arbos.drain() {
+        for (key, item) in cache.long_term.drain() {
             cache.lru.put(key, item); // not all will fit, just a heuristic
         }
     }