Merge branch 'master' into fix-pruning-test

.github/workflows/arbitrator-ci.yml

@@ -20,7 +20,7 @@ env:
   WABT_VERSION: 1.0.32
 
 jobs:
-  coverage:
+  arbitrator:
     name: Run Arbitrator tests
     runs-on: ubuntu-8
     steps:
@@ -31,16 +31,24 @@ jobs:
 
       - name: Install Ubuntu dependencies
         run: |
-          sudo add-apt-repository -y ppa:ethereum/ethereum
           sudo apt-get update && sudo apt-get install -y \
-            build-essential cmake ethereum lld-14 libudev-dev
+            build-essential cmake lld-14 libudev-dev
           sudo ln -s /usr/bin/wasm-ld-14 /usr/local/bin/wasm-ld
 
       - name: Install go
         uses: actions/setup-go@v4
         with:
           go-version: 1.20.x
 
+      - name: Install custom go-ethereum
+        run: |
+          cd /tmp
+          git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git
+          cd go-ethereum
+          # Enable KZG point evaluation precompile early
+          sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go
+          go build -o /usr/local/bin/geth ./cmd/geth
+
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
@@ -54,14 +62,6 @@ jobs:
           components: 'llvm-tools-preview, rustfmt, clippy'
           targets: 'wasm32-wasi, wasm32-unknown-unknown'
 
-      - name: Install grcov
-        uses: jaxxstorm/[email protected]
-        with:
-          repo: mozilla/grcov
-          tag: v0.8.18
-          extension: "\\.bz2"
-          cache: enable
-
       - name: Cache Rust intermediate build products
         uses: actions/cache@v3
         with:
@@ -135,13 +135,6 @@ jobs:
       - name: Make arbitrator libraries
         run: make -j wasm-ci-build
 
-      - name: Enable rust instrumentation
-        run: |
-          echo LLVM_PROFILE_FILE="your_name-%p-%m.profraw" >> $GITHUB_ENV
-          echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
-          echo RUSTFLAGS="-Cinstrument-coverage" >> $GITHUB_ENV
-          echo RUSTDOCFLAGS="-Cpanic=abort" >> $GITHUB_ENV
-
       - name: Clippy check
         run: cargo clippy --all --manifest-path arbitrator/Cargo.toml -- -D warnings
 
@@ -154,18 +147,6 @@ jobs:
       - name: Make proofs from test cases
         run: make -j test-gen-proofs
 
-      - name: Create code-coverage files
-        run: |
-          grcov . --binary-path arbitrator/target/release/ -s . -t lcov --branch --ignore-not-existing --ignore "/*" -o lcov.info
-
-      - name: Upload to codecov.io
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./lcov.info
-          fail_ci_if_error: true
-          verbose: false
-
       - name: Start geth server
         run: |
           geth --dev --http --http.port 8545 &

.github/workflows/arbitrator-skip-ci.yml

@@ -13,7 +13,7 @@ on:
       - 'Makefile'
 
 jobs:
-  coverage:
+  arbitrator:
     name: Run Arbitrator tests
     runs-on: ubuntu-latest
     steps:

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim as brotli-wasm-builder
+FROM debian:bookworm-slim as brotli-wasm-builder
 WORKDIR /workspace
 RUN apt-get update && \
     apt-get install -y cmake make git lbzip2 python3 xz-utils && \
@@ -13,7 +13,7 @@ RUN cd emsdk && . ./emsdk_env.sh && cd .. && ./scripts/build-brotli.sh -w -t /wo
 FROM scratch as brotli-wasm-export
 COPY --from=brotli-wasm-builder /workspace/install/ /
 
-FROM debian:bullseye-slim as brotli-library-builder
+FROM debian:bookworm-slim as brotli-library-builder
 WORKDIR /workspace
 COPY scripts/build-brotli.sh scripts/
 COPY brotli brotli
@@ -24,7 +24,7 @@ RUN apt-get update && \
 FROM scratch as brotli-library-export
 COPY --from=brotli-library-builder /workspace/install/ /
 
-FROM node:16-bullseye-slim as contracts-builder
+FROM node:16-bookworm-slim as contracts-builder
 RUN apt-get update && \
     apt-get install -y git python3 make g++
 WORKDIR /workspace
@@ -34,15 +34,15 @@ COPY contracts contracts/
 COPY Makefile .
 RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-solidity
 
-FROM debian:bullseye-20211220 as wasm-base
+FROM debian:bookworm-20231218 as wasm-base
 WORKDIR /workspace
 RUN apt-get update && apt-get install -y curl build-essential=12.9
 
 FROM wasm-base as wasm-libs-builder
 	# clang / lld used by soft-float wasm
-RUN apt-get install -y clang=1:11.0-51+nmu5 lld=1:11.0-51+nmu5
-    # pinned rust 1.65.0
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.68.2 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
+RUN apt-get install -y clang=1:14.0-55.7~deb12u1 lld=1:14.0-55.7~deb12u1
+    # pinned rust 1.70.0
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.70.0 --target x86_64-unknown-linux-gnu wasm32-unknown-unknown wasm32-wasi
 COPY ./Makefile ./
 COPY arbitrator/arbutil arbitrator/arbutil
 COPY arbitrator/wasm-libraries arbitrator/wasm-libraries
@@ -81,11 +81,11 @@ COPY --from=contracts-builder workspace/contracts/node_modules/@offchainlabs/upg
 COPY --from=contracts-builder workspace/.make/ .make/
 RUN PATH="$PATH:/usr/local/go/bin" NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-wasm-bin
 
-FROM rust:1.68-slim-bullseye as prover-header-builder
+FROM rust:1.70-slim-bookworm as prover-header-builder
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y make && \
+    apt-get install -y make clang && \
     cargo install --force cbindgen
 COPY arbitrator/Cargo.* arbitrator/cbindgen.toml arbitrator/
 COPY ./Makefile ./
@@ -97,15 +97,12 @@ RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-prover-header
 FROM scratch as prover-header-export
 COPY --from=prover-header-builder /workspace/target/ /
 
-FROM rust:1.68-slim-bullseye as prover-builder
+FROM rust:1.75-slim-bookworm as prover-builder
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y make wget gpg software-properties-common zlib1g-dev libstdc++-10-dev wabt
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-    add-apt-repository 'deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main' && \
-    apt-get update && \
-    apt-get install -y llvm-12-dev libclang-common-12-dev
+    apt-get install -y make wget gpg software-properties-common zlib1g-dev \
+        libstdc++-11-dev wabt clang llvm-dev libclang-common-14-dev libpolly-14-dev
 COPY arbitrator/Cargo.* arbitrator/
 COPY arbitrator/arbutil arbitrator/arbutil
 COPY arbitrator/prover/Cargo.toml arbitrator/prover/
@@ -127,7 +124,7 @@ RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make CARGOFLAGS="--features=llvm" build-jit
 FROM scratch as prover-export
 COPY --from=prover-builder /workspace/target/ /
 
-FROM debian:bullseye-slim as module-root-calc
+FROM debian:bookworm-slim as module-root-calc
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
@@ -144,7 +141,7 @@ COPY ./solgen ./solgen
 COPY ./contracts ./contracts
 RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-replay-env
 
-FROM debian:bullseye-slim as machine-versions
+FROM debian:bookworm-slim as machine-versions
 RUN apt-get update && apt-get install -y unzip wget curl
 WORKDIR /workspace/machines
 # Download WAVM machines
@@ -165,8 +162,9 @@ RUN ./download-machine.sh consensus-v10.1 0xda4e3ad5e7feacb817c21c8d0220da7650fe
 RUN ./download-machine.sh consensus-v10.2 0x0754e09320c381566cc0449904c377a52bd34a6b9404432e80afd573b67f7b17
 RUN ./download-machine.sh consensus-v10.3 0xf559b6d4fa869472dabce70fe1c15221bdda837533dfd891916836975b434dec
 RUN ./download-machine.sh consensus-v11 0xf4389b835497a910d7ba3ebfb77aa93da985634f3c052de1290360635be40c4a
+RUN ./download-machine.sh consensus-v11.1 0x68e4fe5023f792d4ef584796c84d710303a5e12ea02d6e37e2b5e9c4332507c4
 
-FROM golang:1.20-bullseye as node-builder
+FROM golang:1.20-bookworm as node-builder
 WORKDIR /workspace
 ARG version=""
 ARG datetime=""
@@ -196,13 +194,13 @@ FROM node-builder as fuzz-builder
 RUN mkdir fuzzers/
 RUN ./scripts/fuzz.bash --build --binary-path /workspace/fuzzers/
 
-FROM debian:bullseye-slim as nitro-fuzzer
+FROM debian:bookworm-slim as nitro-fuzzer
 COPY --from=fuzz-builder /workspace/fuzzers/*.fuzz /usr/local/bin/
 COPY ./scripts/fuzz.bash /usr/local/bin
 RUN mkdir /fuzzcache
 ENTRYPOINT [ "/usr/local/bin/fuzz.bash", "--binary-path", "/usr/local/bin/", "--fuzzcache-path", "/fuzzcache" ]
 
-FROM debian:bullseye-slim as nitro-node-slim
+FROM debian:bookworm-slim as nitro-node-slim
 WORKDIR /home/user
 COPY --from=node-builder /workspace/target/bin/nitro /usr/local/bin/
 COPY --from=node-builder /workspace/target/bin/relay /usr/local/bin/

Makefile

@@ -291,11 +291,15 @@ contracts/test/prover/proofs/float%.json: $(arbitrator_cases)/float%.wasm $(arbi
 contracts/test/prover/proofs/no-stack-pollution.json: $(arbitrator_cases)/no-stack-pollution.wasm $(arbitrator_prover_bin)
 	$(arbitrator_prover_bin) $< -o $@ --allow-hostapi --require-success --always-merkleize
 
-contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/target/wasm32-wasi/release/%.wasm $(arbitrator_prover_bin) $(arbitrator_wasm_libs_nogo)
-	$(arbitrator_prover_bin) $< $(arbitrator_wasm_lib_flags_nogo) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages $(arbitrator_cases)/rust/data/preimages.bin
+target/testdata/preimages.bin:
+	mkdir -p `dirname $@`
+	python3 scripts/create-test-preimages.py $@
 
-contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/main $(arbitrator_prover_bin) $(arbitrator_wasm_libs)
-	$(arbitrator_prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -i 5000000 --require-success --preimages $(arbitrator_cases)/rust/data/preimages.bin
+contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/target/wasm32-wasi/release/%.wasm $(arbitrator_prover_bin) $(arbitrator_wasm_libs_nogo) target/testdata/preimages.bin
+	$(arbitrator_prover_bin) $< $(arbitrator_wasm_lib_flags_nogo) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin
+
+contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/main $(arbitrator_prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin
+	$(arbitrator_prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -i 5000000 --require-success --preimages target/testdata/preimages.bin
 
 # avoid testing read-inboxmsg-10 in onestepproofs. It's used for go challenge testing.
 contracts/test/prover/proofs/read-inboxmsg-10.json: