diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml index f2b141fb46..8c491a421c 100644 --- a/.github/workflows/arbitrator-ci.yml +++ b/.github/workflows/arbitrator-ci.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-8 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive @@ -49,23 +49,18 @@ jobs: cache-dependency-path: '**/yarn.lock' - name: Install rust stable - uses: actions-rs/toolchain@v1 - id: install-rust + uses: dtolnay/rust-toolchain@stable with: - profile: minimal - toolchain: "stable" - override: true components: 'llvm-tools-preview, rustfmt, clippy' + targets: 'wasm32-wasi, wasm32-unknown-unknown' - name: Install grcov - uses: actions-rs/install@v0.1 + uses: jaxxstorm/action-install-gh-release@v1.10.0 with: - crate: grcov - version: latest - use-tool-cache: true - - - name: Install rust wasm targets - run: rustup target add wasm32-wasi wasm32-unknown-unknown + repo: mozilla/grcov + tag: v0.8.18 + extension: "\\.bz2" + cache: enable - name: Cache Rust intermediate build products uses: actions/cache@v3 @@ -148,22 +143,13 @@ jobs: echo RUSTDOCFLAGS="-Cpanic=abort" >> $GITHUB_ENV - name: Clippy check - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --all --manifest-path arbitrator/Cargo.toml -- -D warnings + run: cargo clippy --all --manifest-path arbitrator/Cargo.toml -- -D warnings - name: Run rust tests - uses: actions-rs/cargo@v1 - with: - command: test - args: --all --manifest-path arbitrator/Cargo.toml + run: cargo test --all --manifest-path arbitrator/Cargo.toml - name: Rustfmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all --manifest-path arbitrator/Cargo.toml -- --check + run: cargo fmt --all --manifest-path arbitrator/Cargo.toml -- --check - name: Make proofs from test cases run: make -j test-gen-proofs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0734aecfd0..f2c4fac84c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true @@ -53,27 +53,10 @@ jobs: sudo apt-get update && sudo apt-get install -y lld-14 sudo ln -s /usr/bin/wasm-ld-14 /usr/local/bin/wasm-ld - - name: Install rust wasm32-unknown-unknown - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: "stable" - target: wasm32-unknown-unknown - - - name: Install rust wasm32-wasi - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: "stable" - target: wasm32-wasi - - name: Install rust stable - uses: actions-rs/toolchain@v1 - id: install-rust + uses: dtolnay/rust-toolchain@stable with: - profile: minimal - toolchain: "stable" - override: true + targets: 'wasm32-unknown-unknown, wasm32-wasi' - name: Cache Build Products uses: actions/cache@v3 @@ -125,10 +108,9 @@ jobs: run: make -j build-node-deps - name: Lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: version: latest - skip-go-installation: true skip-pkg-cache: true - name: Custom Lint run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cfb5b6eda6..8fb9d80c21 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -32,7 +32,6 @@ jobs: contents: read security-events: write env: - CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'on' WABT_VERSION: 1.0.32 strategy: @@ -44,7 +43,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true @@ -74,12 +73,7 @@ jobs: go-version: 1.20.x - name: Install rust stable - uses: actions-rs/toolchain@v1 - id: install-rust - with: - profile: minimal - toolchain: "stable" - override: true + uses: dtolnay/rust-toolchain@stable - name: Cache Rust Build Products uses: actions/cache@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 33049d4396..30ad88d91a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,12 +23,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 with: driver-opts: network=host @@ -40,7 +40,7 @@ jobs: restore-keys: ${{ runner.os }}-buildx- - name: Build nitro-node docker - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: target: nitro-node push: true @@ -50,7 +50,7 @@ jobs: cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - name: Build nitro-node-dev docker - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: target: nitro-node-dev push: true diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index ede27b26bc..5a99d59c5a 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -114,6 +114,7 @@ func ConfigDefaultTest() *Config { config.ParentChainReader = headerreader.Config{} config.Sequencer = TestSequencerConfig config.ForwardingTarget = "null" + config.ParentChainReader = headerreader.TestConfig _ = config.Validate() diff --git a/go-ethereum b/go-ethereum index b4221631e1..202caf5bf9 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit b4221631e1e5eac86f01582bd74234e3c0f7f5c7 +Subproject commit 202caf5bf944835ef78b4d7c4ba8b48bdc54d65d diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index 28e72b0653..285548dcdb 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -97,9 +97,8 @@ func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, 32) @@ -121,7 +120,6 @@ func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { if balance.Cmp(expectedBalance) != 0 { Fatal(t, "unexpected balance result for last block, want: ", expectedBalance, " have: ", balance) } - } func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { @@ -133,9 +131,8 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, 32) @@ -168,9 +165,8 @@ func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, 32) @@ -203,9 +199,8 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, headerCacheLimit+5) @@ -249,9 +244,8 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, 32) @@ -285,9 +279,9 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { nodeConfig.Sequencer.MaxBlockSpeed = 0 nodeConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 nodeConfig.Caching.Archive = true - // disable caching of states in BlockChain.stateCache + // disable trie/Database.cleans cache, so as states removed from ChainDb won't be cached there nodeConfig.Caching.TrieCleanCache = 0 - nodeConfig.Caching.TrieDirtyCache = 0 + nodeConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 0 nodeConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 _, execNode, l2client, cancelNode := prepareNodeWithHistory(t, ctx, nodeConfig, blockCacheLimit+4) @@ -421,6 +415,9 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig } for i := genesis + 1; i <= genesis+uint64(txCount); i += i % 10 { _, err = client.BalanceAt(ctx, GetTestAddressForAccountName(t, "User2"), new(big.Int).SetUint64(i)) + if err != nil { + t.Log("skipBlocks:", skipBlocks, "skipGas:", skipGas) + } Require(t, err) } @@ -434,10 +431,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { cacheConfig := gethexec.DefaultCachingConfig cacheConfig.Archive = true - // disable caching of states in BlockChain.stateCache - cacheConfig.TrieCleanCache = 0 - cacheConfig.TrieDirtyCache = 0 - // test defaults + //// test defaults testSkippingSavingStateAndRecreatingAfterRestart(t, &cacheConfig, 512) cacheConfig.MaxNumberOfBlocksToSkipStateSaving = 127 diff --git a/system_tests/triedb_race_test.go b/system_tests/triedb_race_test.go new file mode 100644 index 0000000000..8174a9b6a2 --- /dev/null +++ b/system_tests/triedb_race_test.go @@ -0,0 +1,84 @@ +package arbtest + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/arbitrum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + "github.com/offchainlabs/nitro/execution/gethexec" + "github.com/offchainlabs/nitro/util/testhelpers" +) + +func TestTrieDBCommitRace(t *testing.T) { + _ = testhelpers.InitTestLog(t, log.LvlError) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + execConfig := gethexec.ConfigDefaultTest() + execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth + execConfig.Sequencer.MaxBlockSpeed = 0 + execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 + execConfig.Caching.Archive = true + execConfig.Caching.BlockCount = 127 + execConfig.Caching.BlockAge = 0 + execConfig.Caching.MaxNumberOfBlocksToSkipStateSaving = 127 + execConfig.Caching.MaxAmountOfGasToSkipStateSaving = 0 + l2info, node, l2client, _, _, _, l1stack := createTestNodeOnL1WithConfig(t, ctx, true, nil, execConfig, nil, nil) + cancel = func() { + defer requireClose(t, l1stack) + defer node.StopAndWait() + } + defer cancel() + execNode := getExecNode(t, node) + l2info.GenerateAccount("User2") + bc := execNode.Backend.ArbInterface().BlockChain() + + var wg sync.WaitGroup + quit := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + default: + TransferBalance(t, "Faucet", "User2", common.Big1, l2info, l2client, ctx) + case <-quit: + return + } + } + }() + api := execNode.Backend.APIBackend() + blockNumber := 1 + for i := 0; i < 5; i++ { + var roots []common.Hash + for len(roots) < 1024 { + select { + default: + block, err := api.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) + if err == nil && block != nil { + root := block.Root() + if statedb, err := bc.StateAt(root); err == nil { + err := statedb.Database().TrieDB().Reference(root, common.Hash{}) + Require(t, err) + roots = append(roots, root) + } + blockNumber += 1 + } + case <-quit: + return + } + } + t.Log("dereferencing...") + for _, root := range roots { + err := bc.TrieDB().Dereference(root) + Require(t, err) + time.Sleep(1) + } + } + close(quit) + wg.Wait() +}