Merge branch 'master' into add-logopcodes-stylustrace

.github/workflows/arbitrator-ci.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: '**/yarn.lock'
 

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Setup nodejs
         uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: '**/yarn.lock'
 
@@ -174,7 +174,7 @@ jobs:
         run: |
           packages=`go list ./...`
           stdbuf -oL gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 --no-color=false -- ./... -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -parallel=8 -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee full.log | grep -vE "INFO|seal")
-  
+
       - name: Archive detailed run log
         uses: actions/upload-artifact@v3
         with:

.github/workflows/codeql-analysis.yml

@@ -66,7 +66,7 @@ jobs:
     - name: Setup nodejs
       uses: actions/setup-node@v3
       with:
-        node-version: '16'
+        node-version: '18'
         cache: 'yarn'
         cache-dependency-path: '**/yarn.lock'
 

Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
 FROM scratch as brotli-library-export
 COPY --from=brotli-library-builder /workspace/install/ /
 
-FROM node:16-bookworm-slim as contracts-builder
+FROM node:18-bookworm-slim as contracts-builder
 RUN apt-get update && \
     apt-get install -y git python3 make g++ curl
 RUN curl -L https://foundry.paradigm.xyz | bash && . ~/.bashrc && ~/.foundry/bin/foundryup

arbitrator/prover/src/binary.rs

@@ -616,7 +616,10 @@ impl<'a> WasmBinary<'a> {
         cached_init = cached_init.saturating_add(data_len.saturating_mul(75244) / 100_000);
         cached_init = cached_init.saturating_add(footprint as u64 * 5);
 
-        let mut init = cached_init;
+        let mut init: u64 = 0;
+        if compile.version == 1 {
+            init = cached_init; // in version 1 cached cost is part of init cost
+        }
         init = init.saturating_add(funcs.saturating_mul(8252) / 1000);
         init = init.saturating_add(type_len.saturating_mul(1059) / 1000);
         init = init.saturating_add(wasm_len.saturating_mul(1286) / 10_000);

arbitrator/prover/src/programs/config.rs

@@ -162,8 +162,7 @@ impl CompileConfig {
 
         match version {
             0 => {}
-            1 => {
-                // TODO: settle on reasonable values for the v1 release
+            1 | 2 => {
                 config.bounds.heap_bound = Pages(128); // 8 mb
                 config.bounds.max_frame_size = 10 * 1024;
                 config.bounds.max_frame_contention = 4096;

arbitrator/stylus/tests/return-size.wat

@@ -0,0 +1,71 @@
+;; Copyright 2024, Offchain Labs, Inc.
+;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE
+
+(module
+    (import "vm_hooks" "pay_for_memory_grow" (func (param i32)))
+    (import "vm_hooks" "read_args"           (func $read_args    (param i32)))
+    (import "vm_hooks" "write_result"        (func $write_result (param i32 i32)))
+    (func (export "user_entrypoint") (param $args_len i32) (result i32)
+        (local $size i32)
+
+        ;; read input
+        i32.const 0
+        call $read_args
+
+        ;; read the target size from the last 4 bytes of the input big endian
+        ;; byte 1
+        local.get $args_len
+        i32.const 1
+        i32.sub
+        local.tee $size
+        i32.load8_u
+
+        ;; byte 2
+        local.get $size
+        i32.const 1
+        i32.sub
+        local.tee $size
+        i32.load8_u
+        i32.const 8
+        i32.shl
+        i32.or
+
+        ;; byte 3
+        local.get $size
+        i32.const 1
+        i32.sub
+        local.tee $size
+        i32.load8_u
+        i32.const 16
+        i32.shl
+        i32.or
+
+        ;; byte 4
+        local.get $size
+        i32.const 1
+        i32.sub
+        local.tee $size
+        i32.load8_u
+        i32.const 32
+        i32.shl
+        i32.or
+
+        local.tee $size
+
+        ;; grow memory enough to handle the output
+        ;; we start with one page allocated, so no need to round up
+        i32.const 65536
+        i32.div_u
+        memory.grow
+        drop
+
+        ;; set return data
+        i32.const 0
+        local.get $size
+        call $write_result
+
+        ;; return success
+        i32.const 0
+    )
+    (memory (export "memory") 1)
+)

arbos/arbosState/arbosstate.go

@@ -74,8 +74,8 @@ func OpenArbosState(stateDB vm.StateDB, burner burn.Burner) (*ArbosState, error)
 	}
 	return &ArbosState{
 		arbosVersion,
-		30,
-		30,
+		31,
+		31,
 		backingStorage.OpenStorageBackedUint64(uint64(upgradeVersionOffset)),
 		backingStorage.OpenStorageBackedUint64(uint64(upgradeTimestampOffset)),
 		backingStorage.OpenStorageBackedAddress(uint64(networkFeeAccountOffset)),
@@ -318,6 +318,12 @@ func (state *ArbosState) UpgradeArbosVersion(
 		case 30:
 			programs.Initialize(state.backingStorage.OpenSubStorage(programsSubspace))
 
+		case 31:
+			params, err := state.Programs().Params()
+			ensure(err)
+			ensure(params.UpgradeToVersion(2))
+			ensure(params.Save())
+
 		default:
 			return fmt.Errorf(
 				"the chain is upgrading to unsupported ArbOS version %v, %w",

arbos/programs/native.go

@@ -222,10 +222,10 @@ func handleReqImpl(apiId usize, req_type u32, data *rustSlice, costPtr *u64, out
 
 // Caches a program in Rust. We write a record so that we can undo on revert.
 // For gas estimation and eth_call, we ignore permanent updates and rely on Rust's LRU.
-func cacheProgram(db vm.StateDB, module common.Hash, program Program, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) {
+func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) {
 	if runMode == core.MessageCommitMode {
 		// address is only used for logging
-		asm, err := getLocalAsm(db, module, common.Address{}, code, codeHash, params.PageLimit, time, debug, program)
+		asm, err := getLocalAsm(db, module, addressForLogging, code, codeHash, params.PageLimit, time, debug, program)
 		if err != nil {
 			panic("unable to recreate wasm")
 		}

arbos/programs/params.go

@@ -5,6 +5,7 @@ package programs
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -29,6 +30,8 @@ const initialExpiryDays = 365       // deactivate after 1 year.
 const initialKeepaliveDays = 31     // wait a month before allowing reactivation.
 const initialRecentCacheSize = 32   // cache the 32 most recent programs.
 
+const v2MinInitGas = 69 // charge 69 * 128 = 8832 gas (minCachedGas will also be charged in v2).
+
 const MinCachedGasUnits = 32 /// 32 gas for each unit
 const MinInitGasUnits = 128  // 128 gas for each unit
 const CostScalarPercent = 2  // 2% for each unit
@@ -137,6 +140,18 @@ func (p *StylusParams) Save() error {
 	return nil
 }
 
+func (p *StylusParams) UpgradeToVersion(version uint16) error {
+	if version != 2 {
+		return fmt.Errorf("dest version not supported for upgrade")
+	}
+	if p.Version != 1 {
+		return fmt.Errorf("existing version not supported for upgrade")
+	}
+	p.Version = 2
+	p.MinInitGas = v2MinInitGas
+	return nil
+}
+
 func initStylusParams(sto *storage.Storage) {
 	params := &StylusParams{
 		backingStorage:   sto,

arbos/programs/programs.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/log"
+	gethParams "github.com/ethereum/go-ethereum/params"
 	"github.com/offchainlabs/nitro/arbcompress"
 	"github.com/offchainlabs/nitro/arbos/addressSet"
 	"github.com/offchainlabs/nitro/arbos/storage"
@@ -154,7 +155,7 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c
 	// replace the cached asm
 	if cached {
 		code := statedb.GetCode(address)
-		cacheProgram(statedb, info.moduleHash, programData, code, codeHash, params, debugMode, time, runMode)
+		cacheProgram(statedb, info.moduleHash, programData, address, code, codeHash, params, debugMode, time, runMode)
 	}
 
 	return stylusVersion, codeHash, info.moduleHash, dataFee, false, p.setProgram(codeHash, programData)
@@ -163,6 +164,7 @@ func (p Programs) ActivateProgram(evm *vm.EVM, address common.Address, runMode c
 func (p Programs) CallProgram(
 	scope *vm.ScopeContext,
 	statedb vm.StateDB,
+	arbosVersion uint64,
 	interpreter *vm.EVMInterpreter,
 	tracingInfo *util.TracingInfo,
 	calldata []byte,
@@ -172,6 +174,7 @@ func (p Programs) CallProgram(
 	evm := interpreter.Evm()
 	contract := scope.Contract
 	codeHash := contract.CodeHash
+	startingGas := contract.Gas
 	debugMode := evm.ChainConfig().DebugMode()
 
 	params, err := p.Params()
@@ -200,9 +203,10 @@ func (p Programs) CallProgram(
 
 	// pay for program init
 	cached := program.cached || statedb.GetRecentWasms().Insert(codeHash, params.BlockCacheSize)
-	if cached {
+	if cached || program.version > 1 { // in version 1 cached cost is part of init cost
 		callCost = am.SaturatingUAdd(callCost, program.cachedGas(params))
-	} else {
+	}
+	if !cached {
 		callCost = am.SaturatingUAdd(callCost, program.initGas(params))
 	}
 	if err := contract.BurnGas(callCost); err != nil {
@@ -243,7 +247,26 @@ func (p Programs) CallProgram(
 	if runmode == core.MessageCommitMode {
 		arbos_tag = statedb.Database().WasmCacheTag()
 	}
-	return callProgram(address, moduleHash, localAsm, scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag)
+	ret, err := callProgram(address, moduleHash, localAsm, scope, interpreter, tracingInfo, calldata, evmData, goParams, model, arbos_tag)
+	if len(ret) > 0 && arbosVersion >= gethParams.ArbosVersion_StylusFixes {
+		// Ensure that return data costs as least as much as it would in the EVM.
+		evmCost := evmMemoryCost(uint64(len(ret)))
+		if startingGas < evmCost {
+			contract.Gas = 0
+			return nil, vm.ErrOutOfGas
+		}
+		maxGasToReturn := startingGas - evmCost
+		contract.Gas = am.MinInt(contract.Gas, maxGasToReturn)
+	}
+	return ret, err
+}
+
+func evmMemoryCost(size uint64) uint64 {
+	// It would take 100GB to overflow this calculation, so no need to worry about that
+	words := (size + 31) / 32
+	linearCost := words * gethParams.MemoryGas
+	squareCost := (words * words) / gethParams.QuadCoeffDiv
+	return linearCost + squareCost
 }
 
 func getWasm(statedb vm.StateDB, program common.Address) ([]byte, error) {
@@ -362,10 +385,13 @@ func (p Programs) ProgramCached(codeHash common.Hash) (bool, error) {
 }
 
 // Sets whether a program is cached. Errors if trying to cache an expired program.
+// `address` must be present if setting cache to true as of ArbOS 31,
+// and if `address` is present it must have the specified codeHash.
 func (p Programs) SetProgramCached(
 	emitEvent func() error,
 	db vm.StateDB,
 	codeHash common.Hash,
+	address common.Address,
 	cache bool,
 	time uint64,
 	params *StylusParams,
@@ -378,8 +404,8 @@ func (p Programs) SetProgramCached(
 	}
 	expired := program.ageSeconds > am.DaysToSeconds(params.ExpiryDays)
 
-	if program.version == 0 && cache {
-		return ProgramNeedsUpgradeError(0, params.Version)
+	if program.version != params.Version && cache {
+		return ProgramNeedsUpgradeError(program.version, params.Version)
 	}
 	if expired && cache {
 		return ProgramExpiredError(program.ageSeconds)
@@ -405,7 +431,7 @@ func (p Programs) SetProgramCached(
 		if err != nil {
 			return err
 		}
-		cacheProgram(db, moduleHash, program, code, codeHash, params, debug, time, runMode)
+		cacheProgram(db, moduleHash, program, address, code, codeHash, params, debug, time, runMode)
 	} else {
 		evictProgram(db, moduleHash, program.version, debug, runMode, expired)
 	}
@@ -437,7 +463,12 @@ func (p Programs) ProgramTimeLeft(codeHash common.Hash, time uint64, params *Sty
 
 func (p Programs) ProgramInitGas(codeHash common.Hash, time uint64, params *StylusParams) (uint64, uint64, error) {
 	program, err := p.getActiveProgram(codeHash, time, params)
-	return program.initGas(params), program.cachedGas(params), err
+	cachedGas := program.cachedGas(params)
+	initGas := program.initGas(params)
+	if params.Version > 1 {
+		initGas += cachedGas
+	}
+	return initGas, cachedGas, err
 }
 
 func (p Programs) ProgramMemoryFootprint(codeHash common.Hash, time uint64, params *StylusParams) (uint16, error) {

arbos/programs/wasm.go

@@ -95,7 +95,7 @@ func activateProgram(
 }
 
 // stub any non-consensus, Rust-side caching updates
-func cacheProgram(db vm.StateDB, module common.Hash, program Program, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) {
+func cacheProgram(db vm.StateDB, module common.Hash, program Program, addressForLogging common.Address, code []byte, codeHash common.Hash, params *StylusParams, debug bool, time uint64, runMode core.MessageRunMode) {
 }
 func evictProgram(db vm.StateDB, module common.Hash, version uint16, debug bool, mode core.MessageRunMode, forever bool) {
 }

arbos/tx_processor.go

@@ -123,6 +123,7 @@ func (p *TxProcessor) ExecuteWASM(scope *vm.ScopeContext, input []byte, interpre
 	return p.state.Programs().CallProgram(
 		scope,
 		p.evm.StateDB,
+		p.state.ArbOSVersion(),
 		interpreter,
 		tracingInfo,
 		input,

cmd/conf/init.go

@@ -16,6 +16,7 @@ type InitConfig struct {
 	Url                      string        `koanf:"url"`
 	Latest                   string        `koanf:"latest"`
 	LatestBase               string        `koanf:"latest-base"`
+	ValidateChecksum         bool          `koanf:"validate-checksum"`
 	DownloadPath             string        `koanf:"download-path"`
 	DownloadPoll             time.Duration `koanf:"download-poll"`
 	DevInit                  bool          `koanf:"dev-init"`
@@ -39,6 +40,7 @@ var InitConfigDefault = InitConfig{
 	Url:                      "",
 	Latest:                   "",
 	LatestBase:               "https://snapshot.arbitrum.foundation/",
+	ValidateChecksum:         true,
 	DownloadPath:             "/tmp/",
 	DownloadPoll:             time.Minute,
 	DevInit:                  false,
@@ -62,6 +64,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	f.String(prefix+".url", InitConfigDefault.Url, "url to download initialization data - will poll if download fails")
 	f.String(prefix+".latest", InitConfigDefault.Latest, "if set, searches for the latest snapshot of the given kind "+acceptedSnapshotKindsStr)
 	f.String(prefix+".latest-base", InitConfigDefault.LatestBase, "base url used when searching for the latest")
+	f.Bool(prefix+".validate-checksum", InitConfigDefault.ValidateChecksum, "if true: validate the checksum after downloading the snapshot")
 	f.String(prefix+".download-path", InitConfigDefault.DownloadPath, "path to save temp downloaded file")
 	f.Duration(prefix+".download-poll", InitConfigDefault.DownloadPoll, "how long to wait between polling attempts")
 	f.Bool(prefix+".dev-init", InitConfigDefault.DevInit, "init with dev data (1 account with balance) instead of file import")