Merge branch 'master' into jit-add-argv0

Dockerfile

@@ -164,6 +164,7 @@ RUN ./download-machine.sh consensus-v10 0x6b94a7fc388fd8ef3def759297828dc311761e
 RUN ./download-machine.sh consensus-v10.1 0xda4e3ad5e7feacb817c21c8d0220da7650fe9051ece68a3f0b1c5d38bbb27b21
 RUN ./download-machine.sh consensus-v10.2 0x0754e09320c381566cc0449904c377a52bd34a6b9404432e80afd573b67f7b17
 RUN ./download-machine.sh consensus-v10.3 0xf559b6d4fa869472dabce70fe1c15221bdda837533dfd891916836975b434dec
+RUN ./download-machine.sh consensus-v11 0xf4389b835497a910d7ba3ebfb77aa93da985634f3c052de1290360635be40c4a
 
 FROM golang:1.20-bullseye as node-builder
 WORKDIR /workspace

Makefile

@@ -307,6 +307,38 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(arbitrator_pro
 # strategic rules to minimize dependency building
 
 .make/lint: $(DEP_PREDICATE) build-node-deps $(ORDER_ONLY_PREDICATE) .make
+	go run ./linter/recursivelock ./...
+	go run ./linter/comparesame ./...
+
+	# Disabled since we have a lot of use of math/rand package.
+	# We should probably move to crypto/rand at some point even though most of
+	# our uses doesn't seem to be security sensitive.
+	# TODO fix this and enable.
+	# go run ./linter/cryptorand ./... 
+
+	# This yields lot of legitimate warnings, most of which in practice would
+	# probably never happen.
+	# # TODO fix this and enable.
+	# go run ./linter/errcheck ./...
+
+	go run ./linter/featureconfig ./...
+
+	# Disabled since we have high cognitive complexity several places.
+	# TODO fix this and enable.
+	# go run ./linter/gocognit ./...
+
+	go run ./linter/ineffassign ./...
+	go run ./linter/interfacechecker ./...
+	go run ./linter/logruswitherror ./...
+
+	go run ./linter/shadowpredecl ./...
+	go run ./linter/slicedirect ./...
+
+	# Disabled since it fails many places, although ones I looked into seem
+	# to be false positives logically.
+	# TODO fix this and enable and mark false positives with lint ignore.
+	# go run ./linter/uintcast ./...
+
 	go run ./linter/koanf ./...
 	go run ./linter/pointercheck ./...
 	golangci-lint run --fix

arbnode/dataposter/data_poster.go

@@ -5,11 +5,13 @@
 package dataposter
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"math"
 	"math/big"
 	"net/http"
 	"os"
@@ -378,13 +380,27 @@ func (p *DataPoster) evalMaxFeeCapExpr(backlogOfBatches uint64, elapsed time.Dur
 		"ElapsedTimeBase":       float64(config.ElapsedTimeBase),
 		"ElapsedTimeImportance": config.ElapsedTimeImportance,
 		"TargetPriceGWei":       config.TargetPriceGwei,
-		"GWei":                  params.GWei,
 	}
 	result, err := p.maxFeeCapExpression.Evaluate(parameters)
 	if err != nil {
 		return nil, fmt.Errorf("error evaluating maxFeeCapExpression: %w", err)
 	}
-	return arbmath.FloatToBig(result.(float64)), nil
+	resultFloat, ok := result.(float64)
+	if !ok {
+		// This shouldn't be possible because we only pass in float64s as arguments
+		return nil, fmt.Errorf("maxFeeCapExpression evaluated to non-float64: %v", result)
+	}
+	// 1e9 gwei gas price is practically speaking an infinite gas price, so we cap it there.
+	// This also allows the formula to return positive infinity safely.
+	resultFloat = math.Min(resultFloat, 1e9)
+	resultBig := arbmath.FloatToBig(resultFloat * params.GWei)
+	if resultBig == nil {
+		return nil, fmt.Errorf("maxFeeCapExpression evaluated to float64 not convertible to integer: %v", resultFloat)
+	}
+	if resultBig.Sign() < 0 {
+		return nil, fmt.Errorf("maxFeeCapExpression evaluated < 0: %v", resultFloat)
+	}
+	return resultBig, nil
 }
 
 func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit uint64, lastFeeCap *big.Int, lastTipCap *big.Int, dataCreatedAt time.Time, backlogOfBatches uint64) (*big.Int, *big.Int, error) {
@@ -527,8 +543,24 @@ func (p *DataPoster) PostTransaction(ctx context.Context, dataCreatedAt time.Tim
 
 // the mutex must be held by the caller
 func (p *DataPoster) saveTx(ctx context.Context, prevTx, newTx *storage.QueuedTransaction) error {
-	if prevTx != nil && prevTx.Data.Nonce != newTx.Data.Nonce {
-		return fmt.Errorf("prevTx nonce %v doesn't match newTx nonce %v", prevTx.Data.Nonce, newTx.Data.Nonce)
+	if prevTx != nil {
+		if prevTx.Data.Nonce != newTx.Data.Nonce {
+			return fmt.Errorf("prevTx nonce %v doesn't match newTx nonce %v", prevTx.Data.Nonce, newTx.Data.Nonce)
+		}
+
+		// Check if prevTx is the same as newTx and we don't need to do anything
+		oldEnc, err := rlp.EncodeToBytes(prevTx)
+		if err != nil {
+			return fmt.Errorf("failed to encode prevTx: %w", err)
+		}
+		newEnc, err := rlp.EncodeToBytes(newTx)
+		if err != nil {
+			return fmt.Errorf("failed to encode newTx: %w", err)
+		}
+		if bytes.Equal(oldEnc, newEnc) {
+			// No need to save newTx as it's the same as prevTx
+			return nil
+		}
 	}
 	if err := p.queue.Put(ctx, newTx.Data.Nonce, prevTx, newTx); err != nil {
 		return fmt.Errorf("putting new tx in the queue: %w", err)
@@ -537,10 +569,8 @@ func (p *DataPoster) saveTx(ctx context.Context, prevTx, newTx *storage.QueuedTr
 }
 
 func (p *DataPoster) sendTx(ctx context.Context, prevTx *storage.QueuedTransaction, newTx *storage.QueuedTransaction) error {
-	if prevTx == nil || (newTx.FullTx.Hash() != prevTx.FullTx.Hash()) {
-		if err := p.saveTx(ctx, prevTx, newTx); err != nil {
-			return err
-		}
+	if err := p.saveTx(ctx, prevTx, newTx); err != nil {
+		return err
 	}
 	if err := p.client.SendTransaction(ctx, newTx.FullTx); err != nil {
 		if !strings.Contains(err.Error(), "already known") && !strings.Contains(err.Error(), "nonce too low") {
@@ -839,9 +869,9 @@ func DataPosterConfigAddOptions(prefix string, f *pflag.FlagSet, defaultDataPost
 	f.Bool(prefix+".use-db-storage", defaultDataPosterConfig.UseDBStorage, "uses database storage when enabled")
 	f.Bool(prefix+".use-noop-storage", defaultDataPosterConfig.UseNoOpStorage, "uses noop storage, it doesn't store anything")
 	f.Bool(prefix+".legacy-storage-encoding", defaultDataPosterConfig.LegacyStorageEncoding, "encodes items in a legacy way (as it was before dropping generics)")
-	f.String(prefix+".max-fee-cap-formula", defaultDataPosterConfig.MaxFeeCapFormula, "mathematical formula to calculate maximum fee cap the result of which would be float64.\n"+
+	f.String(prefix+".max-fee-cap-formula", defaultDataPosterConfig.MaxFeeCapFormula, "mathematical formula to calculate maximum fee cap gwei the result of which would be float64.\n"+
 		"This expression is expected to be evaluated please refer https://github.com/Knetic/govaluate/blob/master/MANUAL.md to find all available mathematical operators.\n"+
-		"Currently available variables to construct the formula are BacklogOfBatches, UrgencyGWei, ElapsedTime, ElapsedTimeBase, ElapsedTimeImportance, TargetPriceGWei and GWei")
+		"Currently available variables to construct the formula are BacklogOfBatches, UrgencyGWei, ElapsedTime, ElapsedTimeBase, ElapsedTimeImportance, and TargetPriceGWei")
 	f.Duration(prefix+".elapsed-time-base", defaultDataPosterConfig.ElapsedTimeBase, "unit to measure the time elapsed since creation of transaction used for maximum fee cap calculation")
 	f.Float64(prefix+".elapsed-time-importance", defaultDataPosterConfig.ElapsedTimeImportance, "weight given to the units of time elapsed used for maximum fee cap calculation")
 
@@ -878,7 +908,7 @@ var DefaultDataPosterConfig = DataPosterConfig{
 	LegacyStorageEncoding:  false,
 	Dangerous:              DangerousConfig{ClearDBStorage: false},
 	ExternalSigner:         ExternalSignerCfg{Method: "eth_signTransaction"},
-	MaxFeeCapFormula:       "(((BacklogOfBatches * UrgencyGWei) ** 2) + ((ElapsedTime/ElapsedTimeBase) ** 2) * ElapsedTimeImportance + TargetPriceGWei) * GWei",
+	MaxFeeCapFormula:       "((BacklogOfBatches * UrgencyGWei) ** 2) + ((ElapsedTime/ElapsedTimeBase) ** 2) * ElapsedTimeImportance + TargetPriceGWei",
 	ElapsedTimeBase:        10 * time.Minute,
 	ElapsedTimeImportance:  10,
 }
@@ -904,7 +934,7 @@ var TestDataPosterConfig = DataPosterConfig{
 	UseNoOpStorage:         false,
 	LegacyStorageEncoding:  false,
 	ExternalSigner:         ExternalSignerCfg{Method: "eth_signTransaction"},
-	MaxFeeCapFormula:       "(((BacklogOfBatches * UrgencyGWei) ** 2) + ((ElapsedTime/ElapsedTimeBase) ** 2) * ElapsedTimeImportance + TargetPriceGWei) * GWei",
+	MaxFeeCapFormula:       "((BacklogOfBatches * UrgencyGWei) ** 2) + ((ElapsedTime/ElapsedTimeBase) ** 2) * ElapsedTimeImportance + TargetPriceGWei",
 	ElapsedTimeBase:        10 * time.Minute,
 	ElapsedTimeImportance:  10,
 }

arbnode/dataposter/dataposter_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/signer/core/apitypes"
 	"github.com/google/go-cmp/cmp"
@@ -262,6 +263,14 @@ func TestMaxFeeCapFormulaCalculation(t *testing.T) {
 		t.Fatalf("Error evaluating MaxFeeCap expression: %v", err)
 	}
 	if result.Cmp(common.Big0) != 0 {
-		t.Fatalf("Unexpected result. Got: %d, want: 0", result.Uint64())
+		t.Fatalf("Unexpected result. Got: %d, want: 0", result)
+	}
+
+	result, err = p.evalMaxFeeCapExpr(0, time.Since(time.Time{}))
+	if err != nil {
+		t.Fatalf("Error evaluating MaxFeeCap expression: %v", err)
+	}
+	if result.Cmp(big.NewInt(params.GWei)) <= 0 {
+		t.Fatalf("Unexpected result. Got: %d, want: >0", result)
 	}
 }

arbnode/dataposter/dbstorage/storage.go

@@ -6,6 +6,7 @@ package dbstorage
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"strconv"
@@ -140,7 +141,7 @@ func (s *Storage) Put(ctx context.Context, index uint64, prev, new *storage.Queu
 		return fmt.Errorf("encoding previous item: %w", err)
 	}
 	if !bytes.Equal(stored, prevEnc) {
-		return fmt.Errorf("replacing different item than expected at index: %v, stored: %v, prevEnc: %v", index, stored, prevEnc)
+		return fmt.Errorf("replacing different item than expected at index: %v, stored: %v, prevEnc: %v", index, hex.EncodeToString(stored), hex.EncodeToString(prevEnc))
 	}
 	newEnc, err := s.encDec().Encode(new)
 	if err != nil {

arbnode/dataposter/slice/slicestorage.go

@@ -6,6 +6,7 @@ package slice
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 
@@ -90,7 +91,7 @@ func (s *Storage) Put(_ context.Context, index uint64, prev, new *storage.Queued
 			return fmt.Errorf("encoding previous item: %w", err)
 		}
 		if !bytes.Equal(prevEnc, s.queue[queueIdx]) {
-			return fmt.Errorf("replacing different item than expected at index: %v, stored: %v, prevEnc: %v", index, s.queue[queueIdx], prevEnc)
+			return fmt.Errorf("replacing different item than expected at index: %v, stored: %v, prevEnc: %v", index, hex.EncodeToString(s.queue[queueIdx]), hex.EncodeToString(prevEnc))
 		}
 		s.queue[queueIdx] = newEnc
 	} else {

arbnode/dataposter/storage/storage.go

@@ -1,8 +1,12 @@
+// Copyright 2021-2023, Offchain Labs, Inc.
+// For license information, see https://github.com/nitro/blob/master/LICENSE
+
 package storage
 
 import (
 	"errors"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/ethereum/go-ethereum/core/types"
@@ -30,6 +34,44 @@ type QueuedTransaction struct {
 	NextReplacement time.Time
 }
 
+type queuedTransactionForEncoding struct {
+	FullTx          *types.Transaction
+	Data            types.DynamicFeeTx
+	Meta            []byte
+	Sent            bool
+	Created         *RlpTime `rlp:"optional"` // may be earlier than the tx was given to the tx poster
+	NextReplacement *RlpTime `rlp:"optional"`
+}
+
+func (qt *QueuedTransaction) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, queuedTransactionForEncoding{
+		FullTx:          qt.FullTx,
+		Data:            qt.Data,
+		Meta:            qt.Meta,
+		Sent:            qt.Sent,
+		Created:         (*RlpTime)(&qt.Created),
+		NextReplacement: (*RlpTime)(&qt.NextReplacement),
+	})
+}
+
+func (qt *QueuedTransaction) DecodeRLP(s *rlp.Stream) error {
+	var qtEnc queuedTransactionForEncoding
+	if err := s.Decode(&qtEnc); err != nil {
+		return err
+	}
+	qt.FullTx = qtEnc.FullTx
+	qt.Data = qtEnc.Data
+	qt.Meta = qtEnc.Meta
+	qt.Sent = qtEnc.Sent
+	if qtEnc.Created != nil {
+		qt.Created = time.Time(*qtEnc.Created)
+	}
+	if qtEnc.NextReplacement != nil {
+		qt.NextReplacement = time.Time(*qtEnc.NextReplacement)
+	}
+	return nil
+}
+
 // LegacyQueuedTransaction is used for backwards compatibility.
 // Before https://github.com/OffchainLabs/nitro/pull/1773: the queuedTransaction
 // looked like this and was rlp encoded directly. After the pr, we are store

arbnode/dataposter/storage/time.go

@@ -0,0 +1,42 @@
+// Copyright 2021-2023, Offchain Labs, Inc.
+// For license information, see https://github.com/nitro/blob/master/LICENSE
+
+package storage
+
+import (
+	"io"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// time.Time doesn't encode as anything in RLP. This fixes that.
+// It encodes a timestamp as a uint64 unix timestamp in seconds,
+// so any subsecond precision is lost.
+type RlpTime time.Time
+
+type rlpTimeEncoding struct {
+	Seconds uint64
+	Nanos   uint64
+}
+
+func (b *RlpTime) DecodeRLP(s *rlp.Stream) error {
+	var enc rlpTimeEncoding
+	err := s.Decode(&enc)
+	if err != nil {
+		return err
+	}
+	*b = RlpTime(time.Unix(int64(enc.Seconds), int64(enc.Nanos)))
+	return nil
+}
+
+func (b RlpTime) EncodeRLP(w io.Writer) error {
+	return rlp.Encode(w, rlpTimeEncoding{
+		Seconds: uint64(time.Time(b).Unix()),
+		Nanos:   uint64(time.Time(b).Nanosecond()),
+	})
+}
+
+func (b RlpTime) String() string {
+	return time.Time(b).String()
+}

arbnode/dataposter/storage_test.go

@@ -5,6 +5,7 @@ import (
 	"math/big"
 	"path"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -366,3 +367,19 @@ func TestLength(t *testing.T) {
 
 	}
 }
+
+func TestTimeEncoding(t *testing.T) {
+	now := storage.RlpTime(time.Now())
+	enc, err := rlp.EncodeToBytes(now)
+	if err != nil {
+		t.Fatal("failed to encode time", err)
+	}
+	var dec storage.RlpTime
+	err = rlp.DecodeBytes(enc, &dec)
+	if err != nil {
+		t.Fatal("failed to decode time", err)
+	}
+	if !time.Time(dec).Equal(time.Time(now)) {
+		t.Fatalf("time %v encoded then decoded to %v", now, dec)
+	}
+}

arbnode/inbox_tracker.go

@@ -22,6 +22,7 @@ import (
 	"github.com/offchainlabs/nitro/arbstate"
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/broadcaster"
+	m "github.com/offchainlabs/nitro/broadcaster/message"
 	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/util/containers"
 )
@@ -240,7 +241,7 @@ func (t *InboxTracker) PopulateFeedBacklog(broadcastServer *broadcaster.Broadcas
 	if err != nil {
 		return fmt.Errorf("error getting tx streamer message count: %w", err)
 	}
-	var feedMessages []*broadcaster.BroadcastFeedMessage
+	var feedMessages []*m.BroadcastFeedMessage
 	for seqNum := startMessage; seqNum < messageCount; seqNum++ {
 		message, err := t.txStreamer.GetMessage(seqNum)
 		if err != nil {

arbnode/transaction_streamer.go

@@ -32,6 +32,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/broadcaster"
+	m "github.com/offchainlabs/nitro/broadcaster/message"
 	"github.com/offchainlabs/nitro/execution"
 	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/util/arbmath"
@@ -426,7 +427,7 @@ func (s *TransactionStreamer) AddMessages(pos arbutil.MessageIndex, messagesAreC
 	return s.AddMessagesAndEndBatch(pos, messagesAreConfirmed, messages, nil)
 }
 
-func (s *TransactionStreamer) AddBroadcastMessages(feedMessages []*broadcaster.BroadcastFeedMessage) error {
+func (s *TransactionStreamer) AddBroadcastMessages(feedMessages []*m.BroadcastFeedMessage) error {
 	if len(feedMessages) == 0 {
 		return nil
 	}