From 8790a1fa88f7a1e189810624b2d180e769bbb547 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Thu, 19 Dec 2024 00:25:15 +0800 Subject: [PATCH] Cherry picks recent commits from celestia-integration (#394) * Move tee address (#392) * move espressoTeeVerifierAddr to transaction streamer This commit moves the TEE verifier contract address to the transaction streamer, configurable via the batch posters config. * Fix tests and transaction_streamer * use a string for tee verifier address Koanf cannot parse a common.Address by default, so we should use a string when taking in this config option. * fix compilation * fix e2e test * Fix config parsing * Fix lint and run formatter * Remove outdated test (cherry picked from commit 30689d659730f5a1db6bbd575ddf72c49967cd4b) * Add attestation quote to Espresso payload (#385) * Add attestation quote to Espresso payload * cleanup code * fix lint * add position * fix verify namespace proof bug * Increase hotshot transaction limit * build hotshot payload and add unitests for it * Verify merkle proof first * Rename * push minor fixes --------- Co-authored-by: ImJeremyHe (cherry picked from commit 47e60104ee62313e2f1ed1c1fdef6c52cc3f9bc1) --------- Co-authored-by: Zach Showalter Co-authored-by: Sneh Koul <35871990+Sneh1999@users.noreply.github.com> --- arbnode/batch_poster.go | 95 +------ arbnode/espresso_utils.go | 131 +++++++++ arbnode/espresso_utils_test.go | 124 ++++++++ arbnode/schema.go | 1 + arbnode/transaction_streamer.go | 267 ++++++++++-------- system_tests/espresso_arbos_test.go | 129 --------- .../espresso_sovereign_sequencer_test.go | 2 +- 7 files changed, 424 insertions(+), 325 deletions(-) create mode 100644 arbnode/espresso_utils.go create mode 100644 arbnode/espresso_utils_test.go delete mode 100644 system_tests/espresso_arbos_test.go diff --git a/arbnode/batch_poster.go b/arbnode/batch_poster.go index 18123f582e..2d395a7899 100644 --- a/arbnode/batch_poster.go +++ b/arbnode/batch_poster.go @@ -12,13 +12,10 @@ import ( "fmt" "math" "math/big" - "os" "strings" "sync/atomic" "time" - "github.com/ethereum/go-ethereum/crypto" - "github.com/andybalholm/brotli" "github.com/spf13/pflag" @@ -43,7 +40,6 @@ import ( "github.com/offchainlabs/nitro/arbnode/dataposter" "github.com/offchainlabs/nitro/arbnode/dataposter/storage" "github.com/offchainlabs/nitro/arbnode/redislock" - "github.com/offchainlabs/nitro/arbos" "github.com/offchainlabs/nitro/arbos/arbostypes" "github.com/offchainlabs/nitro/arbstate" "github.com/offchainlabs/nitro/arbstate/daprovider" @@ -185,11 +181,11 @@ type BatchPosterConfig struct { // Espresso specific flags LightClientAddress string `koanf:"light-client-address"` HotShotUrl string `koanf:"hotshot-url"` - UserDataAttestationFile string `koanf:"user-data-attestation-file"` - QuoteFile string `koanf:"quote-file"` UseEscapeHatch bool `koanf:"use-escape-hatch"` EspressoTxnsPollingInterval time.Duration `koanf:"espresso-txns-polling-interval"` EspressoSwitchDelayThreshold uint64 `koanf:"espresso-switch-delay-threshold"` + EspressoMaxTransactionSize uint64 `koanf:"espresso-max-transaction-size"` + EspressoTEEVerifierAddress string `koanf:"espresso-tee-verifier-address"` } func (c *BatchPosterConfig) Validate() error { @@ -246,14 +242,14 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas") f.Duration(prefix+".reorg-resistance-margin", DefaultBatchPosterConfig.ReorgResistanceMargin, "do not post batch if its within this duration from layer 1 minimum bounds. Requires l1-block-bound option not be set to \"ignore\"") f.Bool(prefix+".check-batch-correctness", DefaultBatchPosterConfig.CheckBatchCorrectness, "setting this to true will run the batch against an inbox multiplexer and verifies that it produces the correct set of messages") - f.String(prefix+".user-data-attestation-file", DefaultBatchPosterConfig.UserDataAttestationFile, "specifies the file containing the user data attestation") - f.String(prefix+".quote-file", DefaultBatchPosterConfig.QuoteFile, "specifies the file containing the quote") f.Bool(prefix+".use-escape-hatch", DefaultBatchPosterConfig.UseEscapeHatch, "if true, batches will be posted without doing the espresso verification when hotshot is down. If false, wait for hotshot being up") f.Duration(prefix+".espresso-txns-polling-interval", DefaultBatchPosterConfig.EspressoTxnsPollingInterval, "interval between polling for transactions to be included in the block") f.Uint64(prefix+".espresso-switch-delay-threshold", DefaultBatchPosterConfig.EspressoSwitchDelayThreshold, "specifies the switch delay threshold used to determine hotshot liveness") + f.String(prefix+".espresso-tee-verifier-address", DefaultBatchPosterConfig.EspressoTEEVerifierAddress, "") redislock.AddConfigOptions(prefix+".redis-lock", f) dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig) genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname) + f.Uint64(prefix+".espresso-max-transaction-size", DefaultBatchPosterConfig.EspressoMaxTransactionSize, "specifies the max size of a espresso transasction") } var DefaultBatchPosterConfig = BatchPosterConfig{ @@ -282,13 +278,13 @@ var DefaultBatchPosterConfig = BatchPosterConfig{ GasEstimateBaseFeeMultipleBips: arbmath.OneInUBips * 3 / 2, ReorgResistanceMargin: 10 * time.Minute, CheckBatchCorrectness: true, - UserDataAttestationFile: "", - QuoteFile: "", UseEscapeHatch: false, EspressoTxnsPollingInterval: time.Millisecond * 500, EspressoSwitchDelayThreshold: 350, LightClientAddress: "", HotShotUrl: "", + EspressoMaxTransactionSize: 900 * 1024, + EspressoTEEVerifierAddress: "", } var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{ @@ -325,6 +321,7 @@ var TestBatchPosterConfig = BatchPosterConfig{ EspressoSwitchDelayThreshold: 10, LightClientAddress: "", HotShotUrl: "", + EspressoMaxTransactionSize: 900 * 1024, } type BatchPosterOpts struct { @@ -389,6 +386,8 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e opts.Streamer.UseEscapeHatch = opts.Config().UseEscapeHatch opts.Streamer.espressoTxnsPollingInterval = opts.Config().EspressoTxnsPollingInterval opts.Streamer.espressoSwitchDelayThreshold = opts.Config().EspressoSwitchDelayThreshold + opts.Streamer.espressoMaxTransactionSize = opts.Config().EspressoMaxTransactionSize + opts.Streamer.espressoTEEVerifierAddress = common.HexToAddress(opts.Config().EspressoTEEVerifierAddress) } b := &BatchPoster{ @@ -564,22 +563,11 @@ var EspressoFetchTransactionErr = errors.New("failed to fetch the espresso trans // Adds a block merkle proof to an Espresso justification, providing a proof that a set of transactions // hashes to some light client state root. -func (b *BatchPoster) checkEspressoValidation( - msg *arbostypes.MessageWithMetadata, -) error { +func (b *BatchPoster) checkEspressoValidation() error { if b.streamer.espressoClient == nil && b.streamer.lightClientReader == nil { // We are not using espresso mode since these haven't been set return nil } - // We only submit the user transactions to hotshot. Only those messages created by - // sequencer should wait for the finality - if msg.Message.Header.Kind != arbostypes.L1MessageType_L2Message { - return nil - } - kind := msg.Message.L2msg[0] - if kind != arbos.L2MessageKind_Batch && kind != arbos.L2MessageKind_SignedTx { - return nil - } lastConfirmed, err := b.streamer.getLastConfirmedPos() if err != nil { @@ -605,8 +593,6 @@ func (b *BatchPoster) checkEspressoValidation( log.Warn("skipped espresso verification due to hotshot failure", "pos", b.building.msgCount) return nil } - // TODO: if current position is greater than the `skip`, should set the - // the skip value to nil. This should contribute to better efficiency. } } @@ -1143,7 +1129,7 @@ func (b *BatchPoster) encodeAddBatch( return nil, nil, fmt.Errorf("failed to pack calldata without attestation quote: %w", err) } - attestationQuote, err := b.getAttestationQuote(calldata) + attestationQuote, err := b.streamer.getAttestationQuote(calldata) if err != nil { return nil, nil, fmt.Errorf("failed to get attestation quote: %w", err) } @@ -1174,41 +1160,6 @@ func (b *BatchPoster) encodeAddBatch( return fullCalldata, kzgBlobs, nil } -/** - * This function generates the attestation quote for the user data. - * The user data is hashed using keccak256 and then 32 bytes of padding is added to the hash. - * The hash is then written to a file specified in the config. (For SGX: /dev/attestation/user_report_data) - * The quote is then read from the file specified in the config. (For SGX: /dev/attestation/quote) - */ -func (b *BatchPoster) getAttestationQuote(userData []byte) ([]byte, error) { - if (b.config().UserDataAttestationFile == "") || (b.config().QuoteFile == "") { - return []byte{}, nil - } - - // keccak256 hash of userData - userDataHash := crypto.Keccak256(userData) - - // Add 32 bytes of padding to the user data hash - // because keccak256 hash is 32 bytes and sgx requires 64 bytes of user data - for i := 0; i < 32; i += 1 { - userDataHash = append(userDataHash, 0) - } - - // Write the message to "/dev/attestation/user_report_data" in SGX - err := os.WriteFile(b.config().UserDataAttestationFile, userDataHash, 0600) - if err != nil { - return []byte{}, fmt.Errorf("failed to create user report data file: %w", err) - } - - // Read the quote from "/dev/attestation/quote" in SGX - attestationQuote, err := os.ReadFile(b.config().QuoteFile) - if err != nil { - return []byte{}, fmt.Errorf("failed to read quote file: %w", err) - } - - return attestationQuote, nil -} - var ErrNormalGasEstimationFailed = errors.New("normal gas estimation failed") type estimateGasParams struct { @@ -1471,19 +1422,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) shouldSubmit := b.streamer.shouldSubmitEspressoTransaction() if !b.streamer.UseEscapeHatch || shouldSubmit { for p := b.building.msgCount; p < msgCount; p += 1 { - msg, err := b.streamer.GetMessage(p) - if err != nil { - log.Error("error getting message from streamer", "error", err) - break - } - // We only submit the user transactions to hotshot. - if msg.Message.Header.Kind != arbostypes.L1MessageType_L2Message { - continue - } - kind := msg.Message.L2msg[0] - if kind != arbos.L2MessageKind_Batch && kind != arbos.L2MessageKind_SignedTx { - continue - } err = b.submitEspressoTransactionPos(p) if err != nil { log.Error("error submitting position", "error", err, "pos", p) @@ -1522,7 +1460,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error) break } - err = b.checkEspressoValidation(msg) + err = b.checkEspressoValidation() if err != nil { return false, fmt.Errorf("error checking espresso valdiation: %w", err) } @@ -1820,15 +1758,6 @@ func (b *BatchPoster) Start(ctxIn context.Context) { } b.CallIteratively(func(ctx context.Context) time.Duration { var err error - espresso, _ := b.streamer.isEspressoMode() - if !espresso { - if b.streamer.lightClientReader != nil && b.streamer.espressoClient != nil { - // This mostly happens when a non-espresso nitro is upgrading to espresso nitro. - // The batch poster is set a espresso client and a light client reader, but waiting - // for the upgrade action - return b.config().PollInterval - } - } if common.HexToAddress(b.config().GasRefunderAddress) != (common.Address{}) { gasRefunderBalance, err := b.l1Reader.Client().BalanceAt(ctx, common.HexToAddress(b.config().GasRefunderAddress), nil) if err != nil { diff --git a/arbnode/espresso_utils.go b/arbnode/espresso_utils.go new file mode 100644 index 0000000000..3b7f208d06 --- /dev/null +++ b/arbnode/espresso_utils.go @@ -0,0 +1,131 @@ +package arbnode + +import ( + "bytes" + "encoding/binary" + "errors" + + espressoTypes "github.com/EspressoSystems/espresso-sequencer-go/types" + "github.com/ethereum/go-ethereum/log" + "github.com/offchainlabs/nitro/arbutil" +) + +const MAX_ATTESTATION_QUOTE_SIZE int = 4 * 1024 +const LEN_SIZE int = 8 +const INDEX_SIZE int = 8 + +func buildRawHotShotPayload( + msgPositions []arbutil.MessageIndex, + msgFetcher func(arbutil.MessageIndex) ([]byte, error), + maxSize uint64, +) ([]byte, int) { + + payload := []byte{} + msgCnt := 0 + + for _, p := range msgPositions { + msgBytes, err := msgFetcher(p) + if err != nil { + log.Warn("failed to fetch the message", "pos", p) + break + } + + sizeBuf := make([]byte, LEN_SIZE) + positionBuf := make([]byte, INDEX_SIZE) + + if len(payload)+len(sizeBuf)+len(msgBytes)+len(positionBuf)+MAX_ATTESTATION_QUOTE_SIZE > int(maxSize) { + break + } + binary.BigEndian.PutUint64(sizeBuf, uint64(len(msgBytes))) + binary.BigEndian.PutUint64(positionBuf, uint64(p)) + + // Add the submitted txn position and the size of the message along with the message + payload = append(payload, positionBuf...) + payload = append(payload, sizeBuf...) + payload = append(payload, msgBytes...) + msgCnt += 1 + } + return payload, msgCnt +} + +func signHotShotPayload( + unsigned []byte, + signer func([]byte) ([]byte, error), +) ([]byte, error) { + quote, err := signer(unsigned) + if err != nil { + return nil, err + } + + quoteSizeBuf := make([]byte, LEN_SIZE) + binary.BigEndian.PutUint64(quoteSizeBuf, uint64(len(quote))) + // Put the signature first. That would help easier parsing. + result := quoteSizeBuf + result = append(result, quote...) + result = append(result, unsigned...) + + return result, nil +} + +func validateIfPayloadIsInBlock(p []byte, payloads []espressoTypes.Bytes) bool { + validated := false + for _, payload := range payloads { + if bytes.Equal(p, payload) { + validated = true + break + } + } + return validated +} + +func ParseHotShotPayload(payload []byte) (signature []byte, indices []uint64, messages [][]byte, err error) { + if len(payload) < LEN_SIZE { + return nil, nil, nil, errors.New("payload too short to parse signature size") + } + + // Extract the signature size + signatureSize := binary.BigEndian.Uint64(payload[:LEN_SIZE]) + currentPos := LEN_SIZE + + if len(payload[currentPos:]) < int(signatureSize) { + return nil, nil, nil, errors.New("payload too short for signature") + } + + // Extract the signature + signature = payload[currentPos : currentPos+int(signatureSize)] + currentPos += int(signatureSize) + + indices = []uint64{} + messages = [][]byte{} + + // Parse messages + for { + if currentPos == len(payload) { + break + } + if len(payload[currentPos:]) < LEN_SIZE+INDEX_SIZE { + return nil, nil, nil, errors.New("remaining bytes") + } + + // Extract the index + index := binary.BigEndian.Uint64(payload[currentPos : currentPos+INDEX_SIZE]) + currentPos += INDEX_SIZE + + // Extract the message size + messageSize := binary.BigEndian.Uint64(payload[currentPos : currentPos+LEN_SIZE]) + currentPos += LEN_SIZE + + if len(payload[currentPos:]) < int(messageSize) { + return nil, nil, nil, errors.New("message size mismatch") + } + + // Extract the message + message := payload[currentPos : currentPos+int(messageSize)] + currentPos += int(messageSize) + + indices = append(indices, index) + messages = append(messages, message) + } + + return signature, indices, messages, nil +} diff --git a/arbnode/espresso_utils_test.go b/arbnode/espresso_utils_test.go new file mode 100644 index 0000000000..d91ca10e3b --- /dev/null +++ b/arbnode/espresso_utils_test.go @@ -0,0 +1,124 @@ +package arbnode + +import ( + "bytes" + "encoding/binary" + "fmt" + "testing" + + espressoTypes "github.com/EspressoSystems/espresso-sequencer-go/types" + "github.com/offchainlabs/nitro/arbutil" +) + +func mockMsgFetcher(index arbutil.MessageIndex) ([]byte, error) { + return []byte("message" + fmt.Sprint(index)), nil +} + +func TestParsePayload(t *testing.T) { + msgPositions := []arbutil.MessageIndex{1, 2, 10, 24, 100} + + rawPayload, cnt := buildRawHotShotPayload(msgPositions, mockMsgFetcher, 200*1024) + if cnt != len(msgPositions) { + t.Fatal("exceed transactions") + } + + mockSignature := []byte("fake_signature") + fakeSigner := func(payload []byte) ([]byte, error) { + return mockSignature, nil + } + signedPayload, err := signHotShotPayload(rawPayload, fakeSigner) + if err != nil { + t.Fatalf("failed to sign payload: %v", err) + } + + // Parse the signed payload + signature, indices, messages, err := ParseHotShotPayload(signedPayload) + if err != nil { + t.Fatalf("failed to parse payload: %v", err) + } + + // Validate parsed data + if !bytes.Equal(signature, mockSignature) { + t.Errorf("expected signature 'fake_signature', got %v", mockSignature) + } + + for i, index := range indices { + if arbutil.MessageIndex(index) != msgPositions[i] { + t.Errorf("expected index %d, got %d", msgPositions[i], index) + } + } + + expectedMessages := [][]byte{ + []byte("message1"), + []byte("message2"), + []byte("message10"), + []byte("message24"), + []byte("message100"), + } + for i, message := range messages { + if !bytes.Equal(message, expectedMessages[i]) { + t.Errorf("expected message %s, got %s", expectedMessages[i], message) + } + } +} + +func TestValidateIfPayloadIsInBlock(t *testing.T) { + msgPositions := []arbutil.MessageIndex{1, 2} + + rawPayload, _ := buildRawHotShotPayload(msgPositions, mockMsgFetcher, 200*1024) + fakeSigner := func(payload []byte) ([]byte, error) { + return []byte("fake_signature"), nil + } + signedPayload, err := signHotShotPayload(rawPayload, fakeSigner) + if err != nil { + t.Fatalf("failed to sign payload: %v", err) + } + + // Validate payload in a block + blockPayloads := []espressoTypes.Bytes{ + signedPayload, + []byte("other_payload"), + } + + if !validateIfPayloadIsInBlock(signedPayload, blockPayloads) { + t.Error("expected payload to be validated in block") + } + + if validateIfPayloadIsInBlock([]byte("invalid_payload"), blockPayloads) { + t.Error("did not expect invalid payload to be validated in block") + } +} + +func TestParsePayloadInvalidCases(t *testing.T) { + invalidPayloads := []struct { + description string + payload []byte + }{ + { + description: "Empty payload", + payload: []byte{}, + }, + { + description: "Message size exceeds remaining payload", + payload: func() []byte { + var payload []byte + sigSizeBuf := make([]byte, 8) + binary.BigEndian.PutUint64(sigSizeBuf, 0) + payload = append(payload, sigSizeBuf...) + msgSizeBuf := make([]byte, 8) + binary.BigEndian.PutUint64(msgSizeBuf, 100) + payload = append(payload, msgSizeBuf...) + return payload + }(), + }, + } + + for _, tc := range invalidPayloads { + t.Run(tc.description, func(t *testing.T) { + _, _, _, err := ParseHotShotPayload(tc.payload) + if err == nil { + t.Errorf("expected error for case '%s', but got none", tc.description) + } + }) + } +} diff --git a/arbnode/schema.go b/arbnode/schema.go index 3ce1a3e871..fc5308ee4e 100644 --- a/arbnode/schema.go +++ b/arbnode/schema.go @@ -19,6 +19,7 @@ var ( dbSchemaVersion []byte = []byte("_schemaVersion") // contains a uint64 representing the database schema version espressoSubmittedPos []byte = []byte("_espressoSubmittedPos") // contains the current message indices of the last submitted txns espressoSubmittedHash []byte = []byte("_espressoSubmittedHash") // contains the hash of the last submitted txn + espressoSubmittedPayload []byte = []byte("_espressoSubmittedPayload") // contains the payload of the last submitted espresso txn espressoPendingTxnsPositions []byte = []byte("_espressoPendingTxnsPositions") // contains the index of the pending txns that need to be submitted to espresso espressoLastConfirmedPos []byte = []byte("_espressoLastConfirmedPos") // contains the position of the last confirmed message espressoSkipVerificationPos []byte = []byte("_espressoSkipVerificationPos") // contains the position of the latest message that should skip the validation due to hotshot liveness failure diff --git a/arbnode/transaction_streamer.go b/arbnode/transaction_streamer.go index 4a7296caf5..70a9eee20c 100644 --- a/arbnode/transaction_streamer.go +++ b/arbnode/transaction_streamer.go @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "math/big" + "os" "reflect" "sync" "sync/atomic" @@ -27,6 +28,7 @@ import ( "errors" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -83,15 +85,19 @@ type TransactionStreamer struct { lightClientReader lightclient.LightClientReaderInterface espressoTxnsPollingInterval time.Duration espressoSwitchDelayThreshold uint64 + espressoMaxTransactionSize uint64 // Public these fields for testing - HotshotDown bool - UseEscapeHatch bool + HotshotDown bool + UseEscapeHatch bool + espressoTEEVerifierAddress common.Address } type TransactionStreamerConfig struct { MaxBroadcasterQueueSize int `koanf:"max-broadcaster-queue-size"` MaxReorgResequenceDepth int64 `koanf:"max-reorg-resequence-depth" reload:"hot"` ExecuteMessageLoopDelay time.Duration `koanf:"execute-message-loop-delay" reload:"hot"` + UserDataAttestationFile string `koanf:"user-data-attestation-file"` + QuoteFile string `koanf:"quote-file"` } type TransactionStreamerConfigFetcher func() *TransactionStreamerConfig @@ -100,6 +106,8 @@ var DefaultTransactionStreamerConfig = TransactionStreamerConfig{ MaxBroadcasterQueueSize: 50_000, MaxReorgResequenceDepth: 1024, ExecuteMessageLoopDelay: time.Millisecond * 100, + QuoteFile: "", + UserDataAttestationFile: "", } var TestTransactionStreamerConfig = TransactionStreamerConfig{ @@ -112,6 +120,8 @@ func TransactionStreamerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".max-broadcaster-queue-size", DefaultTransactionStreamerConfig.MaxBroadcasterQueueSize, "maximum cache of pending broadcaster messages") f.Int64(prefix+".max-reorg-resequence-depth", DefaultTransactionStreamerConfig.MaxReorgResequenceDepth, "maximum number of messages to attempt to resequence on reorg (0 = never resequence, -1 = always resequence)") f.Duration(prefix+".execute-message-loop-delay", DefaultTransactionStreamerConfig.ExecuteMessageLoopDelay, "delay when polling calls to execute messages") + f.String(prefix+".user-data-attestation-file", DefaultTransactionStreamerConfig.UserDataAttestationFile, "specifies the file containing the user data attestation") + f.String(prefix+".quote-file", DefaultTransactionStreamerConfig.QuoteFile, "specifies the file containing the quote") } func NewTransactionStreamer( @@ -670,21 +680,6 @@ func (s *TransactionStreamer) AddFakeInitMessage() error { }}) } -func (s *TransactionStreamer) isEspressoMode() (bool, error) { - config, err := s.exec.GetArbOSConfigAtHeight(0) // Pass 0 to get the ArbOS config at current block height. - if err != nil { - return false, fmt.Errorf("error obtaining arbos config: %w", err) - } - if config == nil { - return false, fmt.Errorf("arbos config is not defined") - } - isSetInConfig := config.ArbitrumChainParams.EspressoTEEVerifierAddress != common.Address{} - if !isSetInConfig { - return false, nil - } - return true, nil -} - // Used in redis tests func (s *TransactionStreamer) GetMessageCountSync(t *testing.T) (arbutil.MessageIndex, error) { s.insertionMutex.Lock() @@ -1284,40 +1279,7 @@ func (s *TransactionStreamer) pollSubmittedTransactionForFinality(ctx context.Co return fmt.Errorf("could not get the header (height: %d): %w", height, err) } - // Verify the namespace proof - resp, err := s.espressoClient.FetchTransactionsInBlock(ctx, height, s.chainConfig.ChainID.Uint64()) - if err != nil { - return fmt.Errorf("failed to fetch the transactions in block (height: %d): %w", height, err) - } - - msgs := []arbostypes.L1IncomingMessage{} - for _, p := range submittedTxnPos { - msg, err := s.GetMessage(p) - if err != nil { - return fmt.Errorf("failed to get the message in tx streamer (pos: %d): %w", p, err) - } - if msg.Message != nil { - msgs = append(msgs, *msg.Message) - } - } - - payload, length := buildHotShotPayload(&msgs) - if length != len(msgs) { - return errors.New("failed to rebuild the hotshot payload; the number of messages does not match the expected length") - } - - namespaceOk := espressocrypto.VerifyNamespace( - s.chainConfig.ChainID.Uint64(), - resp.Proof, - *header.Header.GetPayloadCommitment(), - *header.Header.GetNsTable(), - []espressoTypes.Bytes{payload}, - resp.VidCommon, - ) - if !namespaceOk { - return fmt.Errorf("error validating namespace proof (height: %d)", height) - } - + // Verify the merkle proof snapshot, err := s.lightClientReader.FetchMerkleRoot(height, nil) if err != nil { return fmt.Errorf("%w (height: %d): %w", EspressoFetchMerkleRootErr, height, err) @@ -1348,16 +1310,42 @@ func (s *TransactionStreamer) pollSubmittedTransactionForFinality(ctx context.Co return fmt.Errorf("error validating merkle proof (height: %d, snapshot height: %d)", height, snapshot.Height) } + // Verify the namespace proof + resp, err := s.espressoClient.FetchTransactionsInBlock(ctx, height, s.chainConfig.ChainID.Uint64()) + if err != nil { + return fmt.Errorf("failed to fetch the transactions in block (height: %d): %w", height, err) + } + + namespaceOk := espressocrypto.VerifyNamespace( + s.chainConfig.ChainID.Uint64(), + resp.Proof, + *header.Header.GetPayloadCommitment(), + *header.Header.GetNsTable(), + resp.Transactions, + resp.VidCommon, + ) + + if !namespaceOk { + return fmt.Errorf("error validating namespace proof (height: %d)", height) + } + + submittedPayload, err := s.getEspressoSubmittedPayload() + if err != nil { + return fmt.Errorf("submitted payload not found: %w", err) + } + + validated := validateIfPayloadIsInBlock(submittedPayload, resp.Transactions) + if !validated { + return fmt.Errorf("transactions fetched from HotShot doesn't contain the submitted payload") + } + // Validation completed. Update the database s.espressoTxnsStateInsertionMutex.Lock() defer s.espressoTxnsStateInsertionMutex.Unlock() batch := s.db.NewBatch() - if err := s.setEspressoSubmittedPos(batch, nil); err != nil { - return fmt.Errorf("failed to set the submitted pos to nil: %w", err) - } - if err := s.setEspressoSubmittedHash(batch, nil); err != nil { - return fmt.Errorf("failed to set the submitted hash to nil: %w", err) + if err := s.cleanEspressoSubmittedData(batch); err != nil { + return nil } lastConfirmedPos := submittedTxnPos[len(submittedTxnPos)-1] if err := s.setEspressoLastConfirmedPos(batch, &lastConfirmedPos); err != nil { @@ -1410,6 +1398,17 @@ func (s *TransactionStreamer) getEspressoSubmittedHash() (*espressoTypes.TaggedB return hashParsed, nil } +func (s *TransactionStreamer) getEspressoSubmittedPayload() ([]byte, error) { + bytes, err := s.db.Get(espressoSubmittedPayload) + if err != nil { + if dbutil.IsErrNotFound(err) { + return nil, nil + } + return nil, err + } + return bytes, nil +} + func (s *TransactionStreamer) getLastConfirmedPos() (*arbutil.MessageIndex, error) { lastConfirmedBytes, err := s.db.Get(espressoLastConfirmedPos) if err != nil { @@ -1491,7 +1490,34 @@ func (s *TransactionStreamer) setEspressoLastConfirmedPos(batch ethdb.KeyValueWr return nil } -func (s *TransactionStreamer) setSkipVerifiactionPos(batch ethdb.KeyValueWriter, pos *arbutil.MessageIndex) error { +func (s *TransactionStreamer) cleanEspressoSubmittedData(batch ethdb.Batch) error { + if err := s.setEspressoSubmittedPos(batch, nil); err != nil { + return fmt.Errorf("failed to set the submitted pos to nil: %w", err) + } + if err := s.setEspressoSubmittedPayload(batch, nil); err != nil { + return fmt.Errorf("failed to set the submitted pos to nil: %w", err) + } + if err := s.setEspressoSubmittedHash(batch, nil); err != nil { + return fmt.Errorf("failed to set the submitted hash to nil: %w", err) + } + return nil + +} + +func (s *TransactionStreamer) setEspressoSubmittedPayload(batch ethdb.KeyValueWriter, payload []byte) error { + if payload == nil { + err := batch.Delete(espressoSubmittedHash) + return err + } + err := batch.Put(espressoSubmittedPayload, payload) + if err != nil { + return err + + } + return nil +} + +func (s *TransactionStreamer) setSkipVerificationPos(batch ethdb.KeyValueWriter, pos *arbutil.MessageIndex) error { posBytes, err := rlp.EncodeToBytes(pos) if err != nil { return err @@ -1607,21 +1633,27 @@ func (s *TransactionStreamer) submitEspressoTransactions(ctx context.Context) ti } if len(pendingTxnsPos) > 0 { - // get the message at the pending txn position - msgs := []arbostypes.L1IncomingMessage{} - for _, pos := range pendingTxnsPos { + fetcher := func(pos arbutil.MessageIndex) ([]byte, error) { msg, err := s.GetMessage(pos) if err != nil { - log.Error("failed to get espresso submitted pos", "err", err) - return s.espressoTxnsPollingInterval + return nil, err } - if msg.Message != nil { - msgs = append(msgs, *msg.Message) + b, err := rlp.EncodeToBytes(msg) + if err != nil { + return nil, err } + return b, nil } - payload, msgCnt := buildHotShotPayload(&msgs) + + payload, msgCnt := buildRawHotShotPayload(pendingTxnsPos, fetcher, s.espressoMaxTransactionSize) if msgCnt == 0 { - log.Error("failed to build the hotshot transaction: a large message has exceeded the size limit") + log.Error("failed to build the hotshot transaction: a large message has exceeded the size limit or failed to get a message from storage", "size", s.espressoMaxTransactionSize) + return s.espressoTxnsPollingInterval + } + + payload, err = signHotShotPayload(payload, s.getAttestationQuote) + if err != nil { + log.Error("failed to sign the hotshot payload", "err", err) return s.espressoTxnsPollingInterval } @@ -1659,6 +1691,11 @@ func (s *TransactionStreamer) submitEspressoTransactions(ctx context.Context) ti log.Error("failed to set the submitted hash", "err", err) return s.espressoTxnsPollingInterval } + err = s.setEspressoSubmittedPayload(batch, payload) + if err != nil { + log.Error("failed to set the espresso payload", "err", err) + return s.espressoTxnsPollingInterval + } err = batch.Write() if err != nil { @@ -1670,7 +1707,7 @@ func (s *TransactionStreamer) submitEspressoTransactions(ctx context.Context) ti return s.espressoTxnsPollingInterval } -func (s *TransactionStreamer) toggleEscapeHatch(ctx context.Context) error { +func (s *TransactionStreamer) checkEspressoLiveness(ctx context.Context) error { live, err := s.lightClientReader.IsHotShotLive(s.espressoSwitchDelayThreshold) if err != nil { return err @@ -1684,19 +1721,22 @@ func (s *TransactionStreamer) toggleEscapeHatch(ctx context.Context) error { return nil } - // If hotshot is up, escape hatch is disabled - // - check if escape hatch should be activated - // - check if the submitted transaction should be skipped from espresso verification + // If hotshot was previously up, now it is down if !live { log.Warn("enabling the escape hatch, hotshot is down") s.HotshotDown = true } + if !s.UseEscapeHatch { + return nil + } + submittedHash, err := s.getEspressoSubmittedHash() if err != nil { return err } + // No transaction is waiting for espresso finalization if submittedHash == nil { return nil } @@ -1719,6 +1759,7 @@ func (s *TransactionStreamer) toggleEscapeHatch(ctx context.Context) error { return err } if hotshotLive { + // This transaction will be still finalized return nil } submitted, err := s.getEspressoSubmittedPos() @@ -1735,27 +1776,17 @@ func (s *TransactionStreamer) toggleEscapeHatch(ctx context.Context) error { defer s.espressoTxnsStateInsertionMutex.Unlock() batch := s.db.NewBatch() - if s.UseEscapeHatch { - // If escape hatch is used, write down the allowed skip position - // to the database. Batch poster will read this and circumvent the espresso validation - // for certain messages - err = s.setEspressoSubmittedHash(batch, nil) - if err != nil { - return err - } - err = s.setEspressoSubmittedPos(batch, nil) - if err != nil { - return err - } - err = s.setEspressoPendingTxnsPos(batch, nil) - if err != nil { - return err - } - log.Warn("setting last skip verification position", "pos", last) - err = s.setSkipVerifiactionPos(batch, &last) - if err != nil { - return err - } + // If escape hatch is used, write down the allowed skip position + // to the database. Batch poster will read this and circumvent the espresso validation + // for certain messages + err = s.cleanEspressoSubmittedData(batch) + if err != nil { + return err + } + log.Warn("setting last skip verification position", "pos", last) + err = s.setSkipVerificationPos(batch, &last) + if err != nil { + return err } err = batch.Write() if err != nil { @@ -1777,12 +1808,9 @@ func getLogLevel(err error) func(string, ...interface{}) { func (s *TransactionStreamer) espressoSwitch(ctx context.Context, ignored struct{}) time.Duration { retryRate := s.espressoTxnsPollingInterval * 50 - enabledEspresso, err := s.isEspressoMode() - if err != nil { - return retryRate - } + enabledEspresso := s.espressoTEEVerifierAddress != common.Address{} if enabledEspresso { - err := s.toggleEscapeHatch(ctx) + err := s.checkEspressoLiveness(ctx) if err != nil { if ctx.Err() != nil { return 0 @@ -1797,7 +1825,7 @@ func (s *TransactionStreamer) espressoSwitch(ctx context.Context, ignored struct return 0 } logLevel := getLogLevel(err) - logLevel("error polling finality", "err", err) + logLevel("error polling finality, will retry", "err", err) return retryRate } else { espressoMerkleProofEphemeralErrorHandler.Reset() @@ -1833,22 +1861,37 @@ func (s *TransactionStreamer) Start(ctxIn context.Context) error { return stopwaiter.CallIterativelyWith[struct{}](&s.StopWaiterSafe, s.executeMessages, s.newMessageNotifier) } -const ESPRESSO_TRANSACTION_SIZE_LIMIT int = 10 * 1024 +/** + * This function generates the attestation quote for the user data. + * The user data is hashed using keccak256 and then 32 bytes of padding is added to the hash. + * The hash is then written to a file specified in the config. (For SGX: /dev/attestation/user_report_data) + * The quote is then read from the file specified in the config. (For SGX: /dev/attestation/quote) + */ +func (t *TransactionStreamer) getAttestationQuote(userData []byte) ([]byte, error) { + if (t.config().UserDataAttestationFile == "") || (t.config().QuoteFile == "") { + return []byte{}, nil + } -func buildHotShotPayload(msgs *[]arbostypes.L1IncomingMessage) (espressoTypes.Bytes, int) { - payload := []byte{} - msgCnt := 0 + // keccak256 hash of userData + userDataHash := crypto.Keccak256(userData) - sizeBuf := make([]byte, 8) - for _, msg := range *msgs { - if len(payload) >= ESPRESSO_TRANSACTION_SIZE_LIMIT { - break - } - msgByte := msg.L2msg - binary.BigEndian.PutUint64(sizeBuf, uint64(len(msgByte))) - payload = append(payload, sizeBuf...) - payload = append(payload, msgByte...) - msgCnt += 1 + // Add 32 bytes of padding to the user data hash + // because keccak256 hash is 32 bytes and sgx requires 64 bytes of user data + for i := 0; i < 32; i += 1 { + userDataHash = append(userDataHash, 0) + } + + // Write the message to "/dev/attestation/user_report_data" in SGX + err := os.WriteFile(t.config().UserDataAttestationFile, userDataHash, 0600) + if err != nil { + return []byte{}, fmt.Errorf("failed to create user report data file: %w", err) } - return payload, msgCnt + + // Read the quote from "/dev/attestation/quote" in SGX + attestationQuote, err := os.ReadFile(t.config().QuoteFile) + if err != nil { + return []byte{}, fmt.Errorf("failed to read quote file: %w", err) + } + + return attestationQuote, nil } diff --git a/system_tests/espresso_arbos_test.go b/system_tests/espresso_arbos_test.go deleted file mode 100644 index 91474e132a..0000000000 --- a/system_tests/espresso_arbos_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package arbtest - -import ( - "context" - "encoding/json" - "fmt" - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/params" - "github.com/offchainlabs/nitro/arbutil" - "github.com/offchainlabs/nitro/solgen/go/precompilesgen" -) - -func EspressoArbOSTestChainConfig() *params.ChainConfig { - return ¶ms.ChainConfig{ - ChainID: big.NewInt(412346), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - ArbitrumChainParams: EspressoTestChainParams(), - Clique: ¶ms.CliqueConfig{ - Period: 0, - Epoch: 0, - }, - } -} -func EspressoTestChainParams() params.ArbitrumChainParams { - return params.ArbitrumChainParams{ - EnableArbOS: true, - AllowDebugPrecompiles: true, - DataAvailabilityCommittee: false, - InitialArbOSVersion: 31, - InitialChainOwner: common.Address{}, - EspressoTEEVerifierAddress: common.Address{}, - } -} - -func waitForConfigUpdate(t *testing.T, ctx context.Context, builder *NodeBuilder) error { - - return waitForWith(ctx, 120*time.Second, 1*time.Second, func() bool { - newArbOSConfig, err := builder.L2.ExecNode.GetArbOSConfigAtHeight(0) - Require(t, err) - - emptyAddress := common.Address{} - if newArbOSConfig.ArbitrumChainParams.EspressoTEEVerifierAddress != emptyAddress { - return false - } - Require(t, err) - return true - }) -} - -func TestEspressoArbOSConfig(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - builder, cleanup := createL1AndL2Node(ctx, t, true) - defer cleanup() - - err := waitForL1Node(ctx) - Require(t, err) - - cleanEspresso := runEspresso() - defer cleanEspresso() - - // wait for the builder - err = waitForEspressoNode(ctx) - Require(t, err) - - l2Node := builder.L2 - - // Wait for the initial message - expected := arbutil.MessageIndex(1) - err = waitFor(ctx, func() bool { - msgCnt, err := l2Node.ConsensusNode.TxStreamer.GetMessageCount() - if err != nil { - panic(err) - } - - validatedCnt := l2Node.ConsensusNode.BlockValidator.Validated(t) - return msgCnt >= expected && validatedCnt >= expected - }) - Require(t, err) - - initialArbOSConfig, err := builder.L2.ExecNode.GetArbOSConfigAtHeight(0) - Require(t, err) - - // assert that espresso is initially enabled - if initialArbOSConfig.ArbitrumChainParams.EspressoTEEVerifierAddress != common.HexToAddress(verifierAddress) { - err = fmt.Errorf("Initial config should have EspressoTEEVerifierAddress == common.HexToAddress(verifierAddress)!") - - } - Require(t, err) - - newArbOwner, err := precompilesgen.NewArbOwner(common.HexToAddress("0x070"), builder.L2.Client) - Require(t, err) - - newArbDebug, err := precompilesgen.NewArbDebug(common.HexToAddress("0xff"), builder.L2.Client) - Require(t, err) - - l2auth := builder.L2Info.GetDefaultTransactOpts("Owner", ctx) - - _, err = newArbDebug.BecomeChainOwner(&l2auth) - Require(t, err) - chainConfig, err := json.Marshal(EspressoArbOSTestChainConfig()) - Require(t, err) - - chainConfigString := string(chainConfig) - - _, err = newArbOwner.SetChainConfig(&l2auth, chainConfigString) - Require(t, err) - // check if chain config is updated TODO replace this with a wait for with to poll for some time potentially - - err = waitForConfigUpdate(t, ctx, builder) - Require(t, err) -} diff --git a/system_tests/espresso_sovereign_sequencer_test.go b/system_tests/espresso_sovereign_sequencer_test.go index 3d203cf20c..f532b7e4be 100644 --- a/system_tests/espresso_sovereign_sequencer_test.go +++ b/system_tests/espresso_sovereign_sequencer_test.go @@ -25,7 +25,6 @@ func createL1AndL2Node( builder.l1StackConfig.WSHost = "0.0.0.0" builder.l1StackConfig.DataDir = t.TempDir() builder.l1StackConfig.WSModules = append(builder.l1StackConfig.WSModules, "eth") - builder.chainConfig.ArbitrumChainParams.EspressoTEEVerifierAddress = common.HexToAddress(verifierAddress) // poster config builder.nodeConfig.BatchPoster.Enable = true @@ -35,6 +34,7 @@ func createL1AndL2Node( builder.nodeConfig.BatchPoster.MaxDelay = -1000 * time.Hour builder.nodeConfig.BatchPoster.LightClientAddress = lightClientAddress builder.nodeConfig.BatchPoster.HotShotUrl = hotShotUrl + builder.nodeConfig.BatchPoster.EspressoTEEVerifierAddress = verifierAddress // validator config builder.nodeConfig.BlockValidator.Enable = true