diff --git a/execute/plugin.go b/execute/plugin.go index 9cfdc7f09..63ce2d4e9 100644 --- a/execute/plugin.go +++ b/execute/plugin.go @@ -318,6 +318,7 @@ func selectReport( encoder cciptypes.ExecutePluginCodec, tokenDataReader exectypes.TokenDataReader, estimateProvider gas.EstimateProvider, + nonces map[cciptypes.ChainSelector]map[string]uint64, commitReports []exectypes.CommitData, maxReportSizeBytes int, maxGas uint64, @@ -332,6 +333,7 @@ func selectReport( tokenDataReader, encoder, estimateProvider, + nonces, uint64(maxReportSizeBytes), maxGas) var stillPendingReports []exectypes.CommitData @@ -419,29 +421,27 @@ func (p *Plugin) Outcome( mergedMessageObservations, nil) - ////////////////////////// - // common preprocessing // - ////////////////////////// - - // flatten commit reports and sort by timestamp. - var commitReports []exectypes.CommitData - for _, report := range observation.CommitReports { - commitReports = append(commitReports, report...) - } - sort.Slice(commitReports, func(i, j int) bool { - return commitReports[i].Timestamp.Before(commitReports[j].Timestamp) - }) - - p.lggr.Debugw( - fmt.Sprintf("[oracle %d] exec outcome: commit reports", p.reportingCfg.OracleID), - "commitReports", commitReports) - state := previousOutcome.State.Next() switch state { case exectypes.GetCommitReports: + // flatten commit reports and sort by timestamp. + var commitReports []exectypes.CommitData + for _, report := range observation.CommitReports { + commitReports = append(commitReports, report...) + } + sort.Slice(commitReports, func(i, j int) bool { + return commitReports[i].Timestamp.Before(commitReports[j].Timestamp) + }) + + p.lggr.Debugw( + fmt.Sprintf("[oracle %d] exec outcome: commit reports", p.reportingCfg.OracleID), + "commitReports", commitReports) + outcome := exectypes.NewOutcome(state, commitReports, cciptypes.ExecutePluginReport{}) return outcome.Encode() case exectypes.GetMessages: + commitReports := previousOutcome.PendingCommitReports + // add messages to their commitReports. for i, report := range commitReports { report.Messages = nil @@ -464,17 +464,20 @@ func (p *Plugin) Outcome( return outcome.Encode() case exectypes.Filter: + commitReports := previousOutcome.PendingCommitReports + // TODO: this function should be pure, a context should not be needed. - outcomeReports, commitReports, err := - selectReport( - context.Background(), - p.lggr, p.msgHasher, - p.reportCodec, - p.tokenDataReader, - p.estimateProvider, - previousOutcome.PendingCommitReports, - maxReportSizeBytes, - p.cfg.OffchainConfig.BatchGasLimit) + outcomeReports, commitReports, err := selectReport( + context.Background(), + p.lggr, + p.msgHasher, + p.reportCodec, + p.tokenDataReader, + p.estimateProvider, + observation.Nonces, + previousOutcome.PendingCommitReports, + maxReportSizeBytes, + p.cfg.OffchainConfig.BatchGasLimit) if err != nil { return ocr3types.Outcome{}, fmt.Errorf("unable to extract proofs: %w", err) } diff --git a/execute/report/builder.go b/execute/report/builder.go index 912d03166..131bdaaa5 100644 --- a/execute/report/builder.go +++ b/execute/report/builder.go @@ -26,6 +26,7 @@ func NewBuilder( tokenDataReader exectypes.TokenDataReader, encoder cciptypes.ExecutePluginCodec, estimateProvider gas.EstimateProvider, + nonces map[cciptypes.ChainSelector]map[string]uint64, maxReportSizeBytes uint64, maxGas uint64, ) ExecReportBuilder { @@ -37,6 +38,8 @@ func NewBuilder( encoder: encoder, hasher: hasher, estimateProvider: estimateProvider, + sendersNonce: nonces, + expectedNonce: make(map[cciptypes.ChainSelector]map[string]uint64), maxReportSizeBytes: maxReportSizeBytes, maxGas: maxGas, @@ -65,6 +68,7 @@ type execReportBuilder struct { encoder cciptypes.ExecutePluginCodec hasher cciptypes.MessageHasher estimateProvider gas.EstimateProvider + sendersNonce map[cciptypes.ChainSelector]map[string]uint64 // Config maxReportSizeBytes uint64 @@ -72,6 +76,8 @@ type execReportBuilder struct { // State accumulated validationMetadata + // expectedNonce is used to track nonces for multiple messages from the same sender. + expectedNonce map[cciptypes.ChainSelector]map[string]uint64 // Result execReports []cciptypes.ExecutePluginReportSingleChain diff --git a/execute/report/report.go b/execute/report/report.go index 36e10bb47..d102ece75 100644 --- a/execute/report/report.go +++ b/execute/report/report.go @@ -40,8 +40,9 @@ func buildSingleChainReportHelper( numMsg := len(report.Messages) if len(report.TokenData) != numMsg { - return cciptypes.ExecutePluginReportSingleChain{}, - fmt.Errorf("token data length mismatch: got %d, expected %d", len(report.TokenData), numMsg) + lggr.Infow("no messages ready for execution", + "sourceChain", report.SourceChain) + return cciptypes.ExecutePluginReportSingleChain{}, nil } lggr.Infow( @@ -124,12 +125,13 @@ const ( TokenDataNotReady messageStatus = "token_data_not_ready" //nolint:gosec // this is not a password TokenDataFetchError messageStatus = "token_data_fetch_error" InsufficientRemainingBatchGas messageStatus = "insufficient_remaining_batch_gas" + MissingNoncesForChain messageStatus = "missing_nonces_for_chain" + MissingNonce messageStatus = "missing_nonce" + InvalidNonce messageStatus = "invalid_nonce" /* SenderAlreadySkipped messageStatus = "sender_already_skipped" MessageMaxGasCalcError messageStatus = "message_max_gas_calc_error" InsufficientRemainingBatchDataLength messageStatus = "insufficient_remaining_batch_data_length" - MissingNonce messageStatus = "missing_nonce" - InvalidNonce messageStatus = "invalid_nonce" AggregateTokenValueComputeError messageStatus = "aggregate_token_value_compute_error" AggregateTokenLimitExceeded messageStatus = "aggregate_token_limit_exceeded" TokenNotInDestTokenPrices messageStatus = "token_not_in_dest_token_prices" @@ -157,7 +159,7 @@ func (b *execReportBuilder) checkMessage( msg := execReport.Messages[idx] - // Check if the message has already been executed. + // 1. Check if the message has already been executed. if slices.Contains(execReport.ExecutedMessages, msg.Header.SequenceNumber) { b.lggr.Infow( "message already executed", @@ -167,7 +169,7 @@ func (b *execReportBuilder) checkMessage( return execReport, AlreadyExecuted, nil } - // Check if token data is ready. + // 2. Check if token data is ready. if b.tokenDataReader == nil { return execReport, Unknown, fmt.Errorf("token data reader must be initialized") } @@ -182,7 +184,7 @@ func (b *execReportBuilder) checkMessage( "error", err) return execReport, TokenDataNotReady, nil } - b.lggr.Infow( + b.lggr.Errorw( "unable to read token data - unknown error", "messageID", msg.Header.MessageID, "sourceChain", execReport.SourceChain, @@ -200,7 +202,53 @@ func (b *execReportBuilder) checkMessage( "seqNum", msg.Header.SequenceNumber, "data", tokenData) - // TODO: Check for valid nonce + // 3. Check if the message has a valid nonce. + if _, ok := b.sendersNonce[execReport.SourceChain]; !ok { + b.lggr.Errorw("Skipping message - nonces not available for chain", + "messageID", msg.Header.MessageID, + "sourceChain", execReport.SourceChain, + "seqNum", msg.Header.SequenceNumber, + ) + return execReport, MissingNoncesForChain, nil + } + + chainNonces := b.sendersNonce[execReport.SourceChain] + sender := msg.Sender.String() + if _, ok := chainNonces[sender]; !ok { + b.lggr.Errorw("Skipping message - missing nonce", + "messageID", msg.Header.MessageID, + "sourceChain", execReport.SourceChain, + "seqNum", msg.Header.SequenceNumber, + ) + return execReport, MissingNonce, nil + } + + if b.expectedNonce == nil { + // initialize expected nonce if needed. + b.expectedNonce = make(map[cciptypes.ChainSelector]map[string]uint64) + } + if _, ok := b.expectedNonce[execReport.SourceChain]; !ok { + // initialize expected nonce if needed. + b.expectedNonce[execReport.SourceChain] = make(map[string]uint64) + } + if _, ok := b.expectedNonce[execReport.SourceChain][sender]; !ok { + b.expectedNonce[execReport.SourceChain][sender] = chainNonces[sender] + 1 + } + + // Check expected nonce is valid for sequenced messages. + // Sequenced messages have non-zero nonces. + if msg.Header.Nonce > 0 && msg.Header.Nonce != b.expectedNonce[execReport.SourceChain][sender] { + b.lggr.Warnw("Skipping message - invalid nonce", + "messageID", msg.Header.MessageID, + "sourceChain", execReport.SourceChain, + "seqNum", msg.Header.SequenceNumber, + "have", msg.Header.Nonce, + "want", b.expectedNonce[execReport.SourceChain][sender], + ) + return execReport, InvalidNonce, nil + } + b.expectedNonce[execReport.SourceChain][sender] = b.expectedNonce[execReport.SourceChain][sender] + 1 + // TODO: Check for fee boost return execReport, ReadyToExecute, nil @@ -284,6 +332,10 @@ func (b *execReportBuilder) buildSingleChainReport( } } + if len(readyMessages) == 0 { + return cciptypes.ExecutePluginReportSingleChain{}, report, ErrEmptyReport + } + // Attempt to include all messages in the report. finalReport, err := buildSingleChainReportHelper(b.ctx, b.lggr, b.hasher, report, readyMessages) diff --git a/execute/report/report_test.go b/execute/report/report_test.go index 2c1d59fee..89641215e 100644 --- a/execute/report/report_test.go +++ b/execute/report/report_test.go @@ -17,6 +17,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/merklemulti" cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" "github.com/smartcontractkit/chainlink-ccip/execute/exectypes" "github.com/smartcontractkit/chainlink-ccip/execute/internal/gas" @@ -169,10 +170,37 @@ func makeMessage(src cciptypes.ChainSelector, num cciptypes.SeqNum, nonce uint64 SourceChainSelector: src, SequenceNumber: num, MsgHash: cciptypes.Bytes32{}, + Nonce: nonce, }, } } +// makeTestCommitReportWithSenders is the same as makeTestCommitReport but overrides the senders. +func makeTestCommitReportWithSenders( + hasher cciptypes.MessageHasher, + numMessages, + srcChain, + firstSeqNum, + block int, + timestamp int64, + senders []cciptypes.Bytes, + rootOverride cciptypes.Bytes32, + executed []cciptypes.SeqNum, +) exectypes.CommitData { + if len(senders) == 0 || len(senders) != numMessages { + panic("wrong number of senders provided") + } + + data := makeTestCommitReport(hasher, numMessages, srcChain, firstSeqNum, + block, timestamp, senders[0], rootOverride, executed) + + for i := range data.Messages { + data.Messages[i].Sender = senders[i] + } + + return data +} + // makeTestCommitReport creates a basic commit report with messages given different parameters. This function // will panic if the input parameters are inconsistent. func makeTestCommitReport( @@ -182,6 +210,7 @@ func makeTestCommitReport( firstSeqNum, block int, timestamp int64, + sender cciptypes.Bytes, rootOverride cciptypes.Bytes32, executed []cciptypes.SeqNum, ) exectypes.CommitData { @@ -195,10 +224,12 @@ func makeTestCommitReport( } var messages []cciptypes.Message for i := 0; i < numMessages; i++ { - messages = append(messages, makeMessage( + msg := makeMessage( cciptypes.ChainSelector(srcChain), cciptypes.SeqNum(i+firstSeqNum), - uint64(i))) + uint64(i)+1) + msg.Sender = sender + messages = append(messages, msg) } commitReport := exectypes.CommitData{ @@ -384,9 +415,25 @@ func Test_Builder_Build(t *testing.T) { codec := mocks.NewExecutePluginJSONReportCodec() lggr := logger.Test(t) tokenDataReader := tdr{mode: good} + sender, err := cciptypes.NewBytesFromString(utils.RandomAddress().String()) + require.NoError(t, err) + defaultNonces := map[cciptypes.ChainSelector]map[string]uint64{ + 1: { + sender.String(): 0, + }, + 2: { + sender.String(): 0, + }, + } + tenSenders := make([]cciptypes.Bytes, 10) + for i := range tenSenders { + tenSenders[i], err = cciptypes.NewBytesFromString(utils.RandomAddress().String()) + require.NoError(t, err) + } type args struct { reports []exectypes.CommitData + nonces map[cciptypes.ChainSelector]map[string]uint64 maxReportSize uint64 maxGasLimit uint64 } @@ -410,10 +457,12 @@ func Test_Builder_Build(t *testing.T) { { name: "half report", args: args{ - maxReportSize: 2300, + maxReportSize: 2529, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), }, @@ -428,8 +477,10 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 10000, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), }, @@ -443,11 +494,14 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 15000, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), makeTestCommitReport(hasher, 20, 2, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), }, @@ -459,13 +513,16 @@ func Test_Builder_Build(t *testing.T) { { name: "one and half reports", args: args{ - maxReportSize: 8500, + maxReportSize: 9500, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), makeTestCommitReport(hasher, 20, 2, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), }, @@ -478,13 +535,16 @@ func Test_Builder_Build(t *testing.T) { { name: "exactly one report", args: args{ - maxReportSize: 4200, + maxReportSize: 4600, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), makeTestCommitReport(hasher, 20, 2, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil), }, @@ -497,10 +557,16 @@ func Test_Builder_Build(t *testing.T) { { name: "execute remainder of partially executed report", args: args{ - maxReportSize: 2500, + maxReportSize: 2600, maxGasLimit: 10000000, + nonces: map[cciptypes.ChainSelector]map[string]uint64{ + 1: { + sender.String(): 5, + }, + }, reports: []exectypes.CommitData{ - makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + makeTestCommitReportWithSenders(hasher, 10, 1, 100, 999, 10101010101, + tenSenders, cciptypes.Bytes32{}, // generate a correct root. []cciptypes.SeqNum{100, 101, 102, 103, 104}), }, @@ -512,10 +578,16 @@ func Test_Builder_Build(t *testing.T) { { name: "partially execute remainder of partially executed report", args: args{ - maxReportSize: 2050, + maxReportSize: 2200, maxGasLimit: 10000000, + nonces: map[cciptypes.ChainSelector]map[string]uint64{ + 1: { + sender.String(): 5, + }, + }, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. []cciptypes.SeqNum{100, 101, 102, 103, 104}), }, @@ -530,8 +602,18 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 3500, maxGasLimit: 10000000, + nonces: map[cciptypes.ChainSelector]map[string]uint64{ + 1: { + tenSenders[1].String(): 1, + tenSenders[3].String(): 3, + tenSenders[5].String(): 5, + tenSenders[7].String(): 7, + tenSenders[9].String(): 9, + }, + }, reports: []exectypes.CommitData{ - makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + makeTestCommitReportWithSenders(hasher, 10, 1, 100, 999, 10101010101, + tenSenders, cciptypes.Bytes32{}, // generate a correct root. []cciptypes.SeqNum{100, 102, 104, 106, 108}), }, @@ -543,10 +625,20 @@ func Test_Builder_Build(t *testing.T) { { name: "partially execute remainder of partially executed sparse report", args: args{ - maxReportSize: 2050, + maxReportSize: 2250, maxGasLimit: 10000000, + nonces: map[cciptypes.ChainSelector]map[string]uint64{ + 1: { + tenSenders[1].String(): 1, + tenSenders[3].String(): 3, + tenSenders[5].String(): 5, + tenSenders[7].String(): 7, + tenSenders[9].String(): 9, + }, + }, reports: []exectypes.CommitData{ - makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + makeTestCommitReportWithSenders(hasher, 10, 1, 100, 999, 10101010101, + tenSenders, cciptypes.Bytes32{}, // generate a correct root. []cciptypes.SeqNum{100, 102, 104, 106, 108}), }, @@ -561,8 +653,10 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 10000, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ breakCommitReport(makeTestCommitReport(hasher, 10, 1, 101, 1000, 10101010102, + sender, cciptypes.Bytes32{}, // generate a correct root. nil)), }, @@ -572,8 +666,10 @@ func Test_Builder_Build(t *testing.T) { { name: "invalid merkle root", args: args{ + nonces: defaultNonces, reports: []exectypes.CommitData{ makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, mustMakeBytes(""), // random root nil), }, @@ -585,9 +681,11 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 10000, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ setMessageData(5, 20000, makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil)), }, @@ -602,10 +700,12 @@ func Test_Builder_Build(t *testing.T) { args: args{ maxReportSize: 10000, maxGasLimit: 10000000, + nonces: defaultNonces, reports: []exectypes.CommitData{ setMessageData(8, 20000, setMessageData(5, 20000, makeTestCommitReport(hasher, 10, 1, 100, 999, 10101010101, + sender, cciptypes.Bytes32{}, // generate a correct root. nil))), }, @@ -631,6 +731,7 @@ func Test_Builder_Build(t *testing.T) { tokenDataReader, codec, evm.EstimateProvider{}, + tt.args.nonces, tt.args.maxReportSize, tt.args.maxGasLimit) var updatedMessages []exectypes.CommitData