Merge develop and resolve conflict

commit/merkleroot/rmn/controller.go

@@ -16,6 +16,7 @@ import (
 
 	mapset "github.com/deckarep/golang-set/v2"
 	"golang.org/x/exp/maps"
+	rand2 "golang.org/x/exp/rand"
 	"google.golang.org/protobuf/proto"
 
 	chainsel "github.com/smartcontractkit/chain-selectors"
@@ -364,7 +365,7 @@ func (c *controller) sendObservationRequests(
 		}
 
 		req := &rmnpb.Request{
-			RequestId: newRequestID(),
+			RequestId: newRequestID(c.lggr),
 			Request: &rmnpb.Request_ObservationRequest{
 				ObservationRequest: &rmnpb.ObservationRequest{
 					LaneDest:                    destChain,
@@ -810,7 +811,7 @@ func (c *controller) sendReportSignatureRequest(
 		}
 
 		req := &rmnpb.Request{
-			RequestId: newRequestID(),
+			RequestId: newRequestID(c.lggr),
 			Request: &rmnpb.Request_ReportSignatureRequest{
 				ReportSignatureRequest: reportSigReq,
 			},
@@ -908,7 +909,7 @@ func (c *controller) listenForRmnReportSignatures(
 					continue
 				}
 				req := &rmnpb.Request{
-					RequestId: newRequestID(),
+					RequestId: newRequestID(c.lggr),
 					Request: &rmnpb.Request_ReportSignatureRequest{
 						ReportSignatureRequest: reportSigReq,
 					},
@@ -1072,11 +1073,17 @@ func randomShuffle[T any](s []T) []T {
 	return ret
 }
 
-func newRequestID() uint64 {
+// newRequestID generates a new unique request ID.
+func newRequestID(lggr logger.Logger) uint64 {
 	b := make([]byte, 8)
 	_, err := crand.Read(b)
 	if err != nil {
-		panic(err)
+		// fallback to time-based id in the very rare scenario that the random number generator fails
+		lggr.Warnw("failed to generate random request id, falling back to golang.org/x/exp/rand",
+			"err", err,
+		)
+		rand2.Seed(uint64(time.Now().UnixNano()))
+		return rand2.Uint64()
 	}
 	randomUint64 := binary.LittleEndian.Uint64(b)
 	return randomUint64

commit/merkleroot/rmn/controller_test.go

@@ -729,6 +729,16 @@ func Test_controller_validateSignedObservationResponse(t *testing.T) {
 	}
 }
 
+func Test_newRequestID(t *testing.T) {
+	ids := map[uint64]struct{}{}
+	for i := 0; i < 1000; i++ {
+		id := newRequestID(logger.Test(t))
+		_, ok := ids[id]
+		assert.False(t, ok)
+		ids[id] = struct{}{}
+	}
+}
+
 func getDeterministicPubKey(t *testing.T) *ed25519.PublicKey {
 	// deterministically create a public key by seeding with a 32char string.
 	publicKey, _, err := ed25519.GenerateKey(

execute/exectypes/observation.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 
+	"github.com/smartcontractkit/chainlink-ccip/execute/internal"
 	dt "github.com/smartcontractkit/chainlink-ccip/internal/plugincommon/discovery/discoverytypes"
 	cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
 )
@@ -17,6 +18,8 @@ type MessageObservations map[cciptypes.ChainSelector]map[cciptypes.SeqNum]ccipty
 
 type MessageHashes map[cciptypes.ChainSelector]map[cciptypes.SeqNum]cciptypes.Bytes32
 
+type EncodedMsgAndTokenDataSizes map[cciptypes.ChainSelector]map[cciptypes.SeqNum]int
+
 // Flatten nested maps into a slice of messages.
 func (mo MessageObservations) Flatten() []cciptypes.Message {
 	var results []cciptypes.Message
@@ -43,6 +46,19 @@ func GetHashes(ctx context.Context, mo MessageObservations, hasher cciptypes.Mes
 	return hashes, nil
 }
 
+// GetEncodedMsgAndTokenDataSizes calculates the encoded sizes of messages and their token data counterpart.
+func GetEncodedMsgAndTokenDataSizes(mo MessageObservations, tds TokenDataObservations) EncodedMsgAndTokenDataSizes {
+	sizes := make(EncodedMsgAndTokenDataSizes)
+	for chain, msgs := range mo {
+		sizes[chain] = make(map[cciptypes.SeqNum]int)
+		for seq, msg := range msgs {
+			td := tds[chain][seq]
+			sizes[chain][seq] = internal.EncodedSize(msg) + internal.EncodedSize(td)
+		}
+	}
+	return sizes
+}
+
 // NonceObservations contain the latest nonce for senders in the previously observed messages.
 // Nonces are organized by source chain selector and the string encoded sender address. The address
 // must be encoding according to the destination chain requirements with typeconv.AddressBytesToString.

execute/internal/gas/gas_estimate_provider.go

@@ -1,6 +1,8 @@
 package gas
 
-import "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
+import (
+	"github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
+)
 
 // EstimateProvider is used to estimate the gas cost of a message or a merkle tree.
 // TODO: Move to pkg/types/ccipocr3 or remove.

execute/internal/utils.go

@@ -0,0 +1,20 @@
+package internal
+
+import "encoding/json"
+
+func EncodedSize[T any](obj T) int {
+	enc, err := json.Marshal(obj)
+	if err != nil {
+		return 0
+	}
+	return len(enc)
+}
+
+func RemoveIthElement[T any](slice []T, i int) []T {
+	if i < 0 || i >= len(slice) {
+		return slice // Return the original slice if index is out of bounds
+	}
+	newSlice := make([]T, 0, len(slice)-1)
+	newSlice = append(newSlice, slice[:i]...)
+	return append(newSlice, slice[i+1:]...)
+}

execute/internal/utils_test.go

@@ -0,0 +1,31 @@
+package internal
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemoveIthElement(t *testing.T) {
+	tests := []struct {
+		name     string
+		slice    []int
+		index    int
+		expected []int
+	}{
+		{"Remove middle element", []int{1, 2, 3, 4, 5}, 2, []int{1, 2, 4, 5}},
+		{"Remove first element", []int{1, 2, 3, 4, 5}, 0, []int{2, 3, 4, 5}},
+		{"Remove last element", []int{1, 2, 3, 4, 5}, 4, []int{1, 2, 3, 4}},
+		{"Index out of bounds (negative)", []int{1, 2, 3, 4, 5}, -1, []int{1, 2, 3, 4, 5}},
+		{"Index out of bounds (too large)", []int{1, 2, 3, 4, 5}, 5, []int{1, 2, 3, 4, 5}},
+		{"Single element slice", []int{1}, 0, []int{}},
+		{"Empty slice", []int{}, 0, []int{}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := RemoveIthElement(tt.slice, tt.index)
+			assert.Equal(t, tt.expected, result)
+		})
+	}
+}

execute/observation.go

@@ -236,6 +236,9 @@ func (p *Plugin) getMessagesObservation(
 	if err1 != nil {
 		return exectypes.Observation{}, fmt.Errorf("unable to process token data %w", err1)
 	}
+	if validateTokenDataObservations(messageObs, tkData) != nil {
+		return exectypes.Observation{}, fmt.Errorf("invalid token data observations")
+	}
 
 	costlyMessages, err := p.costlyMessageObserver.Observe(ctx, messageObs.Flatten(), messageTimestamps)
 	if err != nil {
@@ -252,9 +255,11 @@ func (p *Plugin) getMessagesObservation(
 	observation.Hashes = hashes
 	observation.CostlyMessages = costlyMessages
 	observation.TokenData = tkData
+	//observation.MessageAndTokenDataEncodedSizes = exectypes.GetEncodedMsgAndTokenDataSizes(messageObs, tkData)
 
 	// Make sure encoded observation fits within the maximum observation size.
-	observation, err = truncateObservation(observation, maxObservationLength)
+	//observation, err = truncateObservation(observation, maxObservationLength, p.emptyEncodedSizes)
+	observation, err = p.observationOptimizer.TruncateObservation(observation)
 	if err != nil {
 		return exectypes.Observation{}, fmt.Errorf("unable to truncate observation: %w", err)
 	}