Merge branch 'ccip-develop' into feature/CCIP-2594-ConfigPagination

.github/actions/setup-postgres/.env

@@ -0,0 +1,5 @@
+POSTGRES_USER=postgres
+POSTGRES_OPTIONS="-c max_connections=1000 -c shared_buffers=2GB -c log_lock_waits=true"
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=chainlink_test
+POSTGRES_HOST_AUTH_METHOD=trust

.github/actions/setup-postgres/action.yml

@@ -0,0 +1,18 @@
+name: Setup Postgresql
+description: Setup postgres docker container via docker-compose, allowing usage of a custom command, see https://github.com/orgs/community/discussions/26688
+inputs:
+  base-path:
+    description: Path to the base of the repo
+    required: false
+    default: .
+runs:
+  using: composite
+  steps:
+    - name: Start postgres service
+      run: docker compose up -d
+      shell: bash
+      working-directory: ${{ inputs.base-path }}/.github/actions/setup-postgres
+    - name: Wait for postgres service to be healthy
+      run: ./wait-for-healthy-postgres.sh
+      shell: bash
+      working-directory: ${{ inputs.base-path }}/.github/actions/setup-postgres

.github/actions/setup-postgres/bin/pg_dump

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# This script acts as a docker replacement around pg_dump so that developers can
+# run DB involved tests locally without having postgres installed.
+#
+# Installation:
+# - Make sure that your PATH doesn't already contain a postgres installation
+# - Add this script to your PATH
+#
+# Usage:
+# You should be able to setup your test db via:
+# - go build -o chainlink.test . # Build the chainlink binary to run test db prep commands from
+# - export CL_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable"
+# - pushd .github/actions/setup-postgres/ # Navigate to the setup-postgres action so we can spin up a docker postgres
+# instance
+# - docker compose up # Spin up postgres
+# - ./chainlink.test local db preparetest # Run the db migration, which will shell out to our pg_dump wrapper too.
+# - popd 
+# - go test -timeout 30s ./core/services/workflows/... -v # Run tests that use the database
+
+cd "$(dirname "$0")" || exit
+
+docker compose exec -T postgres pg_dump "$@" 

.github/actions/setup-postgres/docker-compose.yml

@@ -0,0 +1,15 @@
+name: gha_postgres
+services:
+  postgres:
+    ports:
+      - "5432:5432"
+    container_name: cl_pg
+    image: postgres:14-alpine
+    command: postgres ${POSTGRES_OPTIONS}
+    env_file:
+      - .env
+    healthcheck:
+      test: "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}"
+      interval: 2s
+      timeout: 5s
+      retries: 5

.github/actions/setup-postgres/wait-for-healthy-postgres.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+RETRIES=10
+
+until [ $RETRIES -eq 0 ]; do
+  DOCKER_OUTPUT=$(docker compose ps postgres --status running --format json)
+  JSON_TYPE=$(echo "$DOCKER_OUTPUT" | jq -r 'type')
+
+  if [ "$JSON_TYPE" == "array" ]; then
+    HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.[0].Health')
+  elif [ "$JSON_TYPE" == "object" ]; then
+    HEALTH_STATUS=$(echo "$DOCKER_OUTPUT" | jq -r '.Health')
+  else
+    HEALTH_STATUS="Unknown JSON type: $JSON_TYPE"
+  fi
+
+  echo "postgres health status: $HEALTH_STATUS"
+  if [ "$HEALTH_STATUS" == "healthy" ]; then
+    exit 0
+  fi
+
+  echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
+  sleep 2
+done
+
+exit 1

.github/workflows/ccip-integration-test.yml

@@ -0,0 +1,68 @@
+name: "Run CCIP OCR3 Integration Test"
+
+on:
+  pull_request:
+  push:
+    branches:
+      - 'ccip-develop'
+
+jobs:
+  integration-test-ccip-ocr3:
+    env:
+      # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests
+      # when they should not be using it, while still allowing us to DRY up the setup
+      DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: ['1.22.5']
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Display Go version
+        run: go version
+      - name: Clone CCIP repo
+        run: |
+          git clone https://github.com/smartcontractkit/ccip.git
+          cd ccip
+          git fetch
+          git checkout ccip-develop
+      - name: Update chainlink-ccip dependency in ccip
+        run: |
+          cd ccip
+          go get github.com/smartcontractkit/chainlink-ccip@${{ github.event.pull_request.head.sha }}
+          make gomodtidy
+      - name: Setup Postgres
+        uses: ./.github/actions/setup-postgres
+      - name: Download Go vendor packages
+        run: |
+          cd ccip
+          go mod download
+      - name: Build binary
+        run: |
+          cd ccip
+          go build -o ccip.test .
+      - name: Setup DB
+        run: |
+          cd ccip
+          ./ccip.test local db preparetest
+        env:
+          CL_DATABASE_URL: ${{ env.DB_URL }}
+      - name: Run ccip ocr3 integration test
+        run: |
+          cd ccip
+          go test -v -timeout 3m -run "^TestIntegration_OCR3Nodes$" ./core/capabilities/ccip/ccip_integration_tests
+          EXITCODE=${PIPESTATUS[0]}
+          if [ $EXITCODE -ne 0 ]; then
+            echo "Integration test failed"
+          else
+            echo "Integration test passed!"
+          fi
+          exit $EXITCODE
+        env:
+          CL_DATABASE_URL: ${{ env.DB_URL }}

commit/plugin.go

@@ -226,11 +226,11 @@ func (p *Plugin) ValidateObservation(
 		return fmt.Errorf("validate observer %d reading eligibility: %w", ao.Observer, err)
 	}
 
-	if err := validateObservedTokenPrices(obs.TokenPrices); err != nil {
+	if err := ValidateObservedTokenPrices(obs.TokenPrices); err != nil {
 		return fmt.Errorf("validate token prices: %w", err)
 	}
 
-	if err := validateObservedGasPrices(obs.GasPrices); err != nil {
+	if err := ValidateObservedGasPrices(obs.GasPrices); err != nil {
 		return fmt.Errorf("validate gas prices: %w", err)
 	}
 
@@ -347,7 +347,7 @@ func (p *Plugin) ShouldTransmitAcceptedReport(
 		return false, fmt.Errorf("decode commit plugin report: %w", err)
 	}
 
-	isValid, err := validateMerkleRootsState(ctx, p.lggr, decodedReport, p.ccipReader)
+	isValid, err := ValidateMerkleRootsState(ctx, p.lggr, decodedReport, p.ccipReader)
 	if !isValid {
 		return false, nil
 	}

commit/plugin_e2e_test.go

@@ -16,6 +16,7 @@ import (
 	libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
 	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"
 
 	"github.com/smartcontractkit/chainlink-ccip/chainconfig"
@@ -433,6 +434,10 @@ func setupHomeChainPoller(lggr logger.Logger, chainConfigInfos []reader.ChainCon
 		// to prevent linting error because of logging after finishing tests, we close the poller after each test, having
 		// lower polling interval make it catch up faster
 		10*time.Millisecond,
+		types.BoundContract{
+			Address: "0xCCIPConfigFakeAddress",
+			Name:    consts.ContractNameCCIPConfig,
+		},
 	)
 
 	return homeChain

commit/plugin_functions.go

@@ -559,7 +559,7 @@ func validateObserverReadingEligibility(
 	return nil
 }
 
-func validateObservedTokenPrices(tokenPrices []cciptypes.TokenPrice) error {
+func ValidateObservedTokenPrices(tokenPrices []cciptypes.TokenPrice) error {
 	tokensWithPrice := mapset.NewSet[types.Account]()
 	for _, t := range tokenPrices {
 		if tokensWithPrice.Contains(t.TokenID) {
@@ -575,7 +575,7 @@ func validateObservedTokenPrices(tokenPrices []cciptypes.TokenPrice) error {
 	return nil
 }
 
-func validateObservedGasPrices(gasPrices []cciptypes.GasPriceChain) error {
+func ValidateObservedGasPrices(gasPrices []cciptypes.GasPriceChain) error {
 	// Duplicate gas prices must not appear for the same chain and must not be empty.
 	gasPriceChains := mapset.NewSet[cciptypes.ChainSelector]()
 	for _, g := range gasPrices {
@@ -591,8 +591,8 @@ func validateObservedGasPrices(gasPrices []cciptypes.GasPriceChain) error {
 	return nil
 }
 
-// validateMerkleRootsState merkle roots seq nums validation by comparing with on-chain state.
-func validateMerkleRootsState(
+// ValidateMerkleRootsState merkle roots seq nums validation by comparing with on-chain state.
+func ValidateMerkleRootsState(
 	ctx context.Context,
 	lggr logger.Logger,
 	report cciptypes.CommitPluginReport,

commit/plugin_functions_test.go

@@ -647,7 +647,7 @@ func Test_validateObservedTokenPrices(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validateObservedTokenPrices(tc.tokenPrices)
+			err := ValidateObservedTokenPrices(tc.tokenPrices)
 			if tc.expErr {
 				assert.Error(t, err)
 				return
@@ -700,7 +700,7 @@ func Test_validateObservedGasPrices(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validateObservedGasPrices(tc.gasPrices)
+			err := ValidateObservedGasPrices(tc.gasPrices)
 			if tc.expErr {
 				assert.Error(t, err)
 				return
@@ -1546,7 +1546,7 @@ func Test_validateMerkleRootsState(t *testing.T) {
 				chains = append(chains, snc.ChainSel)
 			}
 			reader.On("NextSeqNum", ctx, chains).Return(tc.onchainNextSeqNums, nil)
-			valid, err := validateMerkleRootsState(ctx, lggr, rep, reader)
+			valid, err := ValidateMerkleRootsState(ctx, lggr, rep, reader)
 			if tc.expErr {
 				assert.Error(t, err)
 				return

commitrmnocb/README.md

@@ -0,0 +1,53 @@
+# OCR3 Commit Plugin
+
+## Context
+The purpose of the OCR3 Commit Plugin is to write reports to a configured destination chain. These reports
+contain metadata of cross-chain messages, from a set of source chains, that can be executed on the destination chain.
+
+## Commit Plugin Design
+
+The plugin is implemented as a state machine, and moves from state to state each round. There are 3 states:
+1. SelectingIntervalsForReport
+    - Determine intervals to be included in the next report
+2. BuildingReport
+    - Build a report from the intervals determined in the previous round
+3. WaitingForReportTransmission
+    - Check if the maximum committed sequence numbers on the dest chain have changed since generating the most
+      recent report, i.e. check if the report has been committed.
+    - If the maximum committed sequence numbers have changed (i.e. the report has been committed) or the maximum
+      number of check attempts have been exhausted, move to the SelectingIntervalsForReport state and generate a new
+      report.
+    - If the maximum committed sequence numbers have _not_ changed (i.e. the report is still in-flight) and the
+      maximum number of check attempts are not been exhausted, move to the WaitingForReportTransmission state in order
+      to check again.
+
+This approach leads to a clear separation of concerns and addresses the complications that can arise if a report
+is not successfully transmitted (as we explicitly only continue once we know the previous report has been committed).
+In this design, full messages are no longer in the observations, only merkle roots and intervals are. This reduces the
+size of observations, which reduces bandwidth and improves performance.
+
+This is the state machine diagram. States are in boxes, outcomes are within arrows.
+
+              Start
+                |
+                V
+    -------------------------------
+    | SelectingIntervalsForReport | <---------|
+    -------------------------------           |
+                |                             |
+        ReportIntervalsSelected               |
+                |                             |
+                V                             |
+        ------------------                    |
+        | BuildingReport | -- ReportEmpty --->|
+        ------------------                    |
+                |                     ReportTransmitted
+         ReportGenerated                     or
+                |                    ReportNotTransmitted
+                V                             |
+    --------------------------------          |
+    | WaitingForReportTransmission | -------->|
+    --------------------------------
+            |           ^
+            |           |
+        ReportNotYetTransmitted

commitrmnocb/chain_support.go

@@ -0,0 +1,73 @@
+package commitrmnocb
+
+import (
+	"fmt"
+
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/smartcontractkit/libocr/commontypes"
+	libocrtypes "github.com/smartcontractkit/libocr/ragep2p/types"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"
+
+	"github.com/smartcontractkit/chainlink-ccip/internal/libs/slicelib"
+	"github.com/smartcontractkit/chainlink-ccip/internal/reader"
+)
+
+// ChainSupport contains functions that enable an oracle to determine which chains are accessible by itself and
+// other oracles
+type ChainSupport interface {
+	// SupportedChains returns the set of chains that the given Oracle is configured to access
+	SupportedChains(oracleID commontypes.OracleID) (mapset.Set[cciptypes.ChainSelector], error)
+
+	// SupportsDestChain returns true if the given oracle supports the dest chain, returns false otherwise
+	SupportsDestChain(oracle commontypes.OracleID) (bool, error)
+
+	// KnownSourceChainsSlice returns a list of all known source chains
+	KnownSourceChainsSlice() ([]cciptypes.ChainSelector, error)
+}
+
+type CCIPChainSupport struct {
+	lggr            logger.Logger
+	homeChain       reader.HomeChain
+	oracleIDToP2pID map[commontypes.OracleID]libocrtypes.PeerID
+	nodeID          commontypes.OracleID
+	destChain       cciptypes.ChainSelector
+}
+
+func (c CCIPChainSupport) KnownSourceChainsSlice() ([]cciptypes.ChainSelector, error) {
+	knownSourceChains, err := c.homeChain.GetKnownCCIPChains()
+	if err != nil {
+		c.lggr.Errorw("error getting known chains", "err", err)
+		return nil, fmt.Errorf("error getting known chains: %w", err)
+	}
+	knownSourceChainsSlice := knownSourceChains.ToSlice()
+	return slicelib.Filter(knownSourceChainsSlice, func(ch cciptypes.ChainSelector) bool { return ch != c.destChain }), nil
+}
+
+// SupportedChains returns the set of chains that the given Oracle is configured to access
+func (c CCIPChainSupport) SupportedChains(oracleID commontypes.OracleID) (mapset.Set[cciptypes.ChainSelector], error) {
+	p2pID, exists := c.oracleIDToP2pID[oracleID]
+	if !exists {
+		return nil, fmt.Errorf("oracle ID %d not found in oracleIDToP2pID", c.nodeID)
+	}
+	supportedChains, err := c.homeChain.GetSupportedChainsForPeer(p2pID)
+	if err != nil {
+		c.lggr.Warnw("error getting supported chains", err)
+		return mapset.NewSet[cciptypes.ChainSelector](), fmt.Errorf("error getting supported chains: %w", err)
+	}
+
+	return supportedChains, nil
+}
+
+// SupportsDestChain returns true if the given oracle supports the dest chain, returns false otherwise
+func (c CCIPChainSupport) SupportsDestChain(oracle commontypes.OracleID) (bool, error) {
+	destChainConfig, err := c.homeChain.GetChainConfig(c.destChain)
+	if err != nil {
+		return false, fmt.Errorf("get chain config: %w", err)
+	}
+	return destChainConfig.SupportedNodes.Contains(c.oracleIDToP2pID[oracle]), nil
+}
+
+// Interface compliance check
+var _ ChainSupport = (*CCIPChainSupport)(nil)