[TT-367] [TT-745] Quick and Dirty OCRv2 Soak Test (#11487)

* Quick and Dirty OCRv2 Soak Test

* Enable version in on-demand action

* Remove inputs

* Default input

* Fix echos

* Debug

* Bash shell

* Increment another way

* No 2130 silly

* When did incrementing get hard?

* Cleanup debug

* Include in reporter

* Fix version input

* Instantiate map

* Fix OCR2 job names

* Change default intpus

* Bridge work

* Build pair IDs properly

* Cleanup paths

* Build more bridges

* Fix configuration

* Fix config build

* Fix import

.github/workflows/on-demand-ocr-soak-test.yml

@@ -55,18 +55,16 @@ on:
         description: Container image version for the Chainlink nodes
         required: true
         default: "2.7.0"
-      testDuration:
-        description: Duration of the test (time string)
-        required: false
-        default: 15m
-      chainlinkNodeFunding:
-        description: How much to fund each Chainlink node (in ETH)
-        required: false
-        default: ".001"
-      timeBetweenRounds:
-        description: How long to wait before starting a new round
+      ocrVersion:
+        description: Version of OCR to Use
+        type: choice
+        options:
+          - 1
+          - 2
+      testInputs:
+        description: Duration;Funding;TimeBetweenRounds
         required: false
-        default: 1m
+        default: "10m;.1;1m"
 
 jobs:
   ocr_soak_test:
@@ -84,9 +82,7 @@ jobs:
       SELECTED_NETWORKS: ${{ inputs.network }}
       SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
       SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
-      OCR_TEST_DURATION: ${{ inputs.testDuration }}
-      OCR_CHAINLINK_NODE_FUNDING: ${{ inputs.chainlinkNodeFunding }}
-      OCR_TIME_BETWEEN_ROUNDS: ${{ inputs.timeBetweenRounds }}
+      OCR_VERSION: ${{ inputs.ocrVersion }}
       TEST_LOG_LEVEL: debug
       REF_NAME: ${{ github.head_ref || github.ref_name }}
       ENV_JOB_IMAGE_BASE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests
@@ -101,6 +97,7 @@ jobs:
         continue-on-error: true
       - name: Get Inputs
         run: |
+          # Mask the sensitive inputs
           EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH)
           EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH)
           EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH)
@@ -115,6 +112,26 @@ jobs:
           echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV
           echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV
           echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV
+
+          # Read in our test inputs
+          IFS=';' read -ra parts <<< "${{ inputs.testInputs }}"
+          index=0
+          for part in "${parts[@]}"; do
+              # A little hacky, but good enough for this
+              if [ $index -eq 0 ]; then
+                  echo "OCR_TEST_DURATION=$part"
+                  echo "OCR_TEST_DURATION=$part" >> $GITHUB_ENV
+              elif [ $index -eq 1 ]; then
+                  echo "OCR_CHAINLINK_NODE_FUNDING=$part"
+                  echo "OCR_CHAINLINK_NODE_FUNDING=$part" >> $GITHUB_ENV
+              elif [ $index -eq 2 ]; then
+                  echo "OCR_TIME_BETWEEN_ROUNDS=$part"
+                  echo "OCR_TIME_BETWEEN_ROUNDS=$part" >> $GITHUB_ENV
+              else
+                  echo "Additional Unregistered Input: $part"
+              fi
+              ((index+=1))
+          done
       - name: Checkout the repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:

integration-tests/actions/ocr2_helpers.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/google/uuid"
 	"github.com/lib/pq"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -23,6 +24,7 @@ import (
 	ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client"
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/testhelpers"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
@@ -104,12 +106,15 @@ func ConfigureOCRv2AggregatorContracts(
 }
 
 // BuildMedianOCR2Config builds a default OCRv2 config for the given chainlink nodes for a standard median aggregation job
-func BuildMedianOCR2Config(workerNodes []*client.ChainlinkK8sClient) (*contracts.OCRv2Config, error) {
+func BuildMedianOCR2Config(
+	workerNodes []*client.ChainlinkK8sClient,
+	ocrOffchainOptions contracts.OffchainOptions,
+) (*contracts.OCRv2Config, error) {
 	S, oracleIdentities, err := GetOracleIdentities(workerNodes)
 	if err != nil {
 		return nil, err
 	}
-	signerKeys, transmitterAccounts, f_, onchainConfig, offchainConfigVersion, offchainConfig, err := confighelper.ContractSetConfigArgsForTests(
+	signerKeys, transmitterAccounts, f_, _, offchainConfigVersion, offchainConfig, err := confighelper.ContractSetConfigArgsForTests(
 		30*time.Second,   // deltaProgress time.Duration,
 		30*time.Second,   // deltaResend time.Duration,
 		10*time.Second,   // deltaRound time.Duration,
@@ -149,14 +154,16 @@ func BuildMedianOCR2Config(workerNodes []*client.ChainlinkK8sClient) (*contracts
 		transmitterAddresses = append(transmitterAddresses, common.HexToAddress(string(account)))
 	}
 
+	onchainConfig, err := testhelpers.GenerateDefaultOCR2OnchainConfig(ocrOffchainOptions.MinimumAnswer, ocrOffchainOptions.MaximumAnswer)
+
 	return &contracts.OCRv2Config{
 		Signers:               signerAddresses,
 		Transmitters:          transmitterAddresses,
 		F:                     f_,
 		OnchainConfig:         onchainConfig,
 		OffchainConfigVersion: offchainConfigVersion,
 		OffchainConfig:        []byte(fmt.Sprintf("0x%s", offchainConfig)),
-	}, nil
+	}, err
 }
 
 // GetOracleIdentities retrieves all chainlink nodes' OCR2 config identities with defaul key index
@@ -257,7 +264,6 @@ func CreateOCRv2Jobs(
 	bootstrapNode *client.ChainlinkK8sClient,
 	workerChainlinkNodes []*client.ChainlinkK8sClient,
 	mockserver *ctfClient.MockserverClient,
-	mockServerPath string, // Path on the mock server for the Chainlink nodes to query
 	mockServerValue int, // Value to get from the mock server when querying the path
 	chainId uint64, // EVM chain ID
 	forwardingAllowed bool,
@@ -268,28 +274,36 @@ func CreateOCRv2Jobs(
 		return err
 	}
 	p2pV2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PIds.Data[0].Attributes.PeerID, bootstrapNode.InternalIP(), 6690)
-	// Set the value for the jobs to report on
-	err = mockserver.SetValuePath(mockServerPath, mockServerValue)
-	if err != nil {
-		return err
-	}
+	mockJuelsPath := "ocr2/juelsPerFeeCoinSource"
 	// Set the juelsPerFeeCoinSource config value
-	err = mockserver.SetValuePath(fmt.Sprintf("%s/juelsPerFeeCoinSource", mockServerPath), mockServerValue)
+	err = mockserver.SetValuePath(mockJuelsPath, mockServerValue)
 	if err != nil {
 		return err
 	}
 
+	// Create the juels bridge for each node only once
+	juelsBridge := &client.BridgeTypeAttributes{
+		Name: "juels",
+		URL:  fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, mockJuelsPath),
+	}
+	for _, chainlinkNode := range workerChainlinkNodes {
+		err = chainlinkNode.MustCreateBridge(juelsBridge)
+		if err != nil {
+			return fmt.Errorf("failed creating bridge %s on CL node : %w", juelsBridge.Name, err)
+		}
+	}
+
 	for _, ocrInstance := range ocrInstances {
 		bootstrapSpec := &client.OCR2TaskJobSpec{
-			Name:    "ocr2 bootstrap node",
+			Name:    fmt.Sprintf("ocr2-bootstrap-%s", ocrInstance.Address()),
 			JobType: "bootstrap",
 			OCR2OracleSpec: job.OCR2OracleSpec{
 				ContractID: ocrInstance.Address(),
 				Relay:      "evm",
 				RelayConfig: map[string]interface{}{
 					"chainID": chainId,
 				},
-				MonitoringEndpoint:                null.StringFrom(fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, mockServerPath)),
+				MonitoringEndpoint:                null.StringFrom(fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, "ocr2")),
 				ContractConfigTrackerPollInterval: *models.NewInterval(15 * time.Second),
 			},
 		}
@@ -309,25 +323,22 @@ func CreateOCRv2Jobs(
 			}
 			nodeOCRKeyId := nodeOCRKeys.Data[0].ID
 
-			bta := &client.BridgeTypeAttributes{
-				Name: mockServerPath,
-				URL:  fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, mockServerPath),
+			nodeContractPairID, err := BuildOCR2NodeContractPairID(chainlinkNode, ocrInstance)
+			if err != nil {
+				return err
 			}
-			juelsBridge := &client.BridgeTypeAttributes{
-				Name: "juels",
-				URL:  fmt.Sprintf("%s/%s/juelsPerFeeCoinSource", mockserver.Config.ClusterURL, mockServerPath),
+			bta := &client.BridgeTypeAttributes{
+				Name: nodeContractPairID,
+				URL:  fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, strings.TrimPrefix(nodeContractPairID, "/")),
 			}
+
 			err = chainlinkNode.MustCreateBridge(bta)
 			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
-			}
-			err = chainlinkNode.MustCreateBridge(juelsBridge)
-			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
+				return fmt.Errorf("failed creating bridge %s on CL node: %w", bta.Name, err)
 			}
 
 			ocrSpec := &client.OCR2TaskJobSpec{
-				Name:              "ocr2",
+				Name:              fmt.Sprintf("ocr2-%s", uuid.NewString()),
 				JobType:           "offchainreporting2",
 				MaxTaskDuration:   "1m",
 				ObservationSource: client.ObservationSourceSpecBridge(bta),
@@ -379,3 +390,90 @@ func StartNewOCR2Round(
 	}
 	return nil
 }
+
+// SetOCR2AdapterResponse sets a single adapter response that correlates with an ocr contract and a chainlink node
+// used for OCR2 tests
+func SetOCR2AdapterResponse(
+	response int,
+	ocrInstance contracts.OffchainAggregatorV2,
+	chainlinkNode *client.ChainlinkK8sClient,
+	mockserver *ctfClient.MockserverClient,
+) error {
+	nodeContractPairID, err := BuildOCR2NodeContractPairID(chainlinkNode, ocrInstance)
+	if err != nil {
+		return err
+	}
+	path := fmt.Sprintf("/%s", nodeContractPairID)
+	err = mockserver.SetValuePath(path, response)
+	if err != nil {
+		return fmt.Errorf("setting mockserver value path failed: %w", err)
+	}
+	return nil
+}
+
+// SetOCR2AllAdapterResponsesToTheSameValue sets the mock responses in mockserver that are read by chainlink nodes
+// to simulate different adapters. This sets all adapter responses for each node and contract to the same response
+// used for OCR2 tests
+func SetOCR2AllAdapterResponsesToTheSameValue(
+	response int,
+	ocrInstances []contracts.OffchainAggregatorV2,
+	chainlinkNodes []*client.ChainlinkK8sClient,
+	mockserver *ctfClient.MockserverClient,
+) error {
+	eg := &errgroup.Group{}
+	for _, o := range ocrInstances {
+		ocrInstance := o
+		for _, n := range chainlinkNodes {
+			node := n
+			eg.Go(func() error {
+				return SetOCR2AdapterResponse(response, ocrInstance, node, mockserver)
+			})
+		}
+	}
+	return eg.Wait()
+}
+
+// SetOCR2AllAdapterResponsesToDifferentValues sets the mock responses in mockserver that are read by chainlink nodes
+// to simulate different adapters. This sets all adapter responses for each node and contract to different responses
+// used for OCR2 tests
+func SetOCR2AllAdapterResponsesToDifferentValues(
+	responses []int,
+	ocrInstances []contracts.OffchainAggregatorV2,
+	chainlinkNodes []*client.ChainlinkK8sClient,
+	mockserver *ctfClient.MockserverClient,
+) error {
+	if len(responses) != len(ocrInstances)*len(chainlinkNodes) {
+		return fmt.Errorf(
+			"amount of responses %d should be equal to the amount of OCR instances %d times the amount of Chainlink nodes %d",
+			len(responses), len(ocrInstances), len(chainlinkNodes),
+		)
+	}
+	eg := &errgroup.Group{}
+	for _, o := range ocrInstances {
+		ocrInstance := o
+		for ni := 1; ni < len(chainlinkNodes); ni++ {
+			nodeIndex := ni
+			eg.Go(func() error {
+				return SetOCR2AdapterResponse(responses[nodeIndex-1], ocrInstance, chainlinkNodes[nodeIndex], mockserver)
+			})
+		}
+	}
+	return eg.Wait()
+}
+
+// BuildOCR2NodeContractPairID builds a UUID based on a related pair of a Chainlink node and OCRv2 contract
+func BuildOCR2NodeContractPairID(node *client.ChainlinkK8sClient, ocrInstance contracts.OffchainAggregatorV2) (string, error) {
+	if node == nil {
+		return "", fmt.Errorf("chainlink node is nil")
+	}
+	if ocrInstance == nil {
+		return "", fmt.Errorf("OCR Instance is nil")
+	}
+	nodeAddress, err := node.PrimaryEthAddress()
+	if err != nil {
+		return "", fmt.Errorf("getting chainlink node's primary ETH address failed: %w", err)
+	}
+	shortNodeAddr := nodeAddress[2:12]
+	shortOCRAddr := ocrInstance.Address()[2:12]
+	return strings.ToLower(fmt.Sprintf("node_%s_contract_%s", shortNodeAddr, shortOCRAddr)), nil
+}

integration-tests/actions/ocr2_helpers_local.go

@@ -20,12 +20,13 @@ import (
 
 	"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
 
-	"github.com/smartcontractkit/chainlink/integration-tests/client"
-	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/testhelpers"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
+
+	"github.com/smartcontractkit/chainlink/integration-tests/client"
+	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 )
 
 func CreateOCRv2JobsLocal(
@@ -95,11 +96,11 @@ func CreateOCRv2JobsLocal(
 			}
 			err = chainlinkNode.MustCreateBridge(bta)
 			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
+				return fmt.Errorf("creating bridge on CL node failed: %w", err)
 			}
 			err = chainlinkNode.MustCreateBridge(juelsBridge)
 			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
+				return fmt.Errorf("creating bridge on CL node failed: %w", err)
 			}
 
 			ocrSpec := &client.OCR2TaskJobSpec{

integration-tests/actions/ocr_helpers.go

@@ -236,7 +236,7 @@ func CreateOCRJobs(
 			}
 			err = node.MustCreateBridge(bta)
 			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
+				return fmt.Errorf("creating bridge on CL node failed: %w", err)
 			}
 
 			bootstrapPeers := []*client.ChainlinkClient{bootstrapNode.ChainlinkClient}

integration-tests/actions/ocr_helpers_local.go

@@ -196,7 +196,7 @@ func CreateOCRJobsLocal(
 			}
 			err = node.MustCreateBridge(bta)
 			if err != nil {
-				return fmt.Errorf("creating bridge job have failed: %w", err)
+				return fmt.Errorf("creating bridge on CL node failed: %w", err)
 			}
 
 			bootstrapPeers := []*client.ChainlinkClient{bootstrapNode}

integration-tests/client/chainlink.go

@@ -267,6 +267,7 @@ func (c *ChainlinkClient) DeleteSpec(id string) (*http.Response, error) {
 // MustCreateBridge creates a bridge on the Chainlink node based on the provided attributes and returns error if
 // the request is unsuccessful
 func (c *ChainlinkClient) MustCreateBridge(bta *BridgeTypeAttributes) error {
+	c.l.Debug().Str(NodeURL, c.Config.URL).Str("Name", bta.Name).Msg("Creating Bridge")
 	resp, err := c.CreateBridge(bta)
 	if err != nil {
 		return err
@@ -275,7 +276,7 @@ func (c *ChainlinkClient) MustCreateBridge(bta *BridgeTypeAttributes) error {
 }
 
 func (c *ChainlinkClient) CreateBridge(bta *BridgeTypeAttributes) (*http.Response, error) {
-	c.l.Info().Str(NodeURL, c.Config.URL).Str("Name", bta.Name).Msg("Creating Bridge")
+	c.l.Debug().Str(NodeURL, c.Config.URL).Str("Name", bta.Name).Msg("Creating Bridge")
 	resp, err := c.APIClient.R().
 		SetBody(bta).
 		Post("/v2/bridge_types")
@@ -288,7 +289,7 @@ func (c *ChainlinkClient) CreateBridge(bta *BridgeTypeAttributes) (*http.Respons
 // ReadBridge reads a bridge from the Chainlink node based on the provided name
 func (c *ChainlinkClient) ReadBridge(name string) (*BridgeType, *http.Response, error) {
 	bt := BridgeType{}
-	c.l.Info().Str(NodeURL, c.Config.URL).Str("Name", name).Msg("Reading Bridge")
+	c.l.Debug().Str(NodeURL, c.Config.URL).Str("Name", name).Msg("Reading Bridge")
 	resp, err := c.APIClient.R().
 		SetPathParams(map[string]string{
 			"name": name,
@@ -304,7 +305,7 @@ func (c *ChainlinkClient) ReadBridge(name string) (*BridgeType, *http.Response,
 // ReadBridges reads bridges from the Chainlink node
 func (c *ChainlinkClient) ReadBridges() (*Bridges, *resty.Response, error) {
 	result := &Bridges{}
-	c.l.Info().Str(NodeURL, c.Config.URL).Msg("Getting all bridges")
+	c.l.Debug().Str(NodeURL, c.Config.URL).Msg("Getting all bridges")
 	resp, err := c.APIClient.R().
 		SetResult(&result).
 		Get("/v2/bridge_types")
@@ -316,7 +317,7 @@ func (c *ChainlinkClient) ReadBridges() (*Bridges, *resty.Response, error) {
 
 // DeleteBridge deletes a bridge on the Chainlink node based on the provided name
 func (c *ChainlinkClient) DeleteBridge(name string) (*http.Response, error) {
-	c.l.Info().Str(NodeURL, c.Config.URL).Str("Name", name).Msg("Deleting Bridge")
+	c.l.Debug().Str(NodeURL, c.Config.URL).Str("Name", name).Msg("Deleting Bridge")
 	resp, err := c.APIClient.R().
 		SetPathParams(map[string]string{
 			"name": name,