add workflow that verifies compatibility with different versions of e…

…vm nodes (#12332)

* add workflow that verifies compatibility with different versions of evm nodes

* fix lints

* fix worklow definion

* another workflow fix

* another fix

* send slack notifications only if go-ethereum dep was modified

* bump go-ethereum to v1.13.9 to test the workflow

* remove hardcoded commit hashes...

* fix comparison

* bump

* use correctly git diff

* fetch develop before comparing

* remove unnecesary outputs declaration

* add outputs debug

* remove step id

* use explicit outputs

* execute workflow also on dispatch if go-ethereum version did not change

* fix script output

* fix quoting of custom images in TOML, echo test compatibility matrix

* don't use pyroscope in compatibility tests

* use newer CTF

* latest ctf, couple of fixes

* bump CTF, run compatibility tests only for geth

* all smoke tests will now use TOML config

* set ethereum version to empty

* adjust TOML created in CI to latest CTF changes

* run also ocr2 test

* adjust regex capture for Slack message

* try to group compatibilty tests results by product

* group by product, not full matrix

* run even more product tests to check compatibility

* fix parameter field name

* log details before starting private ethereum network

* fix how TOML is build for custom eth client image

* fix typo in argument name

* fix versions + always notify slack

* add a ContractBackend wrapper that is backward-compatible with go-ethereum < 1.11.0

* add comment to contract calelr

* try running compatibility fix on broken version

* just trigger

* try to build image in CI

* try to checkout old commit

* fix contract caller, and let's see if it fails or not

* remove code used for testing, fix slack reporting

* fix a situation, when we lose transaction timeout setting for networks that are not overwritten

* trigger compatibility tests for PR bumping go-ethereum to 1.13.14

* add gas limit for Fiji, fix a situation when new networks were ignored

* fix lints

* remove hardcoded github sha

* add changeset

* separate method for restarting CL cluster

* update go.mod

* go.mod

* lints

* do not bump go-eth

* use latest CTF v1.27.0

* dump down go-ethereum, fix some comments and descriptions

* add helper that exposes legacy methods to use them in test wrapper and rpc client

* remove stale comment

.changeset/moody-ligers-walk.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Add new pipeline for testing EVM node compatibility on go-ethereum dependency bump

.github/actions/notify-slack-jobs-result/action.yml

@@ -36,9 +36,10 @@ runs:
         # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out
         echo "Querying test results at https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs"
 
+        # we can get a maximum of 100 jobs per page, after that we need to start using pagination
         PARSED_RESULTS=$(curl \
         -H "Authorization: Bearer ${{ inputs.github_token }}" \
-        'https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs' \
+        'https://api.github.com/repos/${{inputs.github_repository}}/actions/runs/${{ inputs.workflow_run_id }}/jobs?per_page=100' \
         | jq -r --arg pattern "${{ inputs.github_job_name_regex }}" '.jobs[]
         | select(.name | test($pattern)) as $job
         | $job.steps[]
@@ -59,9 +60,9 @@ runs:
 
         echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT
 
-        FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] 
-        | { 
-          conclusion: .conclusion, 
+        FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[]
+        | {
+          conclusion: .conclusion,
           cap: .cap,
           html_url: .html_url
           }

.github/actions/setup-create-base64-config/action.yml

@@ -35,6 +35,10 @@ inputs:
     description: Grafana URL
   grafanaDashboardUrl:
     description: Grafana dashboard URL
+  ethExecutionClient:
+    description: Ethereum execution client to use (geth, besu, nethermind or erigon)
+  customEthClientDockerImage:
+    description: custom docker image to use for eth client (e.g. hyperledger/besu:21.10.0)
 
 runs:
   using: composite
@@ -58,6 +62,8 @@ runs:
         LOGSTREAM_LOG_TARGETS: ${{ inputs.logstreamLogTargets }}
         GRAFANA_URL: ${{ inputs.grafanaUrl }}
         GRAFANA_DASHBOARD_URL: ${{ inputs.grafanaDashboardUrl }}
+        ETH_EXECUTION_CLIENT: ${{ inputs.ethExecutionClient }}
+        CUSTOM_ETH_CLIENT_DOCKER_IMAGE: ${{ inputs.customEthClientDockerImage }}
       run: |
         echo ::add-mask::$CHAINLINK_IMAGE
         function convert_to_toml_array() {
@@ -88,6 +94,21 @@ runs:
           test_log_collect=false
         fi
 
+        custom_images=""
+        ethereum_version=""
+
+        if [ -n "$CUSTOM_ETH_CLIENT_DOCKER_IMAGE" ]; then
+            ethereum_version="ethereum_version=\"\""
+            custom_images+="[PrivateEthereumNetwork.CustomDockerImages]"
+            custom_images+=$'\n'"execution_layer=\"$CUSTOM_ETH_CLIENT_DOCKER_IMAGE\""
+        fi
+
+        if [ -n "$ETH_EXECUTION_CLIENT" ]; then
+            execution_layer="$ETH_EXECUTION_CLIENT"
+          else
+            execution_layer="geth"
+        fi
+
         cat << EOF > config.toml
         [Network]
         selected_networks=$selected_networks
@@ -118,6 +139,12 @@ runs:
         [Logging.Grafana]
         base_url="$GRAFANA_URL"
         dashboard_url="$GRAFANA_DASHBOARD_URL"
+
+        [PrivateEthereumNetwork]
+        execution_layer="$execution_layer"
+        $ethereum_version
+
+        $custom_images
         EOF
 
         BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)