CCIP-1903 : offchain compatibility tests (#639)

## Motivation Offchain Version compatibility validation ## Solution Enables offchain version compatibility test github workflow. The workflow works in following steps - 1. Fetches the last release in the repo. 2. Fetches the test image related to last release and deploys ccip environment with the test image pertaining to last release. It ensures that contracts and offchain nodes are deployed with last release version. 3. It then upgrades few of the nodes with current commit sha, keeping rest of the node with last release version. 4. With this mixed version node it runs a short duration load test to ensure that - off chain version is always compatible with contracts deployed in previous release - a DON having nodes with two different versions are working as expected.
smartcontractkit · Apr 13, 2024 · 8298c0d · 8298c0d
1 parent f06d3f3
commit 8298c0d
Show file tree

Hide file tree

Showing 22 changed files with 833 additions and 199 deletions.
diff --git a/.github/actions/setup-create-base64-config-ccip/action.yml b/.github/actions/setup-create-base64-config-ccip/action.yml
@@ -4,6 +4,8 @@ description: A composite action that creates a base64-encoded config to be used
 inputs:
   runId:
     description: The run id
+  existingNamespace:
+    description: If test needs to run against already deployed namespace
   testLogCollect:
     description: Whether to always collect logs, even for passing tests
     default: "false"
@@ -14,6 +16,11 @@ inputs:
     default: "public.ecr.aws/chainlink/chainlink"
   chainlinkVersion:
     description: The git commit sha to use for the image tag
+  upgradeImage:
+    description: The chainlink image to upgrade to
+    default: ""
+  upgradeVersion:
+    description: The git commit sha to use for the image tag
   lokiEndpoint:
     description: Loki push endpoint
   lokiTenantId:
@@ -37,9 +44,12 @@ runs:
       id: base64-config-override
       env:
         RUN_ID: ${{ inputs.runId }}
+        EXISTING_NAMESPACE: ${{ inputs.existingNamespace }}
         TEST_LOG_COLLECT: ${{ inputs.testLogCollect }}
         CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }}
         CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }}
+        UPGRADE_IMAGE: ${{ inputs.upgradeImage }}
+        UPGRADE_VERSION: ${{ inputs.upgradeVersion }}
         LOKI_ENDPOINT: ${{ inputs.lokiEndpoint }}
         LOKI_TENANT_ID: ${{ inputs.lokiTenantId }}
         LOKI_BASIC_AUTH: ${{ inputs.lokiBasicAuth }}
@@ -116,12 +126,17 @@ runs:
         cat << EOF > config.toml
         [CCIP]
         [CCIP.Env]
+        EnvToConnect="$EXISTING_NAMESPACE"
         [CCIP.Env.NewCLCluster]
         [CCIP.Env.NewCLCluster.Common]
         [CCIP.Env.NewCLCluster.Common.ChainlinkImage]
         image="$CHAINLINK_IMAGE"
         version="$CHAINLINK_VERSION"
 
+        [CCIP.Env.NewCLCluster.Common.ChainlinkUpgradeImage]
+        image="$UPGRADE_IMAGE"
+        version="$UPGRADE_VERSION"
+        
         $custom_nodes_toml
 
         [CCIP.Env.Logging]
@@ -139,6 +154,13 @@ runs:
         [CCIP.Env.Logging.Grafana]
         base_url="$GRAFANA_URL"
         dashboard_url="$GRAFANA_DASHBOARD_URL"
+        
+        [CCIP.Groups.load]
+        TestRunName = '$EXISTING_NAMESPACE'
+        
+        [CCIP.Groups.smoke]
+        TestRunName = '$EXISTING_NAMESPACE'
+        
         EOF
 
         BASE64_CCIP_SECRETS_CONFIG=$(cat config.toml | base64 -w 0)

diff --git a/.../workflows/client-compatibility-tests.yml → ...flows/ccip-client-compatibility-tests.yml b/.../workflows/client-compatibility-tests.yml → ...flows/ccip-client-compatibility-tests.yml
diff --git a/.github/workflows/ccip-load-tests.yml b/.github/workflows/ccip-load-tests.yml
@@ -91,7 +91,6 @@ jobs:
           QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
 
   ccip-load-test:
-    name: CCIP Load Test
     environment: integration
     needs: [ build-chainlink, build-test-image ]
     if: ${{ always() && !contains(needs.*.result, 'failure') }}
@@ -121,6 +120,7 @@ jobs:
             config_path: ./integration-tests/ccip-tests/testconfig/tomls/load-with-arm-curse-uncurse.toml
             os: ubuntu-latest
     runs-on: ${{ matrix.type.os }}
+    name: CCIP ${{ matrix.type.name }}
     steps:
       - name: Collect Metrics
         id: collect-gha-metrics
@@ -197,98 +197,3 @@ jobs:
           cache_restore_only: "true"
           should_cleanup: "true"
 
-
-  ccip-version-compatibility-test:
-    name: CCIP Load With Version Compatibility Test
-    environment: integration
-    runs-on: ubuntu-latest
-    needs: [ build-chainlink, build-test-image ]
-    if: ${{ always() && !contains(needs.*.result, 'failure') }}
-    permissions:
-      issues: read
-      checks: write
-      pull-requests: write
-      id-token: write
-      contents: read
-    env:
-      SLACK_USER: ${{ inputs.slackMemberID }}
-      SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }}
-      SLACK_CHANNEL: ${{ secrets.QA_SLACK_CHANNEL }}
-      TEST_LOG_LEVEL: info
-      REF_NAME: ${{ github.head_ref || github.ref_name }}
-
-    steps:
-      - name: Collect Metrics
-        id: collect-gha-metrics
-        uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0
-        with:
-          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
-          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
-          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
-          this-job-name: CCIP Load Test
-        continue-on-error: true
-      - name: Checkout the repo
-        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
-        with:
-          ref: ${{ env.REF_NAME }}
-      - name: Sets env vars
-        id : set_env_vars
-        shell: bash
-        run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-            BASE64_CCIP_CONFIG_OVERRIDE=$(jq -r '.inputs.base64_test_input' $GITHUB_EVENT_PATH)
-            echo ::add-mask::$BASE64_CCIP_CONFIG_OVERRIDE
-            echo "BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV
-            echo "TEST_BASE64_CCIP_CONFIG_OVERRIDE=$BASE64_CCIP_CONFIG_OVERRIDE" >> $GITHUB_ENV
-          fi
-          echo "UPGRADE_VERSION=${{ env.CHAINLINK_VERSION }}" >> $GITHUB_ENV
-          echo "UPGRADE_IMAGE=${{ env.CHAINLINK_IMAGE }}" >> $GITHUB_ENV
-          untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases | jq -r --arg SUFFIX "release" '.[] | select(.tag_name | endswith("release"))  | .tag_name' | sort -V | tail -n 1)
-          last_release="${untrimmed_ver:1}"
-          echo "last_release=${last_release}" >> $GITHUB_OUTPUT
-      - name: step summary
-        shell: bash
-        run: |
-          echo "### chainlink upgrade image used for this test run :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.CHAINLINK_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.INPUT_CHAINLINK_TEST_VERSION }}\`" >> $GITHUB_STEP_SUMMARY
-          echo "### testing upgrade against release :link:" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ steps.set_env_vars.outputs.last_release }}\`" >>$GITHUB_STEP_SUMMARY
-      - name: Prepare Base64 TOML override for CCIP secrets
-        uses: ./.github/actions/setup-create-base64-config-ccip
-        with:
-          runId: ${{ github.run_id }}
-          testLogCollect: ${{ vars.TEST_LOG_COLLECT }}
-          chainlinkImage: ${{ env.CHAINLINK_IMAGE }}
-          chainlinkVersion: ${{ github.sha }}
-          lokiEndpoint: ${{ secrets.LOKI_URL }}
-          lokiTenantId: ${{ vars.LOKI_TENANT_ID }}
-          logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
-          grafanaUrl: ${{ vars.GRAFANA_URL }}
-          grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests"
-      - name: Run Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@ea889b3133bd7f16ab19ba4ba130de5d9162c669 # v2.3.4
-        env:
-          TEST_SUITE: load
-          TEST_ARGS: -test.timeout 900h
-          DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable
-          RR_MEM: 8Gi
-          RR_CPU: 4
-          TEST_TRIGGERED_BY: ccip-load-upgrade-test-ci
-        with:
-          test_command_to_run: cd ./integration-tests/ccip-tests && go test -v -timeout 70m -count=1 -json -run ^TestLoadCCIPWithUpgradeNodeVersion$ ./load 2>&1 | tee /tmp/gotest.log | gotestfmt
-          test_download_vendor_packages_command: cd ./integration-tests && go mod download
-          cl_repo: ${{ env.AWS_ECR_REPO_PUBLIC_REGISTRY }}/w0i8p0z9/chainlink-ccip # releases are published to public registry
-          cl_image_tag:  ${{ steps.set_env_vars.outputs.last_release }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-          go_mod_path: ./integration-tests/go.mod
-          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
-          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
-          triggered_by: ${{ env.TEST_TRIGGERED_BY }}
-          artifacts_location: ./integration-tests/load/logs/payload_ccip.json
-          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
-          cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "true"
-          should_cleanup: "true"