From 530225a1918556cbee65bbe2e5aff6bfaeb05abd Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Tue, 21 Nov 2023 10:06:19 -0500 Subject: [PATCH] [TT-524] [TT-729] More Products and Reporting for Live Tests (#11327) * More Specific Tests * Disable sepolia * Specify automation test * Don't give up * Reduce default funding * Fix test name * Messing with jq * Change all OCR config * Fiddling with jq more * Little better reports * I think I got the formatting now * Ensure prerequisites * Debugging * Remove Sepolia * Formatting * Remove if (we'll make it pretty later) * Strip out quotes * Fix GHA label * Remove debug * More accurate colors * Ease the debugging process * More debugging * Fix needs check * Check fail * Check fail better * Learned some tricks * Successes * Extra paren * Fix emojis * Purty * Remove debug and mocks * Cleanup --- .github/workflows/live-testnet-tests.yml | 205 ++++++++++++++--------- integration-tests/soak/ocr_test.go | 11 +- 2 files changed, 128 insertions(+), 88 deletions(-) diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index 7eb1669b35f..f174e8847bd 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -11,12 +11,16 @@ env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com MOD_CACHE_VERSION: 2 + CHAINLINK_NODE_FUNDING: .1 CHAINLINK_COMMIT_SHA: ${{ github.sha }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_LOG_LEVEL: debug EVM_KEYS: ${{ secrets.QA_EVM_KEYS }} + SEPOLIA_URLS: ${{ secrets.QA_SEPOLIA_URLS }} + SEPOLIA_HTTP_URLS: ${{ secrets.QA_SEPOLIA_HTTP_URLS }} + OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }} OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }} @@ -55,61 +59,66 @@ jobs: AWS_REGION: ${{ secrets.QA_AWS_REGION }} AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - sepolia-smoke-tests: - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: [build-chainlink] - env: - SELECTED_NETWORKS: SEPOLIA - strategy: - fail-fast: false - matrix: - product: [ocr, automation] - name: Sepolia ${{ matrix.product }} Tests - runs-on: ubuntu-latest - steps: - - name: Checkout the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 - env: - PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-sepolia - PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} - dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} - artifacts_location: ./integration-tests/smoke/logs - publish_check_name: Seplia ${{ matrix.product }} Smoke Test Results - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 - with: - basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} - this-job-name: Sepolia ${{ matrix.product }} Tests - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true + # TODO: Re-enable when we have secrets properly configured + # sepolia-smoke-tests: + # environment: integration + # permissions: + # checks: write + # pull-requests: write + # id-token: write + # contents: read + # needs: [build-chainlink] + # env: + # SELECTED_NETWORKS: SEPOLIA + # strategy: + # max-parallel: 1 + # fail-fast: false + # matrix: + # include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations + # - product: OCR + # test: TestOCRBasic + # - product: Automation + # test: TestAutomationBasic/registry_2_0 + # name: Sepolia ${{ matrix.product }} Tests + # runs-on: ubuntu-latest + # steps: + # - name: Checkout the repo + # uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + # with: + # ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + # - name: Run Tests + # uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@e865e376b8c2d594028c8d645dd6c47169b72974 # v2.2.16 + # env: + # PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + # PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-sepolia + # PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + # with: + # test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt + # test_download_vendor_packages_command: cd ./integration-tests && go mod download + # cl_repo: ${{ env.CHAINLINK_IMAGE }} + # cl_image_tag: ${{ github.sha }} + # aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + # dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} + # dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} + # artifacts_location: ./integration-tests/smoke/logs + # publish_check_name: Seplia ${{ matrix.product }} Smoke Test Results + # token: ${{ secrets.GITHUB_TOKEN }} + # go_mod_path: ./integration-tests/go.mod + # cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + # cache_restore_only: "true" + # QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + # QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + # QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + # - name: Collect Metrics + # if: always() + # id: collect-gha-metrics + # uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + # with: + # basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + # hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + # this-job-name: Sepolia ${{ matrix.product }} Tests + # test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' + # continue-on-error: true optimism-goerli-smoke-tests: environment: integration @@ -123,8 +132,13 @@ jobs: SELECTED_NETWORKS: OPTIMISM_GOERLI strategy: fail-fast: false + max-parallel: 1 matrix: - product: [ocr, automation] + include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations + - product: OCR + test: TestOCRBasic + - product: Automation + test: TestAutomationBasic/registry_2_0 name: Optimism Goerli ${{ matrix.product }} Tests runs-on: ubuntu-latest steps: @@ -139,7 +153,7 @@ jobs: PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-optimism-goerli PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt + test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ github.sha }} @@ -177,9 +191,14 @@ jobs: env: SELECTED_NETWORKS: ARBITRUM_GOERLI strategy: + max-parallel: 1 fail-fast: false matrix: - product: [ocr, automation] + include: # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#example-adding-configurations + - product: OCR + test: TestOCRBasic + - product: Automation + test: TestAutomationBasic/registry_2_0 name: Arbitrum Goerli ${{ matrix.product }} Tests runs-on: ubuntu-latest steps: @@ -194,7 +213,7 @@ jobs: PYROSCOPE_ENVIRONMENT: ci-smoke-${{ matrix.product }}-arbitrum-goerli PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/${{ matrix.product }}_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt + test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 -run ${{ matrix.test }} ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ github.sha }} @@ -233,10 +252,10 @@ jobs: id-token: write contents: read runs-on: ubuntu-latest - needs: [sepolia-smoke-tests, optimism-goerli-smoke-tests, arbitrum-goerli-smoke-tests] + needs: [optimism-goerli-smoke-tests, arbitrum-goerli-smoke-tests] steps: - name: Debug Result - run: echo ${{needs.*.result}} + run: echo ${{ join(needs.*.result, ',') }} - name: Main Slack Notification uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 id: slack @@ -246,13 +265,13 @@ jobs: { "attachments": [ { - "color": "${{ needs.*.result == 'success' && '#2E7D32' || '#C62828' }}", + "color": "${{ contains(join(needs.*.result, ','), 'failure') && '#C62828' || '#2E7D32' }}", "blocks": [ { "type": "header", "text": { "type": "plain_text", - "text": "Live Smoke Test Results ${{ needs.*.result == 'success' && ':white_check_mark:' || ':x:'}}", + "text": "Live Smoke Test Results ${{ contains(join(needs.*.result, ','), 'failure') && ':x:' || ':white_check_mark:'}}", "emoji": true } }, @@ -274,7 +293,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} testnet-smoke-tests-results: - name: Post Test Results + name: Post Test Results for ${{ matrix.network }} if: ${{ always() && needs.*.result != 'skipped' && needs.*.result != 'cancelled' }} environment: integration permissions: @@ -287,23 +306,53 @@ jobs: strategy: fail-fast: false matrix: - testnet: [sepolia, optimism-goerli, arbitrum-goerli] + network: [Optimism Goerli, Arbitrum Goerli] steps: - name: Get Results id: test-results run: | + # I feel like there's some clever, fully jq way to do this, but I ain't got the motivation to figure it out echo "Querying test results" - echo "status=$(curl \ + PARSED_RESULTS=$(curl \ -H "Authorization: Bearer ${{ github.token }}" \ 'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \ - | jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet }}-smoke-tests").steps[] | select(.name == "Run Tests").conclusion')" >> $GITHUB_OUTPUT + | jq -r --arg pattern "${{ matrix.network }} (?\\w+) Tests" '.jobs[] + | select(.name | test($pattern)) as $job + | $job.steps[] + | select(.name == "Run Tests") + | { conclusion: (if .conclusion == "success" then ":white_check_mark:" else ":x:" end), product: ("*" + ($job.name | capture($pattern).product) + "*") }') - echo "status=$(curl \ - -H "Authorization: Bearer ${{ github.token }}" \ - 'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \ - | jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet }}-smoke-tests"").steps[] | select(.name == "Run Tests").conclusion')" - echo "thread_ts=${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}" + echo "Parsed Results:" + echo $PARSED_RESULTS + + ALL_SUCCESS=true + for row in $(echo "$PARSED_RESULTS" | jq -s | jq -r '.[] | select(.conclusion != ":white_check_mark:")'); do + success=false + break + done + + echo all_success=$ALL_SUCCESS >> $GITHUB_OUTPUT + + FORMATTED_RESULTS=$(echo $PARSED_RESULTS | jq -s '[.[] + | { + conclusion: .conclusion, + product: .product + } + ] + | map("{\"type\": \"section\", \"text\": {\"type\": \"mrkdwn\", \"text\": \"\(.product): \(.conclusion)\"}}") + | join(",")') + + echo "Formatted Results:" + echo $FORMATTED_RESULTS + + # Cleans out backslashes and quotes from jq + CLEAN_RESULTS=$(echo "$FORMATTED_RESULTS" | sed 's/\\\"/"/g' | sed 's/^"//;s/"$//') + + echo "Clean Results" + echo $CLEAN_RESULTS + + echo results=$CLEAN_RESULTS >> $GITHUB_OUTPUT - name: Test Details uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0 @@ -314,26 +363,20 @@ jobs: "thread_ts": "${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}", "attachments": [ { - "color": "${{ steps.test-results.outputs.status == 'success' && '#2E7D32' || '#C62828' }}", + "color": "${{ steps.test-results.outputs.all_success && '#2E7D32' || '#C62828' }}", "blocks": [ { "type": "header", "text": { "type": "plain_text", - "text": "${{ matrix.testnet }} Smoke Test Results ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}", + "text": "${{ matrix.network }} ${{ steps.test-results.outputs.all_success && ':white_check_mark:' || ':x: Notifying <@U01Q4N37KFG>'}}", "emoji": true } }, { "type": "divider" }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "OCR ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}" - } - } + ${{ steps.test-results.outputs.results }} ] } ] diff --git a/integration-tests/soak/ocr_test.go b/integration-tests/soak/ocr_test.go index 4f42ff803e5..4d7971d678e 100644 --- a/integration-tests/soak/ocr_test.go +++ b/integration-tests/soak/ocr_test.go @@ -1,16 +1,13 @@ package soak import ( - "fmt" "testing" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-testing-framework/logging" - "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/config" "github.com/smartcontractkit/chainlink/integration-tests/testsetups" ) @@ -19,10 +16,10 @@ func TestOCRSoak(t *testing.T) { // Use this variable to pass in any custom EVM specific TOML values to your Chainlink nodes customNetworkTOML := `` // Uncomment below for debugging TOML issues on the node - network := networks.MustGetSelectedNetworksFromEnv()[0] - fmt.Println("Using Chainlink TOML\n---------------------") - fmt.Println(networks.AddNetworkDetailedConfig(config.BaseOCR1Config, customNetworkTOML, network)) - fmt.Println("---------------------") + // network := networks.MustGetSelectedNetworksFromEnv()[0] + // fmt.Println("Using Chainlink TOML\n---------------------") + // fmt.Println(networks.AddNetworkDetailedConfig(config.BaseOCR1Config, customNetworkTOML, network)) + // fmt.Println("---------------------") ocrSoakTest, err := testsetups.NewOCRSoakTest(t, false) require.NoError(t, err, "Error creating soak test")