Automation Benchmark Test #500
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automation Benchmark Test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| testType: | |
| description: Type of test to run (benchmark, soak) | |
| required: true | |
| default: benchmark | |
| type: string | |
| test_config_override_path: | |
| description: Path to a test config file used to override the default test config | |
| required: false | |
| type: string | |
| test_secrets_override_key: | |
| description: Key to run tests with custom test secrets | |
| required: false | |
| type: string | |
| slackMemberID: | |
| description: Notifies test results (Not your @) | |
| required: true | |
| default: U02Q14G80TY | |
| type: string | |
| jobs: | |
| automation_benchmark: | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| name: Automation Benchmark Test | |
| runs-on: ubuntu22.04-16cores-64GB | |
| env: | |
| SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} | |
| SLACK_CHANNEL: C03KJ5S7KEK | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ env.REF_NAME }} | |
| - name: Get Slack config and mask base64 config | |
| run: | | |
| SLACK_USER=$(jq -r '.inputs.slackMemberID' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$SLACK_USER | |
| echo SLACK_USER=$SLACK_USER >> $GITHUB_ENV | |
| - name: Show test config override path in summary | |
| if: ${{ inputs.test_config_override_path }} | |
| run: | | |
| echo "### Test config override path" >> $GITHUB_STEP_SUMMARY | |
| echo "[${{ inputs.test_config_override_path }}]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/blob/${{ github.sha }}/${{ inputs.test_config_override_path }})" >> $GITHUB_STEP_SUMMARY | |
| - name: Build Test Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| suites: benchmark chaos reorg load | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@94cb11f4bd545607a2f221c6685052b3abee723d # v2.3.32 | |
| env: | |
| DETACH_RUNNER: true | |
| TEST_SUITE: benchmark | |
| TEST_ARGS: -test.timeout 720h | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} | |
| INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com | |
| TEST_TYPE: ${{ github.event.inputs.testType }} | |
| TEST_TEST_TYPE: ${{ github.event.inputs.testType }} | |
| RR_MEM: 4Gi | |
| TEST_LOG_LEVEL: info | |
| E2E_TEST_CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| E2E_TEST_CHAINLINK_VERSION: ${{ github.sha }} # Default chainlink version if not overrided in the test config | |
| with: | |
| test_command_to_run: cd integration-tests && go test -timeout 30m -v -run ^TestAutomationBenchmark$ ./benchmark -count=1 | |
| test_download_vendor_packages_command: make gomod | |
| test_secrets_override_base64: ${{ secrets[inputs.test_secrets_override_key] }} | |
| test_config_override_path: ${{ inputs.test_config_override_path }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| should_cleanup: false | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
| with: | |
| id: automation-benchmark-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Automation Benchmark Test | |
| continue-on-error: true |