From 0cbad1b0c8aadd2a8daebdfe63b3c735d473e2b6 Mon Sep 17 00:00:00 2001 From: Lukasz <120112546+lukaszcl@users.noreply.github.com> Date: Wed, 21 Aug 2024 18:40:52 +0200 Subject: [PATCH] TT-1473 Refactor integration-tests.yml workflow to use run-e2e-tests-reusable-workflow.yml (#14134) * wip * fix secrets and test log level * comment eth-smoke-tests-matrix-automation and use reusable workflow instead * comment eth-smoke-tests-matrix-log-poller and use reusable workflow instead * Fix * migrate node upgrade tests * fix test_config_chainlink_upgrade_version * use default workflow name * bump run-tests * comment compare-tests * add name to reusable workflow * Add chainlink-testing-framework/show-test-summary to reusable workflow and run collect metrics step for aggregated result * Move Upload CL Node Coverage Data Artifacts to reusable workflow * Fix * Update cl node coverage artifact prefix * Update reusable workflow to have test results as output * Fix * fix * test * fix * remove old node migration tests * migrate otel traces steps to reusable workflow * uncomment eth-smoke-go-mod-cache * rename some steps and jobs * Use citool from CTF * Fix * Fix * Add cache for citool * Update setup-go GHA * Cache ctf builds * Bump * Fix * Bump * Remove eth-smoke-go-mod-cache as cache is saved in run-tests * Bump * Remove duplicated step * Bump * Bump * trigger build * Clean up after_tests job * Run all PR tests * Fix record test result step * Remove duplicated tests from e2e-tests.yml * Bump GHA * Run ccip tests in reusable workflow * fix go mod * Fix * Update test_env names * Install citool bin intead of checking out ctf repo * Add more ccip tests * chore: Update Go setup in reusable workflow * Update test names * Use ctf from branch * Run all PR tests * fix plugins test * Run core and ccip tests based on changes * Fix * Bump citool * Bump run-tests * Fix collect metrics * Bump ctf * Simplify test ids * uncomment tags * fix * Fix TEST_TYPE * Fix test env vars * Update check-e2e-test-results * Fix check results * test * fix * test * fix * Do not build integration-tests, run only lint * fix test_result artifact * Fix cache * Fix unique workflow id * Fix e2e-tests and skip one test * test when ccip tests skipped * fix --- .github/e2e-tests.yml | 640 +++++---- .github/workflows/integration-tests.yml | 1218 ++--------------- .../run-automation-ondemand-e2e-tests.yml | 68 +- .../run-e2e-tests-reusable-workflow.yml | 305 +++-- .gitignore | 1 - .../ccip-tests/testconfig/global.go | 87 ++ .../citool/cmd/check_tests_cmd.go | 166 --- .../citool/cmd/check_tests_cmd_test.go | 47 - .../citool/cmd/create_test_config_cmd.go | 179 --- .../citool/cmd/csv_export_cmd.go | 96 -- integration-tests/citool/cmd/filter_cmd.go | 165 --- .../citool/cmd/filter_cmd_test.go | 64 - integration-tests/citool/cmd/root_cmd.go | 32 - .../citool/cmd/test_config_cmd.go | 123 -- .../citool/cmd/test_config_cmd_test.go | 75 - integration-tests/citool/cmd/types.go | 26 - integration-tests/citool/main.go | 9 - integration-tests/go.mod | 4 +- 18 files changed, 838 insertions(+), 2467 deletions(-) delete mode 100644 integration-tests/citool/cmd/check_tests_cmd.go delete mode 100644 integration-tests/citool/cmd/check_tests_cmd_test.go delete mode 100644 integration-tests/citool/cmd/create_test_config_cmd.go delete mode 100644 integration-tests/citool/cmd/csv_export_cmd.go delete mode 100644 integration-tests/citool/cmd/filter_cmd.go delete mode 100644 integration-tests/citool/cmd/filter_cmd_test.go delete mode 100644 integration-tests/citool/cmd/root_cmd.go delete mode 100644 integration-tests/citool/cmd/test_config_cmd.go delete mode 100644 integration-tests/citool/cmd/test_config_cmd_test.go delete mode 100644 integration-tests/citool/cmd/types.go delete mode 100644 integration-tests/citool/main.go diff --git a/.github/e2e-tests.yml b/.github/e2e-tests.yml index b2c9f12fcaf..6a46e76f01e 100644 --- a/.github/e2e-tests.yml +++ b/.github/e2e-tests.yml @@ -3,493 +3,472 @@ # Each entry in this file includes the following: # - The GitHub runner (runs_on field) that will execute tests. # - The tests that will be run by the runner. -# - The workflows (e.g., Run PR E2E Tests, Run Nightly E2E Tests) that should trigger these tests. +# - The workflows (e.g., Run PR E2E Tests, Nightly E2E Tests) that should trigger these tests. # runner-test-matrix: # START: OCR tests # Example of 1 runner for all tests in integration-tests/smoke/ocr_test.go - - id: integration-tests/smoke/ocr_test.go:* + - id: smoke/ocr_test.go:* path: integration-tests/smoke/ocr_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json pyroscope_env: ci-smoke-ocr-evm-simulated - # Example of 2 separate runners for the same test file but different tests. Can be used if tests if are too heavy to run on the same runner - - id: integration-tests/smoke/ocr2_test.go:^TestOCRv2Request$ - path: integration-tests/smoke/ocr2_test.go - test_env_type: docker - runs_on: ubuntu-latest - workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests - test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Request$ -test.parallel=1 -timeout 30m -count=1 -json - pyroscope_env: ci-smoke-ocr2-evm-simulated-nightly - - - id: integration-tests/smoke/ocr2_test.go:^TestOCRv2Basic$ - path: integration-tests/smoke/ocr2_test.go - test_env_type: docker - runs_on: ubuntu-latest - workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests - test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -test.run ^TestOCRv2Basic$ -test.parallel=1 -timeout 30m -count=1 -json - pyroscope_env: ci-smoke-ocr2-evm-simulated-nightly - # Example of a configuration for running a single soak test in Kubernetes Remote Runner - - id: integration-tests/soak/ocr_test.go:^TestOCRv1Soak$ + - id: soak/ocr_test.go:^TestOCRv1Soak$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRv2Soak$ + - id: soak/ocr_test.go:^TestOCRv2Soak$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestForwarderOCRv1Soak$ + - id: soak/ocr_test.go:^TestForwarderOCRv1Soak$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv1Soak$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestForwarderOCRv2Soak$ + - id: soak/ocr_test.go:^TestForwarderOCRv2Soak$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestForwarderOCRv2Soak$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ + - id: soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ + - id: soak/ocr_test.go:^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_GasSpike$ + - id: soak/ocr_test.go:^TestOCRSoak_GasSpike$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_GasSpike$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_ChangeBlockGasLimit$ + - id: soak/ocr_test.go:^TestOCRSoak_ChangeBlockGasLimit$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_ChangeBlockGasLimit$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_RPCDownForAllCLNodes$ + - id: soak/ocr_test.go:^TestOCRSoak_RPCDownForAllCLNodes$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForAllCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/soak/ocr_test.go:^TestOCRSoak_RPCDownForHalfCLNodes$ + - id: soak/ocr_test.go:^TestOCRSoak_RPCDownForHalfCLNodes$ path: integration-tests/soak/ocr_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest test_cmd: cd integration-tests/ && go test soak/ocr_test.go -v -test.run ^TestOCRSoak_RPCDownForHalfCLNodes$ -test.parallel=1 -timeout 30m -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: soak + test_env_vars: + TEST_SUITE: soak - - id: integration-tests/smoke/forwarder_ocr_test.go:* + - id: smoke/forwarder_ocr_test.go:* path: integration-tests/smoke/forwarder_ocr_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/forwarder_ocr_test.go -timeout 30m -count=1 -test.parallel=2 -json pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - - id: integration-tests/smoke/forwarders_ocr2_test.go:* + - id: smoke/forwarders_ocr2_test.go:* path: integration-tests/smoke/forwarders_ocr2_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/forwarders_ocr2_test.go -timeout 30m -count=1 -test.parallel=2 -json pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - - id: integration-tests/smoke/ocr2_test.go:* + - id: smoke/ocr2_test.go:* path: integration-tests/smoke/ocr2_test.go test_env_type: docker runs_on: ubuntu22.04-16cores-64GB workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json pyroscope_env: ci-smoke-ocr2-evm-simulated - - id: integration-tests/smoke/ocr2_test.go:*-plugins + - id: smoke/ocr2_test.go:*-plugins path: integration-tests/smoke/ocr2_test.go test_env_type: docker runs_on: ubuntu22.04-16cores-64GB workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/ocr2_test.go -timeout 30m -count=1 -test.parallel=6 -json pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated - test_inputs: - # chainlink_version: '{{ env.GITHUB_SHA_PLUGINS }}' # This is the chainlink version that has the plugins - chainlink_version: develop-plugins + test_env_vars: + E2E_TEST_CHAINLINK_VERSION: '{{ env.GITHUB_SHA_PLUGINS }}' # This is the chainlink version that has the plugins + ENABLE_OTEL_TRACES: true # END: OCR tests # START: Automation tests - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_0|TestAutomationBasic/registry_2_1_conditional|TestAutomationBasic/registry_2_1_logtrigger$" -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_1_with_mercury_v02|TestAutomationBasic/registry_2_1_with_mercury_v03|TestAutomationBasic/registry_2_1_with_logtrigger_and_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_conditional|TestAutomationBasic/registry_2_2_logtrigger|TestAutomationBasic/registry_2_2_with_mercury_v02$" -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_2_with_mercury_v03|TestAutomationBasic/registry_2_2_with_logtrigger_and_mercury_v02$" -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_conditional_native|TestAutomationBasic/registry_2_3_conditional_link$" -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_logtrigger_native|TestAutomationBasic/registry_2_3_logtrigger_link$" -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$ + - id: smoke/automation_test.go:^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run "^TestAutomationBasic/registry_2_3_with_mercury_v03_link|TestAutomationBasic/registry_2_3_with_logtrigger_and_mercury_v02_link$" -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestSetUpkeepTriggerConfig$ + - id: smoke/automation_test.go:^TestSetUpkeepTriggerConfig$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetUpkeepTriggerConfig$ -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationAddFunds$ + - id: smoke/automation_test.go:^TestAutomationAddFunds$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationPauseUnPause$ + - id: smoke/automation_test.go:^TestAutomationPauseUnPause$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseUnPause$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationRegisterUpkeep$ + - id: smoke/automation_test.go:^TestAutomationRegisterUpkeep$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationPauseRegistry$ + - id: smoke/automation_test.go:^TestAutomationPauseRegistry$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPauseRegistry$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationKeeperNodesDown$ + - id: smoke/automation_test.go:^TestAutomationKeeperNodesDown$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationKeeperNodesDown$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationPerformSimulation$ + - id: smoke/automation_test.go:^TestAutomationPerformSimulation$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationPerformSimulation$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestAutomationCheckPerformGasLimit$ + - id: smoke/automation_test.go:^TestAutomationCheckPerformGasLimit$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationCheckPerformGasLimit$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestUpdateCheckData$ + - id: smoke/automation_test.go:^TestUpdateCheckData$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestUpdateCheckData$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/automation_test.go:^TestSetOffchainConfigWithMaxGasPrice$ + - id: smoke/automation_test.go:^TestSetOffchainConfigWithMaxGasPrice$ path: integration-tests/smoke/automation_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestSetOffchainConfigWithMaxGasPrice$ -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-automation-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperBasicSmoke$ + - id: smoke/keeper_test.go:^TestKeeperBasicSmoke$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBasicSmoke$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperBlockCountPerTurn$ + - id: smoke/keeper_test.go:^TestKeeperBlockCountPerTurn$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperBlockCountPerTurn$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperSimulation$ + - id: smoke/keeper_test.go:^TestKeeperSimulation$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperSimulation$ -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperCheckPerformGasLimit$ + - id: smoke/keeper_test.go:^TestKeeperCheckPerformGasLimit$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperCheckPerformGasLimit$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperRegisterUpkeep$ + - id: smoke/keeper_test.go:^TestKeeperRegisterUpkeep$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRegisterUpkeep$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperAddFunds$ + - id: smoke/keeper_test.go:^TestKeeperAddFunds$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperAddFunds$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperRemove$ + - id: smoke/keeper_test.go:^TestKeeperRemove$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperRemove$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperPauseRegistry$ + - id: smoke/keeper_test.go:^TestKeeperPauseRegistry$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseRegistry$ -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperMigrateRegistry$ + - id: smoke/keeper_test.go:^TestKeeperMigrateRegistry$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperMigrateRegistry$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperNodeDown$ + - id: smoke/keeper_test.go:^TestKeeperNodeDown$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperNodeDown$ -test.parallel=3 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperPauseUnPauseUpkeep$ + - id: smoke/keeper_test.go:^TestKeeperPauseUnPauseUpkeep$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperPauseUnPauseUpkeep$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperUpdateCheckData$ + - id: smoke/keeper_test.go:^TestKeeperUpdateCheckData$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperUpdateCheckData$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/smoke/keeper_test.go:^TestKeeperJobReplacement$ + - id: smoke/keeper_test.go:^TestKeeperJobReplacement$ path: integration-tests/smoke/keeper_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestKeeperJobReplacement$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-keeper-evm-simulated - - id: integration-tests/load/automationv2_1/automationv2_1_test.go:TestLogTrigger + - id: load/automationv2_1/automationv2_1_test.go:TestLogTrigger path: integration-tests/load/automationv2_1/automationv2_1_test.go runs_on: ubuntu-latest test_env_type: k8s-remote-runner @@ -497,88 +476,88 @@ runner-test-matrix: remote_runner_memory: 4Gi test_config_override_required: true test_secrets_required: true - test_inputs: - test_suite: automationv2_1 + test_env_vars: + TEST_SUITE: automationv2_1 workflows: - Automation Load Test pyroscope_env: automation-load-test - - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0 + - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_0 path: integration-tests/smoke/automation_upgrade_test.go test_env_type: docker runs_on: ubuntu22.04-8cores-32GB workflows: - Run Automation Product Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_0 -test.parallel=1 -timeout 60m -count=1 -json - test_inputs: - chainlink_image: public.ecr.aws/chainlink/chainlink - chainlink_version: latest - chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' - chainlink_upgrade_version: develop + test_env_vars: + E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink + E2E_TEST_CHAINLINK_VERSION: latest + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}' + E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop pyroscope_env: ci-smoke-automation-upgrade-tests - - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1 + - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_1 path: integration-tests/smoke/automation_upgrade_test.go test_env_type: docker runs_on: ubuntu22.04-8cores-32GB workflows: - Run Automation Product Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_1 -test.parallel=5 -timeout 60m -count=1 -json - test_inputs: - chainlink_image: public.ecr.aws/chainlink/chainlink - chainlink_version: latest - chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' - chainlink_upgrade_version: develop + test_env_vars: + E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink + E2E_TEST_CHAINLINK_VERSION: latest + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}' + E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop pyroscope_env: ci-smoke-automation-upgrade-tests - - id: integration-tests/smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2 + - id: smoke/automation_upgrade_test.go:^TestAutomationNodeUpgrade/registry_2_2 path: integration-tests/smoke/automation_upgrade_test.go test_env_type: docker runs_on: ubuntu22.04-8cores-32GB workflows: - Run Automation Product Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestAutomationNodeUpgrade/registry_2_2 -test.parallel=5 -timeout 60m -count=1 -json - test_inputs: - chainlink_image: public.ecr.aws/chainlink/chainlink - chainlink_version: latest - chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' - chainlink_upgrade_version: develop + test_env_vars: + E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink + E2E_TEST_CHAINLINK_VERSION: latest + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}' + E2E_TEST_CHAINLINK_UPGRADE_VERSION: develop pyroscope_env: ci-smoke-automation-upgrade-tests - - id: integration-tests/reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_0 + - id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_0 path: integration-tests/reorg/automation_reorg_test.go runs_on: ubuntu-latest test_env_type: docker - test_inputs: - test_suite: reorg + test_env_vars: + TEST_SUITE: reorg workflows: - Run Automation On Demand Tests (TEST WORKFLOW) test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_0 -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg - - id: integration-tests/reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_1 + - id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_1 path: integration-tests/reorg/automation_reorg_test.go runs_on: ubuntu-latest test_env_type: docker - test_inputs: - test_suite: reorg + test_env_vars: + TEST_SUITE: reorg workflows: - Run Automation On Demand Tests (TEST WORKFLOW) test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_1 -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg - - id: integration-tests/reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_2 + - id: reorg/automation_reorg_test.go^TestAutomationReorg/registry_2_2 path: integration-tests/reorg/automation_reorg_test.go runs_on: ubuntu-latest test_env_type: docker - test_inputs: - test_suite: reorg + test_env_vars: + TEST_SUITE: reorg workflows: - Run Automation On Demand Tests (TEST WORKFLOW) test_cmd: cd integration-tests/reorg && DETACH_RUNNER=false go test -v -test.run ^TestAutomationReorg/registry_2_2 -test.parallel=2 -timeout 30m -count=1 -json pyroscope_env: ci-automation-on-demand-reorg - - id: integration-tests/chaos/automation_chaos_test.go + - id: chaos/automation_chaos_test.go path: integration-tests/chaos/automation_chaos_test.go test_env_type: k8s-remote-runner runs_on: ubuntu-latest @@ -586,26 +565,27 @@ runner-test-matrix: - Run Automation On Demand Tests (TEST WORKFLOW) test_cmd: cd integration-tests/chaos && DETACH_RUNNER=false go test -v -test.run ^TestAutomationChaos$ -test.parallel=20 -timeout 60m -count=1 -json pyroscope_env: ci-automation-on-demand-chaos - test_inputs: - test_suite: chaos + test_env_vars: + TEST_SUITE: chaos - - id: integration-tests/benchmark/keeper_test.go:^TestAutomationBenchmark$ + - id: benchmark/keeper_test.go:^TestAutomationBenchmark$ path: integration-tests/benchmark/keeper_test.go test_env_type: k8s-remote-runner remote_runner_memory: 4Gi runs_on: ubuntu-latest # workflows: - # - Run Nightly E2E Tests + # - Nightly E2E Tests test_cmd: cd integration-tests/benchmark && go test -v -test.run ^TestAutomationBenchmark$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-benchmark-automation-nightly - test_inputs: - test_suite: benchmark + test_env_vars: + TEST_SUITE: benchmark + TEST_TYPE: benchmark # END: Automation tests # START: VRF tests - - id: integration-tests/smoke/vrfv2_test.go:TestVRFv2Basic + - id: smoke/vrfv2_test.go:TestVRFv2Basic path: integration-tests/smoke/vrfv2_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker @@ -615,55 +595,55 @@ runner-test-matrix: workflows: - On Demand VRFV2 Smoke Test (Ethereum clients) - - id: integration-tests/load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusPerformance$Smoke + - id: load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusPerformance$Smoke path: integration-tests/load/vrfv2plus/vrfv2plus_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusPerformance$ -test.parallel=1 -timeout 24h -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_type: Smoke + test_env_vars: + TEST_TYPE: Smoke workflows: - On Demand VRFV2 Plus Performance Test - - id: integration-tests/load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusBHSPerformance$Smoke + - id: load/vrfv2plus/vrfv2plus_test.go:^TestVRFV2PlusBHSPerformance$Smoke path: integration-tests/load/vrfv2plus/vrfv2plus_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker test_cmd: cd integration-tests/load/vrfv2plus && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_type: Smoke + test_env_vars: + TEST_TYPE: Smoke workflows: - On Demand VRFV2 Plus Performance Test - - id: integration-tests/load/vrfv2/vrfv2_test.go:^TestVRFV2Performance$Smoke + - id: load/vrfv2/vrfv2_test.go:^TestVRFV2Performance$Smoke path: integration-tests/load/vrfv2/vrfv2_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2Performance$ -test.parallel=1 -timeout 24h -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_type: Smoke + test_env_vars: + TEST_TYPE: Smoke workflows: - On Demand VRFV2 Performance Test - - id: integration-tests/load/vrfv2/vrfv2_test.go:^TestVRFV2PlusBHSPerformance$Smoke + - id: load/vrfv2/vrfv2_test.go:^TestVRFV2PlusBHSPerformance$Smoke path: integration-tests/load/vrfv2/vrfv2_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker test_cmd: cd integration-tests/load/vrfv2 && go test -v -test.run ^TestVRFV2PlusBHSPerformance$ -test.parallel=1 -timeout 24h -count=1 -json test_config_override_required: true test_secrets_required: true - test_inputs: - test_type: Smoke + test_env_vars: + TEST_TYPE: Smoke workflows: - On Demand VRFV2 Performance Test - - id: integration-tests/smoke/vrfv2plus_test.go:^TestVRFv2Plus$/^Link_Billing$ + - id: smoke/vrfv2plus_test.go:^TestVRFv2Plus$/^Link_Billing$ path: integration-tests/smoke/vrfv2plus_test.go runs_on: ubuntu22.04-8cores-32GB test_env_type: docker @@ -673,33 +653,33 @@ runner-test-matrix: workflows: - On Demand VRFV2Plus Smoke Test (Ethereum clients) - - id: integration-tests/smoke/vrf_test.go:* + - id: smoke/vrf_test.go:* path: integration-tests/smoke/vrf_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/vrf_test.go -timeout 30m -count=1 -test.parallel=2 -json pyroscope_env: ci-smoke-vrf-evm-simulated - - id: integration-tests/smoke/vrfv2_test.go:* + - id: smoke/vrfv2_test.go:* path: integration-tests/smoke/vrfv2_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/vrfv2_test.go -timeout 30m -count=1 -test.parallel=6 -json pyroscope_env: ci-smoke-vrf2-evm-simulated - - id: integration-tests/smoke/vrfv2plus_test.go:* + - id: smoke/vrfv2plus_test.go:* path: integration-tests/smoke/vrfv2plus_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/vrfv2plus_test.go -timeout 30m -count=1 -test.parallel=9 -json pyroscope_env: ci-smoke-vrf2plus-evm-simulated @@ -707,83 +687,83 @@ runner-test-matrix: # START: LogPoller tests - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerFewFiltersFixedDepth$ + - id: smoke/log_poller_test.go:^TestLogPollerFewFiltersFixedDepth$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerFewFiltersFinalityTag$ + - id: smoke/log_poller_test.go:^TestLogPollerFewFiltersFinalityTag$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerFewFiltersFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosFixedDepth$ + - id: smoke/log_poller_test.go:^TestLogPollerWithChaosFixedDepth$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosFinalityTag$ + - id: smoke/log_poller_test.go:^TestLogPollerWithChaosFinalityTag$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFinalityTag$ + - id: smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFinalityTag$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFixedDepth$ + - id: smoke/log_poller_test.go:^TestLogPollerWithChaosPostgresFixedDepth$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerWithChaosPostgresFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerReplayFixedDepth$ + - id: smoke/log_poller_test.go:^TestLogPollerReplayFixedDepth$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFixedDepth$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated - - id: integration-tests/smoke/log_poller_test.go:^TestLogPollerReplayFinalityTag$ + - id: smoke/log_poller_test.go:^TestLogPollerReplayFinalityTag$ path: integration-tests/smoke/log_poller_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/smoke && go test -test.run ^TestLogPollerReplayFinalityTag$ -test.parallel=1 -timeout 30m -count=1 -json pyroscope_env: ci-smoke-log_poller-evm-simulated @@ -791,58 +771,188 @@ runner-test-matrix: # START: Other tests - - id: integration-tests/smoke/runlog_test.go:* + - id: smoke/runlog_test.go:* path: integration-tests/smoke/runlog_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests - test_cmd: cd integration-tests/ && go test smoke/runlog_test.go -timeout 30m -count=1 -json + - PR E2E Core Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ && go test smoke/runlog_test.go -timeout 30m -test.parallel=2 -count=1 -json pyroscope_env: ci-smoke-runlog-evm-simulated - - id: integration-tests/smoke/cron_test.go:* + - id: smoke/cron_test.go:* path: integration-tests/smoke/cron_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/cron_test.go -timeout 30m -count=1 -json pyroscope_env: ci-smoke-cron-evm-simulated - - id: integration-tests/smoke/flux_test.go:* + - id: smoke/flux_test.go:* path: integration-tests/smoke/flux_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/flux_test.go -timeout 30m -count=1 -json pyroscope_env: ci-smoke-flux-evm-simulated - - id: integration-tests/smoke/reorg_above_finality_test.go:* + - id: smoke/reorg_above_finality_test.go:* path: integration-tests/smoke/reorg_above_finality_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/ && go test smoke/reorg_above_finality_test.go -timeout 30m -count=1 -json pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated - - id: integration-tests/migration/upgrade_version_test.go:* + - id: migration/upgrade_version_test.go:* path: integration-tests/migration/upgrade_version_test.go test_env_type: docker runs_on: ubuntu-latest workflows: - - Run PR E2E Tests - - Run Nightly E2E Tests + - PR E2E Core Tests + - Nightly E2E Tests test_cmd: cd integration-tests/migration && go test upgrade_version_test.go -timeout 30m -count=1 -test.parallel=2 -json - test_inputs: - chainlink_image: public.ecr.aws/chainlink/chainlink - chainlink_version: '{{ env.LATEST_CHAINLINK_RELEASE_VERSION }}' - chainlink_upgrade_image: '{{ env.QA_CHAINLINK_IMAGE }}' - chainlink_upgrade_version: develop + test_env_vars: + E2E_TEST_CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink + E2E_TEST_CHAINLINK_VERSION: '{{ env.LATEST_CHAINLINK_RELEASE_VERSION }}' + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: '{{ env.QA_CHAINLINK_IMAGE }}' + E2E_TEST_CHAINLINK_UPGRADE_VERSION: '{{ env.GITHUB_SHA }}' # END: Other tests + + # START: CCIP tests + + - id: ccip-smoke + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-smoke-1.4-pools + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml + + - id: ccip-smoke-usdc + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml + + - id: ccip-smoke-db-compatibility + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml + + - id: ccip-smoke-leader-lane + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + # Leader lane test is flakey in Core repo - Need to fix CCIP-3074 to enable it. + workflows: + # - PR E2E CCIP Tests + # - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPForBidirectionalLane$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + test_config_override_path: integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPTokenPoolRateLimits$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPTokenPoolRateLimits$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPMulticall$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPMulticall$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPOnRampLimits$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPOnRampLimits$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPOffRampCapacityLimit$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPOffRampCapacityLimit$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + - id: ccip-tests/smoke/ccip_test.go:^TestSmokeCCIPOffRampAggRateLimit$ + path: integration-tests/ccip-tests/smoke/ccip_test.go + test_env_type: docker + runs_on: ubuntu-latest + workflows: + - PR E2E CCIP Tests + - Nightly E2E Tests + test_cmd: cd integration-tests/ccip-tests/smoke && go test ccip_test.go -test.run ^TestSmokeCCIPOffRampAggRateLimit$ -timeout 30m -count=1 -test.parallel=1 -json + test_env_vars: + E2E_TEST_SELECTED_NETWORK: SIMULATED_1,SIMULATED_2 + + # END: CCIP tests \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 500966c6a29..8a96a66a68a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -92,7 +92,7 @@ jobs: id: changes with: filters: | - changes: + core_changes: - '**/*.go' - '**/*go.sum' - '**/*go.mod' @@ -101,12 +101,9 @@ jobs: - 'core/**/migrations/*.sql' - 'core/**/config/**/*.toml' - 'integration-tests/**/*.toml' - ccip-changes: - - '.github/workflows/integration-tests.yml' - - 'integration-tests/**/*.toml' - - '**/*Dockerfile' + ccip_changes: - '**/*ccip*' - - '**/*ccip*/**' + - '**/*ccip*/**' - name: Ignore Filter On Workflow Dispatch if: ${{ github.event_name == 'workflow_dispatch' }} id: ignore-filter @@ -123,11 +120,11 @@ jobs: this-job-name: Check Paths That Require Tests To Run continue-on-error: true outputs: - src: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.changes }} - ccip-changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.ccip-changes }} + core_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.core_changes }} + ccip_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.ccip_changes }} - build-lint-integration-tests: - name: Build and Lint ${{ matrix.project.name }} + lint-integration-tests: + name: Lint ${{ matrix.project.name }} runs-on: ubuntu22.04-8cores-32GB # We don't directly merge dependabot PRs, so let's not waste the resources if: github.actor != 'dependabot[bot]' @@ -135,13 +132,13 @@ jobs: matrix: project: - name: integration-tests - id: e2e + id: e2e-tests path: ./integration-tests - cache-id: e2e + cache_id: e2e-tests - name: load id: load path: ./integration-tests/load - cache-id: load + cache_id: load steps: - name: Collect Metrics id: collect-gha-metrics @@ -151,7 +148,7 @@ jobs: org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Build and Lint ${{ matrix.project.name }} + this-job-name: Lint ${{ matrix.project.name }} continue-on-error: true - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -159,17 +156,12 @@ jobs: repository: smartcontractkit/chainlink ref: ${{ inputs.cl_ref }} - name: Setup Go - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30 with: test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download go_mod_path: ${{ matrix.project.path }}/go.mod - cache_key_id: core-${{ matrix.project.cache-id }}-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - - name: Build Go - run: | - cd ${{ matrix.project.path }} - go build ./... - go test -run=^# ./... + cache_key_id: ${{ matrix.project.cache_id }} + cache_restore_only: "true" - name: Lint Go uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0 with: @@ -201,7 +193,7 @@ jobs: needs: [changes, enforce-ctf-version] steps: - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -226,7 +218,7 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} set-git-config: "true" - name: Build Chainlink Image - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' uses: ./.github/actions/build-chainlink-image with: tag_suffix: ${{ matrix.image.tag-suffix }} @@ -236,940 +228,126 @@ jobs: AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} dep_evm_sha: ${{ inputs.evm-ref }} - compare-tests: - needs: [changes] - runs-on: ubuntu-latest - name: Compare/Build Automation Test List - outputs: - automation-matrix: ${{ env.AUTOMATION_JOB_MATRIX_JSON }} - lp-matrix: ${{ env.LP_JOB_MATRIX_JSON }} - steps: - - name: Check for Skip Tests Label - if: contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') - run: | - echo "## \`skip-smoke-tests\` label is active, skipping E2E smoke tests" >>$GITHUB_STEP_SUMMARY - exit 0 - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref }} - - name: Compare Test Lists - run: | - cd ./integration-tests - ./scripts/compareTestList.sh ./smoke/automation_test.go - ./scripts/compareTestList.sh ./smoke/keeper_test.go - ./scripts/compareTestList.sh ./smoke/log_poller_test.go - - name: Build Test Matrix Lists - id: build-test-matrix-list - run: | - cd ./integration-tests - MATRIX_JSON_AUTOMATION=$(./scripts/buildTestMatrixList.sh ./smoke/automation_test.go automation ubuntu-latest 1) - MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu-latest 1) - COMBINED_ARRAY=$(jq -c -n "$MATRIX_JSON_AUTOMATION + $MATRIX_JSON_KEEPER") - echo "combined array = ${COMBINED_ARRAY}" - echo "event name = $GITHUB_EVENT_NAME" - - LOG_POLLER_MATRIX_JSON=$(./scripts/buildTestMatrixList.sh ./smoke/log_poller_test.go log_poller ubuntu-latest 1) - echo "LP_JOB_MATRIX_JSON=${LOG_POLLER_MATRIX_JSON}" >> $GITHUB_ENV - - # if we running a PR against the develop branch we should only run the automation tests unless we are in the merge group event - if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then - echo "We are in a merge_group event, run both automation and keepers tests" - echo "AUTOMATION_JOB_MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV - else - echo "we are not in a merge_group event, if this is a PR to develop run only automation tests, otherwise run everything because we could be running against a release branch" - target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.base.ref) - if [[ "$target_branch" == "develop" ]]; then - echo "only run automation tests" - echo "AUTOMATION_JOB_MATRIX_JSON=${MATRIX_JSON_AUTOMATION}" >> $GITHUB_ENV - else - echo "run both automation and keepers tests" - echo "AUTOMATION_JOB_MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV - fi - fi - - eth-smoke-tests-matrix-automation: - if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: - [build-chainlink, changes, compare-tests, build-lint-integration-tests] - env: - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - product: ${{fromJson(needs.compare-tests.outputs.automation-matrix)}} - runs-on: ${{ matrix.product.os }} - name: ETH Smoke Tests ${{ matrix.product.name }} - steps: - - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.name }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - this-job-name: ETH Smoke Tests ${{ matrix.product.name }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command - run: | - # if the matrix.product.run is set, use it for a different command - if [ "${{ matrix.product.run }}" != "" ]; then - echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - fi - - name: Setup GAP for Grafana - uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - id: setup-gap - with: - # aws inputs - aws-region: ${{ secrets.AWS_REGION }} - aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # other inputs - duplicate-authorization-header: "true" - - ## Run this step when changes that require tests to be run are made - - name: Run Tests - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ inputs.evm-ref || github.sha }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.product.name }}-test-artifacts - artifacts_location: | - ./integration-tests/smoke/logs/ - ./integration-tests/smoke/db_dumps/ - ./integration-tests/smoke/seth_artifacts/ - /tmp/gotest.log - publish_check_name: ${{ matrix.product.name }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - go_coverage_src_dir: /var/tmp/go-coverage - go_coverage_dest_dir: ${{ github.workspace }}/.covdata - DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - - name: Upload Coverage Data - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - timeout-minutes: 2 - continue-on-error: true - with: - name: cl-node-coverage-data-${{ matrix.product.name }} - path: .covdata - retention-days: 1 - - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25 - - eth-smoke-tests-matrix-log-poller: - if: ${{ !(contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') || github.event_name == 'workflow_dispatch') || inputs.distinct_run_name != '' }} - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - needs: - [build-chainlink, changes, compare-tests, build-lint-integration-tests] - env: - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - product: ${{fromJson(needs.compare-tests.outputs.lp-matrix)}} - runs-on: ${{ matrix.product.os }} - name: ETH Smoke Tests ${{ matrix.product.name }} - steps: - - name: Collect Metrics - if: needs.changes.outputs.src == 'true' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.name }} - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: ETH Smoke Tests ${{ matrix.product.name }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command - run: | - # if the matrix.product.run is set, use it for a different command - if [ "${{ matrix.product.run }}" != "" ]; then - echo "run_command=${{ matrix.product.run }} ./smoke/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./smoke/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - fi - - name: Setup GAP for Grafana - uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - id: setup-gap - with: - # aws inputs - aws-region: ${{ secrets.AWS_REGION }} - aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # other inputs - duplicate-authorization-header: "true" - ## Run this step when changes that require tests to be run are made - - name: Run Tests - if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ inputs.evm-ref || github.sha }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.product.name }}-test-artifacts - artifacts_location: | - ./integration-tests/smoke/logs/ - ./integration-tests/smoke/db_dumps/ - ./integration-tests/smoke/seth_artifacts/ - /tmp/gotest.log - publish_check_name: ${{ matrix.product.name }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - go_coverage_src_dir: /var/tmp/go-coverage - go_coverage_dest_dir: ${{ github.workspace }}/.covdata - DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - - name: Upload Coverage Data - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - timeout-minutes: 2 - continue-on-error: true - with: - name: cl-node-coverage-data-${{ matrix.product.name }} - path: .covdata - retention-days: 1 - - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - - eth-smoke-tests-matrix-ccip: - if: steps.changes.outputs.ccip-changes == 'true' && ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} - environment: integration + run-core-e2e-tests-workflow: + name: Run Core E2E Tests permissions: actions: read checks: write pull-requests: write id-token: write contents: read - needs: [build-chainlink, changes, build-lint-integration-tests] - env: - SELECTED_NETWORKS: SIMULATED - CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - product: - - name: ccip-smoke - nodes: 1 - os: ubuntu-latest - file: ccip - dir: ccip-tests/smoke - run: -run ^TestSmokeCCIPForBidirectionalLane$ - - name: ccip-smoke-1.4-pools - nodes: 1 - os: ubuntu-latest - file: ccip - dir: ccip-tests/smoke - run: -run ^TestSmokeCCIPForBidirectionalLane$ - config_path: ./integration-tests/ccip-tests/testconfig/tomls/contract-version1.4.toml - - name: ccip-smoke-usdc - nodes: 1 - os: ubuntu-latest - file: ccip - dir: ccip-tests/smoke - run: -run ^TestSmokeCCIPForBidirectionalLane$ - config_path: ./integration-tests/ccip-tests/testconfig/tomls/usdc_mock_deployment.toml - - name: ccip-smoke-db-compatibility - nodes: 1 - os: ubuntu-latest - file: ccip - dir: ccip-tests/smoke - run: -run ^TestSmokeCCIPForBidirectionalLane$ - config_path: ./integration-tests/ccip-tests/testconfig/tomls/db-compatibility.toml - - name: ccip-smoke-rate-limit - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPRateLimit$ - - name: ccip-smoke-rate-limit - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPTokenPoolRateLimits$ - - name: ccip-smoke-multicall - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPMulticall$ - - name: ccip-smoke-manual-exec - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPManuallyExecuteAfterExecutionFailingDueToInsufficientGas$ - - name: ccip-smoke-on-ramp-limits - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPOnRampLimits$ - - name: ccip-smoke-off-ramp-capacity - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPOffRampCapacityLimit$ - - name: ccip-smoke-off-ramp-agg-rate-limit - nodes: 1 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPOffRampAggRateLimit$ - - name: ccip-smoke-leader-lane - nodes: 15 - dir: ccip-tests/smoke - os: ubuntu-latest - file: ccip - run: -run ^TestSmokeCCIPForBidirectionalLane$ - config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml - runs-on: ${{ matrix.product.os }} - name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - steps: - # Handy for debugging resource usage - # - name: Collect Workflow Telemetry - # uses: catchpoint/workflow-telemetry-action@v2 - - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }} - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command - run: | - # if dir is provided use it, otherwise use the smoke dir - if [ "${{ matrix.product.dir }}" != "" ]; then - dir=${{ matrix.product.dir }} - else - dir=smoke - fi - # if the matrix.product.run is set, use it for a different command - if [ "${{ matrix.product.run }}" != "" ]; then - echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - fi - - name: Check for "enable tracing" label - id: check-label - run: | - label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") - - if [[ -n "$label" ]]; then - if [[ "$label" == "enable tracing" ]]; then - echo "Enable tracing label found." - echo "trace=true" >> $GITHUB_OUTPUT - else - echo "Enable tracing label not found." - echo "trace=false" >> $GITHUB_OUTPUT - fi - else - echo "No labels present or labels are null." - echo "trace=false" >> $GITHUB_OUTPUT - fi - - - name: Setup Grafana and OpenTelemetry - id: docker-setup - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - # Create network - docker network create --driver bridge tracing - - # Make trace directory - cd integration-tests/smoke/ - mkdir ./traces - chmod -R 777 ./traces - - # Switch directory - cd ../../.github/tracing - - # Create a Docker volume for traces - # docker volume create otel-traces - - # Start OpenTelemetry Collector - # Note the user must be set to the same user as the runner for the trace data to be accessible - docker run -d --network=tracing --name=otel-collector \ - -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ - -v $PWD/../../integration-tests/smoke/traces:/tracing \ - --user "$(id -u):$(id -g)" \ - -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml - - - name: Locate Docker Volume - id: locate-volume - if: false - run: | - echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT - - - name: Show Otel-Collector Logs - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - docker logs otel-collector - - - name: Set Override Config - id: set_override_config - run: | - # if the matrix.product.config_path is set, use it as the override config - if [ "${{ matrix.product.config_path }}" != "" ]; then - echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" - fi - - - name: Setup GAP for Grafana - uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - id: setup-gap - with: - # aws inputs - aws-region: ${{ secrets.AWS_REGION }} - aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # other inputs - duplicate-authorization-header: "true" - - - name: Prepare Base64 CCIP TOML secrets - uses: ./.github/actions/setup-create-base64-config-ccip - id: setup_create_base64_config_ccip - with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: SIMULATED_1,SIMULATED_2 - chainlinkVersion: ${{ inputs.evm-ref || github.sha }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - - ## Run this step when changes that require tests to be run are made - - name: Run Tests - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - env: - BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - TEST_BASE64_CCIP_CONFIG_OVERRIDE: ${{ steps.set_override_config.outputs.base_64_override }},${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs - artifacts_location: | - ./integration-tests/smoke/logs/ - ./integration-tests/smoke/db_dumps/ - /tmp/gotest.log - publish_check_name: ${{ matrix.product.name }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - go_coverage_src_dir: /var/tmp/go-coverage - go_coverage_dest_dir: ${{ github.workspace }}/.covdata - DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - - name: Upload Coverage Data - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - timeout-minutes: 2 - continue-on-error: true - with: - name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }} - path: .covdata - retention-days: 1 - - # Run this step when changes that do not need the test to run are made - - name: Run Setup - if: needs.changes.outputs.src == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - test_download_vendor_packages_command: cd ./integration-tests && go mod download - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - - - name: Show Otel-Collector Logs - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - docker logs otel-collector - - - name: Permissions on traces - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - ls -l ./integration-tests/smoke/traces - - - name: Upload Trace Data - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - name: trace-data - path: ./integration-tests/smoke/traces/trace-data.json - - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - test_directories: ./integration-tests/smoke/ - - - eth-smoke-tests-matrix: - if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} - environment: integration + needs: [build-chainlink, changes] + if: needs.changes.outputs.core_changes == 'true' + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + workflow_name: Run Core E2E Tests + chainlink_version: ${{ inputs.evm-ref || github.sha }} + chainlink_upgrade_version: ${{ github.sha }} + test_workflow: PR E2E Core Tests + upload_cl_node_coverage_artifact: true + upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_ + enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }} + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + + run-ccip-e2e-tests-workflow: + name: Run CCIP E2E Tests permissions: actions: read checks: write pull-requests: write id-token: write contents: read - needs: [build-chainlink, changes, build-lint-integration-tests] - env: - SELECTED_NETWORKS: SIMULATED - CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - TEST_LOG_LEVEL: debug - strategy: - fail-fast: false - matrix: - product: - - name: runlog - id: runlog - nodes: 2 - os: ubuntu-latest - pyroscope_env: "ci-smoke-runlog-evm-simulated" - - name: cron - id: cron - nodes: 2 - os: ubuntu-latest - pyroscope_env: "ci-smoke-cron-evm-simulated" - - name: flux - id: flux - nodes: 1 - os: ubuntu-latest - pyroscope_env: "ci-smoke-flux-evm-simulated" - - name: ocr - id: ocr - nodes: 2 - os: ubuntu-latest - file: ocr - pyroscope_env: ci-smoke-ocr-evm-simulated - - name: reorg_above_finality - id: reorg_above_finality - nodes: 1 - os: ubuntu-latest - file: reorg_above_finality - pyroscope_env: ci-smoke-reorg-above-finality-evm-simulated - - name: ocr2 - id: ocr2 - nodes: 6 - os: ubuntu22.04-16cores-64GB - file: ocr2 - pyroscope_env: ci-smoke-ocr2-evm-simulated - - name: ocr2 - id: ocr2-plugins - nodes: 6 - os: ubuntu22.04-16cores-64GB - pyroscope_env: ci-smoke-ocr2-plugins-evm-simulated - tag_suffix: "-plugins" - - name: vrf - id: vrf - nodes: 2 - os: ubuntu-latest - pyroscope_env: ci-smoke-vrf-evm-simulated - - name: vrfv2 - id: vrfv2 - nodes: 6 - os: ubuntu-latest - pyroscope_env: ci-smoke-vrf2-evm-simulated - - name: vrfv2plus - id: vrfv2plus - nodes: 9 - os: ubuntu-latest - pyroscope_env: ci-smoke-vrf2plus-evm-simulated - - name: forwarder_ocr - id: forwarder_ocr - nodes: 2 - os: ubuntu-latest - pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - - name: forwarders_ocr2 - id: forwarders_ocr2 - nodes: 2 - os: ubuntu-latest - pyroscope_env: ci-smoke-forwarder-ocr-evm-simulated - runs-on: ${{ matrix.product.os }} - name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} + needs: [build-chainlink, changes] + if: needs.changes.outputs.ccip_changes == 'true' + uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml + with: + workflow_name: Run CCIP E2E Tests + chainlink_version: ${{ inputs.evm-ref || github.sha }} + chainlink_upgrade_version: ${{ github.sha }} + test_workflow: PR E2E CCIP Tests + upload_cl_node_coverage_artifact: true + upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_ + enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }} + secrets: + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }} + QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }} + GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} + + check-e2e-test-results: + if: always() + name: ETH Smoke Tests + runs-on: ubuntu-latest + needs: [run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow] steps: - # Handy for debugging resource usage - # - name: Collect Workflow Telemetry - # uses: catchpoint/workflow-telemetry-action@v2 - - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-matrix-${{ matrix.product.id }} - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: ETH Smoke Tests ${{ matrix.product.name }}${{ matrix.product.tag_suffix }} - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Build Go Test Command - id: build-go-test-command + - name: Check Core test results + id: check_core_results run: | - # if dir is provided use it, otherwise use the smoke dir - if [ "${{ matrix.product.dir }}" != "" ]; then - dir=${{ matrix.product.dir }} - else - dir=smoke - fi - # if the matrix.product.run is set, use it for a different command - if [ "${{ matrix.product.run }}" != "" ]; then - echo "run_command=${{ matrix.product.run }} ./${dir}/${{ matrix.product.file }}_test.go" >> "$GITHUB_OUTPUT" - else - echo "run_command=./${dir}/${{ matrix.product.name }}_test.go" >> "$GITHUB_OUTPUT" - fi - - name: Check for "enable tracing" label - id: check-label + results='${{ needs.run-core-e2e-tests-workflow.outputs.test_results }}' + echo "Core test results:" + echo "$results" | jq . + + node_migration_tests_failed=$(echo $results | jq '[.[] | select(.id == "integration-tests/migration/upgrade_version_test.go:*" ) | select(.result != "success")] | length > 0') + echo "node_migration_tests_failed=$node_migration_tests_failed" >> $GITHUB_OUTPUT + + - name: Check CCIP test results + id: check_ccip_results run: | - label=$(jq -r '.pull_request.labels[]?.name // empty' "$GITHUB_EVENT_PATH") - - if [[ -n "$label" ]]; then - if [[ "$label" == "enable tracing" ]]; then - echo "Enable tracing label found." - echo "trace=true" >> $GITHUB_OUTPUT - else - echo "Enable tracing label not found." - echo "trace=false" >> $GITHUB_OUTPUT - fi + if [[ '${{ needs.run-ccip-e2e-tests-workflow.result }}' != 'skipped' ]]; then + results='${{ needs.run-ccip-e2e-tests-workflow.outputs.test_results }}' + echo "CCIP test results:" + echo "$results" | jq . else - echo "No labels present or labels are null." - echo "trace=false" >> $GITHUB_OUTPUT + echo "CCIP tests were skipped." fi - - name: Setup Grafana and OpenTelemetry - id: docker-setup - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - # Create network - docker network create --driver bridge tracing - - # Make trace directory - cd integration-tests/smoke/ - mkdir ./traces - chmod -R 777 ./traces - - # Switch directory - cd ../../.github/tracing - - # Create a Docker volume for traces - # docker volume create otel-traces - - # Start OpenTelemetry Collector - # Note the user must be set to the same user as the runner for the trace data to be accessible - docker run -d --network=tracing --name=otel-collector \ - -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ - -v $PWD/../../integration-tests/smoke/traces:/tracing \ - --user "$(id -u):$(id -g)" \ - -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml - - - name: Locate Docker Volume - id: locate-volume - if: false - run: | - echo "VOLUME_PATH=$(docker volume inspect --format '{{ .Mountpoint }}' otel-traces)" >> $GITHUB_OUTPUT - - - name: Show Otel-Collector Logs - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - docker logs otel-collector - - - name: Set Override Config - id: set_override_config - run: | - # if the matrix.product.config_path is set, use it as the override config - if [ "${{ matrix.product.config_path }}" != "" ]; then - echo "base_64_override=$(base64 -w 0 -i ${{ matrix.product.config_path }})" >> "$GITHUB_OUTPUT" - fi - - - name: Setup GAP for Grafana - uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 - id: setup-gap - with: - # aws inputs - aws-region: ${{ secrets.AWS_REGION }} - aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} - api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} - # other inputs - duplicate-authorization-header: "true" - - ## Run this step when changes that require tests to be run are made - - name: Run Tests - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_chainlink_version: ${{ inputs.evm-ref || github.sha }} - test_config_selected_networks: ${{ env.SELECTED_NETWORKS }} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-artifacts - artifacts_location: | - ./integration-tests/smoke/logs/ - ./integration-tests/smoke/db_dumps/ - ./integration-tests/smoke/seth_artifacts/ - /tmp/gotest.log - publish_check_name: ${{ matrix.product.name }} - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - go_coverage_src_dir: /var/tmp/go-coverage - go_coverage_dest_dir: ${{ github.workspace }}/.covdata - DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.product.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.product.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} - - - name: Upload Coverage Data - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - timeout-minutes: 2 - continue-on-error: true - with: - name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }} - path: .covdata - retention-days: 1 - - # Run this step when changes that do not need the test to run are made - - name: Run Setup - if: needs.changes.outputs.src == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - test_download_vendor_packages_command: cd ./integration-tests && go mod download - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - should_tidy: "false" - - - name: Show Otel-Collector Logs - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - docker logs otel-collector - - - name: Permissions on traces - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - run: | - ls -l ./integration-tests/smoke/traces - - - name: Upload Trace Data - if: steps.check-label.outputs.trace == 'true' && matrix.product.name == 'ocr2' && matrix.product.tag_suffix == '-plugins' - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - name: trace-data - path: ./integration-tests/smoke/traces/trace-data.json - - - name: Print failed test summary - if: always() - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25 - with: - test_directories: ./integration-tests/smoke/ - - ### Used to check the required checks box when the matrix completes - eth-smoke-tests: - if: always() - runs-on: ubuntu-latest - name: ETH Smoke Tests - needs: [eth-smoke-tests-matrix, eth-smoke-tests-matrix-automation, eth-smoke-tests-matrix-log-poller] - steps: - - name: Check smoke test matrix status - if: needs.eth-smoke-tests-matrix.result != 'success' || needs.eth-smoke-tests-matrix-automation.result != 'success' || needs.eth-smoke-tests-matrix-log-poller.result != 'success' - run: | - echo "ETH Smoke Tests: ${{ needs.eth-smoke-tests-matrix.result }}" - echo "Automation: ${{ needs.eth-smoke-tests-matrix-automation.result }}" - echo "Log Poller: ${{ needs.eth-smoke-tests-matrix-log-poller.result }}" - exit 1 - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 + - name: Send slack notification for failed migration tests + if: steps.check_core_results.outputs.node_migration_tests_failed == 'true' && github.event_name != 'workflow_dispatch' + uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 + env: + SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} with: - id: ${{ env.COLLECTION_ID }}-matrix-results - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: ETH Smoke Tests - matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix.result }} - continue-on-error: true + channel-id: "#team-test-tooling-internal" + slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying ', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}" - eth-smoke-tests-ccip: - if: always() - runs-on: ubuntu-latest - name: ETH Smoke Tests CCIP - needs: eth-smoke-tests-matrix-ccip - steps: - - name: Check smoke test matrix status - if: needs.eth-smoke-tests-matrix-ccip.result != 'success' + - name: Fail the job if Core tests failed + if: always() && needs.run-core-e2e-tests-workflow.result == 'failure' run: | - echo "ETH Smoke Tests CCIP: ${{ needs.eth-smoke-tests-matrix-ccip.result }}" + echo "Core E2E tests failed" + echo "Job status:" + echo ${{ needs.run-core-e2e-tests-workflow.result }} exit 1 - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-matrix-results-ccip - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: ETH Smoke Tests CCIP - matrix-aggregator-status: ${{ needs.eth-smoke-tests-matrix-ccip.result }} - continue-on-error: true cleanup: name: Clean up integration environment deployments if: always() - needs: [eth-smoke-tests] + needs: [run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow] runs-on: ubuntu-latest steps: - name: Checkout repo @@ -1198,10 +376,10 @@ jobs: this-job-name: Clean up integration environment deployments continue-on-error: true - show-coverage: + show-chainlink-node-coverage: name: Show Chainlink Node Go Coverage if: always() - needs: [cleanup] + needs: [run-core-e2e-tests-workflow, run-ccip-e2e-tests-workflow] runs-on: ubuntu-latest steps: - name: Checkout the repo @@ -1212,151 +390,11 @@ jobs: - name: Download All Artifacts uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6 with: - path: cl-node-coverage-data - pattern: cl-node-coverage-data-* + path: cl_node_coverage_data + pattern: cl_node_coverage_data_* merge-multiple: true - name: Show Coverage - run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged" - - # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss - # this will also only run if both of the matrix jobs pass - eth-smoke-go-mod-cache: - environment: integration - needs: [eth-smoke-tests] - runs-on: ubuntu-latest - name: ETH Smoke Tests Go Mod Cache - continue-on-error: true - steps: - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Run Setup - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-go@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - with: - test_download_vendor_packages_command: | - cd ./integration-tests - go mod download - # force download of test dependencies - go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure" - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "false" - - ### Migration tests - node-migration-tests: - name: Version Migration Tests - environment: integration - permissions: - checks: write - pull-requests: write - id-token: write - contents: read - runs-on: ubuntu-latest - needs: [build-chainlink, changes] - # Only run migration tests on new tags - if: startsWith(github.ref, 'refs/tags/') - env: - SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 - CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }} - CHAINLINK_ENV_USER: ${{ github.actor }} - CHAINLINK_IMAGE: public.ecr.aws/chainlink/chainlink - UPGRADE_VERSION: ${{ inputs.evm-ref || github.sha }} - UPGRADE_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - TEST_LOG_LEVEL: debug - TEST_SUITE: migration - steps: - - name: Collect Metrics - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ env.COLLECTION_ID }}-migration-tests - org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Version Migration Tests - test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' - continue-on-error: true - - name: Checkout the repo - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - with: - repository: smartcontractkit/chainlink - ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - - name: Get Latest Version - id: get_latest_version - run: | - untrimmed_ver=$(curl --header "Authorization: token ${{ secrets.GITHUB_TOKEN }}" --request GET https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r .name) - latest_version="${untrimmed_ver:1}" - # Check if latest_version is empty - if [ -z "$latest_version" ]; then - echo "Error: The latest_version is empty. The migration tests need a verison to run." - exit 1 - fi - echo "latest_version=${latest_version}" >> "$GITHUB_OUTPUT" - - name: Name Versions - run: | - echo "Running migration tests from version '${{ steps.get_latest_version.outputs.latest_version }}' to: '${{ inputs.evm-ref || github.sha }}'" - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-upgrade-config - with: - selectedNetworks: ${{ env.SELECTED_NETWORKS }} - chainlinkVersion: ${{ steps.get_latest_version.outputs.latest_version }} - upgradeVersion: ${{ env.UPGRADE_VERSION }} - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - - name: Run Migration Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 - with: - test_command_to_run: cd ./integration-tests && go test -timeout 20m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download - test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ steps.get_latest_version.outputs.latest_version }} - aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: node-migration-test-artifacts - artifacts_location: | - ./integration-tests/migration/logs - ./integration-tests/migration/db_dumps - ./integration-tests/migration/seth_artifacts - /tmp/gotest.log - publish_check_name: Node Migration Test Results - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} - cache_restore_only: "true" - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: "" - go_coverage_src_dir: /var/tmp/go-coverage - go_coverage_dest_dir: ${{ github.workspace }}/.covdata - should_tidy: "false" - DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ env.UPGRADE_IMAGE }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - - - name: Upload Coverage Data - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 - timeout-minutes: 2 - continue-on-error: true - with: - name: cl-node-coverage-data-migration-tests - path: .covdata - retention-days: 1 - - name: Notify Slack - if: failure() && github.event_name != 'workflow_dispatch' - uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} - with: - channel-id: "#team-test-tooling-internal" - slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying ', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}" + run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl_node_coverage_data/*/merged" ## Solana Section get_solana_sha: @@ -1439,7 +477,7 @@ jobs: steps: - name: Check if image exists id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + uses: smartcontractkit/chainlink-github-actions/docker/image-exists@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30 with: repository: chainlink-solana-tests tag: ${{ needs.get_solana_sha.outputs.sha }} @@ -1464,7 +502,7 @@ jobs: ] steps: - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -1480,7 +518,7 @@ jobs: repository: smartcontractkit/chainlink-solana ref: ${{ needs.get_solana_sha.outputs.sha }} - name: Build contracts - if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' + if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' uses: smartcontractkit/chainlink-solana/.github/actions/build_contract_artifacts@46b1311a5a83f33d08ffa8e1e0ab04f9ad51665d # node20 update on may 10, 2024 with: ref: ${{ needs.get_solana_sha.outputs.sha }} @@ -1507,7 +545,7 @@ jobs: CONTRACT_ARTIFACTS_PATH: contracts/target/deploy steps: - name: Collect Metrics - if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' + if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -1518,13 +556,13 @@ jobs: this-job-name: Solana Build Test Image continue-on-error: true - name: Checkout the repo - if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' + if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: repository: smartcontractkit/chainlink-solana ref: ${{ needs.get_solana_sha.outputs.sha }} - name: Build Test Image - if: (needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' + if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false' uses: ./.github/actions/build-test-image with: tag: ${{ needs.get_solana_sha.outputs.sha }} @@ -1533,7 +571,7 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - run: echo "this exists so we don't have to run anything else if the build is skipped" - if: needs.changes.outputs.src == 'false' || needs.solana-test-image-exists.outputs.exists == 'true' + if: needs.changes.outputs.core_changes == 'false' || needs.solana-test-image-exists.outputs.exists == 'true' solana-smoke-tests: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} @@ -1560,7 +598,7 @@ jobs: CONTRACT_ARTIFACTS_PATH: contracts/target/deploy steps: - name: Collect Metrics - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' id: collect-gha-metrics uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 with: @@ -1577,8 +615,8 @@ jobs: repository: smartcontractkit/chainlink-solana ref: ${{ needs.get_solana_sha.outputs.sha }} - name: Run Setup - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/setup-run-tests-environment@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30 with: go_mod_path: ./integration-tests/go.mod cache_restore_only: true @@ -1590,7 +628,7 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - name: Pull Artifacts - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' run: | IMAGE_NAME=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} # Pull the Docker image @@ -1630,7 +668,7 @@ jobs: # shellcheck disable=SC2086 echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - name: Run Tests - if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' + if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch' uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 with: test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke @@ -1660,7 +698,7 @@ jobs: timeout-minutes: 2 continue-on-error: true with: - name: cl-node-coverage-data-solana-tests + name: cl_node_coverage_data_solana_tests path: .covdata retention-days: 1 diff --git a/.github/workflows/run-automation-ondemand-e2e-tests.yml b/.github/workflows/run-automation-ondemand-e2e-tests.yml index 8dac3c56994..9e62b1557a4 100644 --- a/.github/workflows/run-automation-ondemand-e2e-tests.yml +++ b/.github/workflows/run-automation-ondemand-e2e-tests.yml @@ -91,53 +91,53 @@ jobs: # Always run upgrade tests cat > test_list.yaml <> test_list.yaml <> test_list.yaml <> $GITHUB_OUTPUT - name: Generate K8s Tests Matrix id: set-k8s-runner-matrix run: | - cd integration-tests/citool - MATRIX_JSON=$(go run main.go filter --file ${{ github.workspace }}/.github/e2e-tests.yml --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') + MATRIX_JSON=$(citool filter --file ${{ github.workspace }}/.github/e2e-tests.yml --test-env-type 'k8s-remote-runner' --test-list '${{ inputs.test_list }}' --test-ids '${{ inputs.test_ids }}' --workflow '${{ inputs.test_workflow }}') echo "K8s tests:" echo "$MATRIX_JSON" | jq echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT @@ -301,6 +343,11 @@ jobs: echo "No tests require secrets. Proceeding without additional secret setup." fi + - name: Generate random workflow id + id: gen_id + run: echo "workflow_id=$(uuidgen)" >> $GITHUB_OUTPUT + + # Build Chainlink images required for the tests require-chainlink-image-versions-in-qa-ecr: name: Build Chainlink image @@ -388,7 +435,7 @@ jobs: org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Run E2E Tests / Run ${{ matrix.tests.id }} + this-job-name: ${{ inputs.workflow_name }} / Run ${{ matrix.tests.id }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true @@ -398,8 +445,7 @@ jobs: run: sudo apt-get install -y jq - name: Show test configuration run: echo '${{ toJson(matrix.tests) }}' | jq . - - name: Setup Go - uses: ./.github/actions/setup-go + - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 id: setup-gap @@ -409,24 +455,65 @@ jobs: api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} duplicate-authorization-header: "true" + - name: Setup Grafana and OpenTelemetry + id: docker-setup + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true' + run: | + # Create network + docker network create --driver bridge tracing + + # Make trace directory + cd integration-tests/smoke/ + mkdir ./traces + chmod -R 777 ./traces + + # Switch directory + cd ../../.github/tracing + + # Create a Docker volume for traces + # docker volume create otel-traces + + # Start OpenTelemetry Collector + # Note the user must be set to the same user as the runner for the trace data to be accessible + docker run -d --network=tracing --name=otel-collector \ + -v $PWD/otel-collector-ci.yaml:/etc/otel-collector.yaml \ + -v $PWD/../../integration-tests/smoke/traces:/tracing \ + --user "$(id -u):$(id -g)" \ + -p 4317:4317 otel/opentelemetry-collector:0.88.0 --config=/etc/otel-collector.yaml + - name: Run tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 + id: run_tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@00c164251be2a7c5b2b23a6e5f7014982f232c14 # v2.3.32 env: DETACH_RUNNER: true + E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || inputs.chainlink_version || github.sha }} + E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }} + E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }} + E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }} + E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }} + E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} + E2E_TEST_LOG_COLLECT: ${{ vars.TEST_LOG_COLLECT }} + E2E_TEST_CHAINLINK_IMAGE: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_IMAGE || env.CHAINLINK_IMAGE }} + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }} + E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary" + E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} + E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} + E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} + E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} with: test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs - test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_download_vendor_packages_command: cd $(dirname ${{ matrix.tests.path }}) && go mod download test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }} + test_config_override_path: ${{ matrix.tests.test_config_override_path }} # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 # test_config_override_base64: ${{ inputs.test_config_override_base64 }} - test_config_chainlink_version: ${{ matrix.tests.test_inputs.chainlink_version || inputs.chainlink_version || github.sha }} - test_config_chainlink_upgrade_version: ${{ matrix.tests.test_inputs.chainlink_upgrade_version }} - test_config_chainlink_postgres_version: ${{ matrix.tests.test_inputs.chainlink_postgres_version }} - test_config_selected_networks: ${{ matrix.tests.test_inputs.selected_networks || env.SELECTED_NETWORKS}} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_type: ${{ matrix.tests.test_inputs.test_type }} - test_suite: ${{ matrix.tests.test_inputs.test_suite }} + test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }} + test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} artifacts_name: ${{ matrix.tests.id_sanitized }}-test-logs artifacts_location: | @@ -435,7 +522,7 @@ jobs: /tmp/gotest.log publish_check_name: ${{ matrix.tests.id_sanitized }} token: ${{ secrets.GH_TOKEN }} - no_cache: true # Do not restore cache since go was already configured in the previous step + cache_key_id: e2e-tests go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} @@ -443,19 +530,24 @@ jobs: should_tidy: "false" go_coverage_src_dir: /var/tmp/go-coverage go_coverage_dest_dir: ${{ github.workspace }}/.covdata - DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.test_inputs.chainlink_image || env.CHAINLINK_IMAGE }} - DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_inputs.chainlink_upgrade_image }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} - DEFAULT_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} + - name: Show Otel-Collector logs + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true' + run: | + docker logs otel-collector + + - name: Permissions on traces + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true' + run: | + ls -l ./integration-tests/smoke/traces + + - name: Upload trace data as Github artifact + if: inputs.enable_otel_traces_for_ocr2_plugins && matrix.tests.test_env_vars.ENABLE_OTEL_TRACES == 'true' + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: trace-data + path: ./integration-tests/smoke/traces/trace-data.json + - name: Upload test log as Github artifact uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 if: inputs.test_log_upload_on_failure && failure() @@ -464,6 +556,35 @@ jobs: path: /tmp/gotest.log retention-days: ${{ inputs.test_log_upload_retention_days }} continue-on-error: true + + - name: Upload cl node coverage data as Github artifact + if: inputs.upload_cl_node_coverage_artifact + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + timeout-minutes: 2 + continue-on-error: true + with: + name: ${{ inputs.upload_cl_node_coverage_artifact_prefix }}${{ matrix.tests.id_sanitized }} + path: .covdata + retention-days: 1 + + - name: Record test result + if: ${{ always() }} + run: | + id="${{ matrix.tests.id }}" + result="${{ steps.run_tests.outcome }}" + echo "{\"id\": \"$id\", \"result\": \"$result\"}" > test_result.json + + - name: Upload test result as artifact + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: test_result_${{ needs.load-test-configurations.outputs.workflow_id }}_${{ matrix.tests.id_sanitized }} + path: test_result.json + retention-days: 1 + + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25 + # Run K8s tests using old remote runner @@ -529,7 +650,7 @@ jobs: org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} - this-job-name: Run E2E Tests / Run ${{ matrix.tests.id }} + this-job-name: ${{ inputs.workflow_name }} / Run ${{ matrix.tests.id }} continue-on-error: true - name: Checkout repository @@ -543,7 +664,8 @@ jobs: echo "Remote Runner Version: ${{ needs.prepare-remote-runner-test-image.outputs.remote-runner-version }}" - name: Run tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d2f9642bcc24a73400568756f24b72c188ac7a9a # v2.3.31 + id: run_tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@00c164251be2a7c5b2b23a6e5f7014982f232c14 # v2.3.32 env: DETACH_RUNNER: true RR_MEM: ${{ matrix.tests.remote_runner_memory }} @@ -553,44 +675,42 @@ jobs: # We can comment these out when we have a stable soak test and aren't worried about resource consumption TEST_UPLOAD_CPU_PROFILE: true TEST_UPLOAD_MEM_PROFILE: true - TEST_LOG_LEVEL: debug REF_NAME: ${{ github.head_ref || github.ref_name }} + E2E_TEST_CHAINLINK_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_VERSION || inputs.chainlink_version || github.sha }} + E2E_TEST_CHAINLINK_UPGRADE_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_VERSION }} + E2E_TEST_CHAINLINK_POSTGRES_VERSION: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_POSTGRES_VERSION }} + E2E_TEST_SELECTED_NETWORK: ${{ matrix.tests.test_env_vars.E2E_TEST_SELECTED_NETWORK || env.SELECTED_NETWORKS }} + E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }} + E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} + E2E_TEST_LOG_COLLECT: ${{ vars.TEST_LOG_COLLECT }} + E2E_TEST_CHAINLINK_IMAGE: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_IMAGE || env.CHAINLINK_IMAGE }} + E2E_TEST_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_env_vars.E2E_TEST_CHAINLINK_UPGRADE_IMAGE }} + E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push + E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + E2E_TEST_GRAFANA_BASE_URL: "http://localhost:8080/primary" + E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + E2E_TEST_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} + E2E_TEST_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} + E2E_TEST_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} + E2E_TEST_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} with: test_command_to_run: ${{ matrix.tests.test_cmd }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: make gomod test_secrets_override_base64: ${{ secrets.TEST_SECRETS_OVERRIDE_BASE64 }} - # TODO: Uncomment once Test Config does not have any secrets. Related ticket https://smartcontract-it.atlassian.net/browse/TT-1392 - # test_config_override_base64: ${{ inputs.test_config_override_base64 }} - test_config_chainlink_version: ${{ matrix.tests.test_inputs.chainlink_version || inputs.chainlink_version || github.sha }} - test_config_chainlink_upgrade_version: ${{ matrix.tests.test_inputs.chainlink_upgrade_version }} - test_config_chainlink_postgres_version: ${{ matrix.tests.test_inputs.chainlink_postgres_version }} - test_config_selected_networks: ${{ matrix.tests.test_inputs.selected_networks || env.SELECTED_NETWORKS}} - test_config_logging_run_id: ${{ github.run_id }} - test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - test_type: ${{ matrix.tests.test_inputs.test_type }} - test_suite: ${{ matrix.tests.test_inputs.test_suite }} + test_type: ${{ matrix.tests.test_env_vars.TEST_TYPE }} + test_suite: ${{ matrix.tests.test_env_vars.TEST_SUITE }} token: ${{ secrets.GH_TOKEN }} should_cleanup: false - no_cache: true # Do not restore cache since go was already configured in the previous step + cache_key_id: e2e-tests go_mod_path: ./integration-tests/go.mod QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} - DEFAULT_CHAINLINK_IMAGE: ${{ matrix.tests.test_inputs.chainlink_image || env.CHAINLINK_IMAGE }} - DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ matrix.tests.test_inputs.chainlink_upgrade_image }} - DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} - DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" - DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} - DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_INSTANCE || '' }} - DEFAULT_PYROSCOPE_KEY: ${{ matrix.tests.pyroscope_env != '' && secrets.QA_PYROSCOPE_KEY || '' }} - DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env != '' && 'true' || '' }} - name: Upload test log as Github artifact - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 if: inputs.test_log_upload_on_failure && failure() with: name: test_log_${{ matrix.tests.id_sanitized }} @@ -598,41 +718,40 @@ jobs: retention-days: ${{ inputs.test_log_upload_retention_days }} continue-on-error: true + # TODO: move to run-tests GHA + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@70ccaef155381025e411cf7cd1fa5ef8f668ed75 # v2.3.25 + after_tests: - needs: [run-docker-tests, run-k8s-runner-tests] + needs: [load-test-configurations, run-docker-tests, run-k8s-runner-tests] if: always() - name: After tests notifications + name: After tests runs-on: ubuntu-latest + outputs: + test_results: ${{ steps.set_test_results.outputs.results }} steps: - - name: Determine combined test results - id: combine_results + - name: Download all test result artifacts + uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6 + with: + path: test_results + pattern: test_result_${{ needs.load-test-configurations.outputs.workflow_id }}_* + + - name: Set detailed test results + id: set_test_results run: | - docker_result="${{ needs.run-docker-tests.result }}" - k8s_result="${{ needs.run-k8s-runner-tests.result }}" - - function map_outcome { - case "$1" in - success|skipped) - echo "success" - ;; - cancelled) - echo "cancelled" - ;; - *) - echo "failure" - ;; - esac - } - - combined_docker_result=$(map_outcome $docker_result) - combined_k8s_result=$(map_outcome $k8s_result) - - if [[ $combined_docker_result == "failure" || $combined_k8s_result == "failure" ]]; then - echo "result=failure" >> $GITHUB_OUTPUT - elif [[ $combined_docker_result == "cancelled" || $combined_k8s_result == "cancelled" ]]; then - echo "result=cancelled" >> $GITHUB_OUTPUT + if [ -d "test_results" ]; then + cd test_results + ls -R . + # Combine JSON files into one + find . -name '*.json' -exec cat {} + | jq -s '.' > test_results.json + # Display the combined JSON + jq . test_results.json + # Set the combined results as an output + echo "results=$(jq -c . test_results.json)" >> $GITHUB_OUTPUT else - echo "result=success" >> $GITHUB_OUTPUT + echo "No test results directory found." + echo "results=[]" >> $GITHUB_OUTPUT fi - name: Send Slack notification diff --git a/.gitignore b/.gitignore index 00962a94a39..bd7a6b72cde 100644 --- a/.gitignore +++ b/.gitignore @@ -74,7 +74,6 @@ db_dumps/ integration-tests/**/traces/ benchmark_report.csv benchmark_summary.json -integration-tests/citool/output.csv secrets.toml tmp_laneconfig/ diff --git a/integration-tests/ccip-tests/testconfig/global.go b/integration-tests/ccip-tests/testconfig/global.go index a1658a48416..25e87bac2aa 100644 --- a/integration-tests/ccip-tests/testconfig/global.go +++ b/integration-tests/ccip-tests/testconfig/global.go @@ -174,6 +174,36 @@ type Common struct { func (p *Common) ReadFromEnvVar() error { logger := logging.GetTestLogger(nil) + testLogCollect := ctfconfig.MustReadEnvVar_Boolean(ctfconfig.E2E_TEST_LOG_COLLECT_ENV) + if testLogCollect != nil { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.TestLogCollect", ctfconfig.E2E_TEST_LOG_COLLECT_ENV) + p.Logging.TestLogCollect = testLogCollect + } + + loggingRunID := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOGGING_RUN_ID_ENV) + if loggingRunID != "" { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.RunID", ctfconfig.E2E_TEST_LOGGING_RUN_ID_ENV) + p.Logging.RunId = &loggingRunID + } + + logstreamLogTargets := ctfconfig.MustReadEnvVar_Strings(ctfconfig.E2E_TEST_LOG_STREAM_LOG_TARGETS_ENV, ",") + if len(logstreamLogTargets) > 0 { + if p.Logging == nil { + p.Logging = &ctfconfig.LoggingConfig{} + } + if p.Logging.LogStream == nil { + p.Logging.LogStream = &ctfconfig.LogStreamConfig{} + } + logger.Debug().Msgf("Using %s env var to override Logging.LogStream.LogTargets", ctfconfig.E2E_TEST_LOG_STREAM_LOG_TARGETS_ENV) + p.Logging.LogStream.LogTargets = logstreamLogTargets + } + lokiTenantID := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_LOKI_TENANT_ID_ENV) if lokiTenantID != "" { if p.Logging == nil { @@ -258,6 +288,15 @@ func (p *Common) ReadFromEnvVar() error { p.Logging.Grafana.BearerToken = &grafanaBearerToken } + selectedNetworks := ctfconfig.MustReadEnvVar_Strings(ctfconfig.E2E_TEST_SELECTED_NETWORK_ENV, ",") + if len(selectedNetworks) > 0 { + if p.Network == nil { + p.Network = &ctfconfig.NetworkConfig{} + } + logger.Debug().Msgf("Using %s env var to override Network.SelectedNetworks", ctfconfig.E2E_TEST_SELECTED_NETWORK_ENV) + p.Network.SelectedNetworks = selectedNetworks + } + walletKeys := ctfconfig.ReadEnvVarGroupedMap(ctfconfig.E2E_TEST_WALLET_KEY_ENV, ctfconfig.E2E_TEST_WALLET_KEYS_ENV) if len(walletKeys) > 0 { if p.Network == nil { @@ -301,6 +340,38 @@ func (p *Common) ReadFromEnvVar() error { p.NewCLCluster.Common.ChainlinkImage.Image = &chainlinkImage } + chainlinkVersion := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_VERSION_ENV) + if chainlinkVersion != "" { + if p.NewCLCluster == nil { + p.NewCLCluster = &ChainlinkDeployment{} + } + if p.NewCLCluster.Common == nil { + p.NewCLCluster.Common = &Node{} + } + if p.NewCLCluster.Common.ChainlinkImage == nil { + p.NewCLCluster.Common.ChainlinkImage = &ctfconfig.ChainlinkImageConfig{} + } + + logger.Debug().Msgf("Using %s env var to override NewCLCluster.Common.ChainlinkImage.Version", ctfconfig.E2E_TEST_CHAINLINK_VERSION_ENV) + p.NewCLCluster.Common.ChainlinkImage.Version = &chainlinkVersion + } + + chainlinkPostgresVersion := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_POSTGRES_VERSION_ENV) + if chainlinkPostgresVersion != "" { + if p.NewCLCluster == nil { + p.NewCLCluster = &ChainlinkDeployment{} + } + if p.NewCLCluster.Common == nil { + p.NewCLCluster.Common = &Node{} + } + if p.NewCLCluster.Common.ChainlinkImage == nil { + p.NewCLCluster.Common.ChainlinkImage = &ctfconfig.ChainlinkImageConfig{} + } + + logger.Debug().Msgf("Using %s env var to override NewCLCluster.Common.ChainlinkImage.PostgresVersion", ctfconfig.E2E_TEST_CHAINLINK_POSTGRES_VERSION_ENV) + p.NewCLCluster.Common.ChainlinkImage.PostgresVersion = &chainlinkPostgresVersion + } + chainlinkUpgradeImage := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_UPGRADE_IMAGE_ENV) if chainlinkUpgradeImage != "" { if p.NewCLCluster == nil { @@ -317,6 +388,22 @@ func (p *Common) ReadFromEnvVar() error { p.NewCLCluster.Common.ChainlinkUpgradeImage.Image = &chainlinkUpgradeImage } + chainlinkUpgradeVersion := ctfconfig.MustReadEnvVar_String(ctfconfig.E2E_TEST_CHAINLINK_UPGRADE_VERSION_ENV) + if chainlinkUpgradeVersion != "" { + if p.NewCLCluster == nil { + p.NewCLCluster = &ChainlinkDeployment{} + } + if p.NewCLCluster.Common == nil { + p.NewCLCluster.Common = &Node{} + } + if p.NewCLCluster.Common.ChainlinkImage == nil { + p.NewCLCluster.Common.ChainlinkImage = &ctfconfig.ChainlinkImageConfig{} + } + + logger.Debug().Msgf("Using %s env var to override NewCLCluster.Common.ChainlinkUpgradeImage.Version", ctfconfig.E2E_TEST_CHAINLINK_UPGRADE_VERSION_ENV) + p.NewCLCluster.Common.ChainlinkUpgradeImage.Version = &chainlinkUpgradeVersion + } + return nil } diff --git a/integration-tests/citool/cmd/check_tests_cmd.go b/integration-tests/citool/cmd/check_tests_cmd.go deleted file mode 100644 index 3ef3712a572..00000000000 --- a/integration-tests/citool/cmd/check_tests_cmd.go +++ /dev/null @@ -1,166 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - "path/filepath" - "regexp" - "strings" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" -) - -type JobConfig struct { - Jobs map[string]struct { - Strategy struct { - Matrix struct { - Test []struct { - Path string `yaml:"path"` - TestOpts string `yaml:"testOpts"` - } `yaml:"test"` - } `yaml:"matrix"` - } `yaml:"strategy"` - } `yaml:"jobs"` -} - -var checkTestsCmd = &cobra.Command{ - Use: "check-tests [directory] [yaml file]", - Short: "Check if all tests in a directory are included in the test configurations YAML file", - Args: cobra.ExactArgs(2), - Run: func(_ *cobra.Command, args []string) { - directory := args[0] - yamlFile := args[1] - excludedDirs := []string{"../../citool"} - tests, err := extractTests(directory, excludedDirs) - if err != nil { - fmt.Println("Error extracting tests:", err) - os.Exit(1) - } - - checkTestsInPipeline(yamlFile, tests) - }, -} - -// extractTests scans the given directory and subdirectories (except the excluded ones) -// for Go test files, extracts test function names, and returns a slice of Test. -func extractTests(dir string, excludeDirs []string) ([]Test, error) { - var tests []Test - - // Resolve to absolute path - absDir, err := filepath.Abs(dir) - if err != nil { - return nil, err - } - - // filepath.WalkDir provides more control and is more efficient for skipping directories - err = filepath.WalkDir(absDir, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - - // Check if the current path is one of the excluded directories - for _, exclude := range excludeDirs { - absExclude, _ := filepath.Abs(exclude) - if strings.HasPrefix(path, absExclude) { - if d.IsDir() { - return filepath.SkipDir // Skip this directory - } - return nil // Skip this file - } - } - - if !d.IsDir() && strings.HasSuffix(d.Name(), "_test.go") { - content, err := os.ReadFile(path) - if err != nil { - return err - } - re := regexp.MustCompile(`func (Test\w+)`) - matches := re.FindAllSubmatch(content, -1) - for _, match := range matches { - funcName := string(match[1]) - if funcName == "TestMain" { // Skip "TestMain" - continue - } - tests = append(tests, Test{ - Name: funcName, - Path: mustExtractSubpath(path, "integration-tests"), - }) - } - } - return nil - }) - - return tests, err -} - -// ExtractSubpath extracts a specific subpath from a given full path. -// If the subpath is not found, it returns an error. -func mustExtractSubpath(fullPath, subPath string) string { - index := strings.Index(fullPath, subPath) - if index == -1 { - panic("subpath not found in the provided full path") - } - return fullPath[index:] -} - -func checkTestsInPipeline(yamlFile string, tests []Test) { - data, err := os.ReadFile(yamlFile) - if err != nil { - fmt.Printf("Error reading YAML file: %s\n", err) - return - } - - var config Config - err = yaml.Unmarshal(data, &config) - if err != nil { - fmt.Printf("Error parsing YAML: %s\n", err) - return - } - - missingTests := []string{} // Track missing tests - - for _, test := range tests { - found := false - for _, item := range config.Tests { - if item.Path == test.Path { - if strings.Contains(item.TestCmd, "-test.run") { - if matchTestNameInCmd(item.TestCmd, test.Name) { - found = true - break - } - } else { - found = true - break - } - } - } - if !found { - missingTests = append(missingTests, fmt.Sprintf("ERROR: Test '%s' in file '%s' does not have CI configuration in '%s'", test.Name, test.Path, yamlFile)) - } - } - - if len(missingTests) > 0 { - for _, missing := range missingTests { - fmt.Println(missing) - } - os.Exit(1) // Exit with a failure status - } -} - -// matchTestNameInCmd checks if the given test name matches the -test.run pattern in the command string. -func matchTestNameInCmd(cmd string, testName string) bool { - testRunRegex := regexp.MustCompile(`-test\.run ([^\s]+)`) - matches := testRunRegex.FindStringSubmatch(cmd) - if len(matches) > 1 { - // Extract the regex pattern used in the -test.run command - pattern := matches[1] - - // Escape regex metacharacters in the testName before matching - escapedTestName := regexp.QuoteMeta(testName) - - // Check if the escaped test name matches the extracted pattern - return regexp.MustCompile(pattern).MatchString(escapedTestName) - } - return false -} diff --git a/integration-tests/citool/cmd/check_tests_cmd_test.go b/integration-tests/citool/cmd/check_tests_cmd_test.go deleted file mode 100644 index 4b7f50e7f06..00000000000 --- a/integration-tests/citool/cmd/check_tests_cmd_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package cmd - -import ( - "testing" -) - -func TestMatchTestNameInCmd(t *testing.T) { - tests := []struct { - cmd string - testName string - expected bool - }{ - {"go test -test.run ^TestExample$", "TestExample", true}, - {"go test -test.run ^TestExample$", "TestAnother", false}, - {"go test -test.run ^TestExample$ -v", "TestExample", true}, - {"go test -test.run ^TestExamplePart$", "TestExample", false}, - {"go test -test.run ^TestWithNumbers123$", "TestWithNumbers123", true}, - {"go test -test.run ^Test_With_Underscores$", "Test_With_Underscores", true}, - {"go test -test.run ^Test-With-Dash$", "Test-With-Dash", true}, - {"go test -test.run ^TestWithSpace Space$", "TestWithSpace Space", true}, - {"go test -test.run ^TestWithNewline\nNewline$", "TestWithNewline\nNewline", true}, - {"go test -test.run ^TestOne$|^TestTwo$", "TestOne", true}, - {"go test -test.run ^TestOne$|^TestTwo$", "TestTwo", true}, - {"go test -test.run ^TestOne$|^TestTwo$", "TestThree", false}, - {"go test -test.run TestOne|TestTwo", "TestTwo", true}, - {"go test -test.run TestOne|TestTwo", "TestOne", true}, - {"go test -test.run TestOne|TestTwo|TestThree", "TestFour", false}, - {"go test -test.run ^TestOne$|TestTwo$", "TestTwo", true}, - {"go test -test.run ^TestOne$|TestTwo|TestThree$", "TestThree", true}, - {"go test -test.run TestOne|TestTwo|TestThree", "TestOne", true}, - {"go test -test.run TestOne|TestTwo|TestThree", "TestThree", true}, - {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestA", true}, - {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestB", true}, - {"go test -test.run ^TestA$|^TestB$|^TestC$", "TestD", false}, - {"go test -test.run TestA|^TestB$|TestC", "TestB", true}, - {"go test -test.run ^TestA|^TestB|TestC$", "TestA", true}, - {"go test -test.run ^TestA|^TestB|TestC$", "TestC", true}, - {"go test -test.run ^TestA|^TestB|TestC$", "TestD", false}, - } - - for _, tt := range tests { - result := matchTestNameInCmd(tt.cmd, tt.testName) - if result != tt.expected { - t.Errorf("matchTestNameInCmd(%s, %s) = %t; expected %t", tt.cmd, tt.testName, result, tt.expected) - } - } -} diff --git a/integration-tests/citool/cmd/create_test_config_cmd.go b/integration-tests/citool/cmd/create_test_config_cmd.go deleted file mode 100644 index c0cd91b05fb..00000000000 --- a/integration-tests/citool/cmd/create_test_config_cmd.go +++ /dev/null @@ -1,179 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/pelletier/go-toml/v2" - "github.com/spf13/cobra" - - ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" - ctf_config_types "github.com/smartcontractkit/chainlink-testing-framework/config/types" -) - -var createTestConfigCmd = &cobra.Command{ - Use: "create", - Short: "Create a test config from the provided flags", - Run: func(cmd *cobra.Command, _ []string) { - var tc ctf_config.TestConfig - - var version, postgresVersion *string - if cmd.Flags().Changed(ChainlinkVersionFlag) { - version = &oc.ChainlinkVersion - } - if cmd.Flags().Changed(ChainlinkPostgresVersionFlag) { - version = &oc.ChainlinkPostgresVersion - } - if version != nil || postgresVersion != nil { - tc.ChainlinkImage = &ctf_config.ChainlinkImageConfig{ - Version: version, - PostgresVersion: postgresVersion, - } - } - - var upgradeVersion *string - if cmd.Flags().Changed(ChainlinkUpgradeVersionFlag) { - upgradeVersion = &oc.ChainlinkUpgradeVersion - } - if upgradeVersion != nil { - tc.ChainlinkUpgradeImage = &ctf_config.ChainlinkImageConfig{ - Version: upgradeVersion, - } - } - - var selectedNetworks *[]string - if cmd.Flags().Changed(SelectedNetworksFlag) { - selectedNetworks = &oc.SelectedNetworks - } - if selectedNetworks != nil { - tc.Network = &ctf_config.NetworkConfig{ - SelectedNetworks: oc.SelectedNetworks, - } - } - - var peryscopeEnabled *bool - var pyroscopeServerURL, pyroscopeEnvironment, pyroscopeKey *string - if cmd.Flags().Changed(PyroscopeEnabledFlag) { - peryscopeEnabled = &oc.PyroscopeEnabled - } - if cmd.Flags().Changed(PyroscopeServerURLFlag) { - pyroscopeServerURL = &oc.PyroscopeServerURL - } - if cmd.Flags().Changed(PyroscopeKeyFlag) { - pyroscopeKey = &oc.PyroscopeKey - } - if cmd.Flags().Changed(PyroscopeEnvironmentFlag) { - pyroscopeEnvironment = &oc.PyroscopeEnvironment - } - if peryscopeEnabled != nil { - tc.Pyroscope = &ctf_config.PyroscopeConfig{ - Enabled: peryscopeEnabled, - ServerUrl: pyroscopeServerURL, - Environment: pyroscopeEnvironment, - Key: pyroscopeKey, - } - } - - var testLogCollect *bool - if cmd.Flags().Changed(LoggingTestLogCollectFlag) { - testLogCollect = &oc.LoggingTestLogCollect - } - var loggingRunID *string - if cmd.Flags().Changed(LoggingRunIDFlag) { - loggingRunID = &oc.LoggingRunID - } - var loggingLogTargets []string - if cmd.Flags().Changed(LoggingLogTargetsFlag) { - loggingLogTargets = oc.LoggingLogTargets - } - var loggingLokiTenantID *string - if cmd.Flags().Changed(LoggingLokiTenantIDFlag) { - loggingLokiTenantID = &oc.LoggingLokiTenantID - } - var loggingLokiBasicAuth *string - if cmd.Flags().Changed(LoggingLokiBasicAuthFlag) { - loggingLokiBasicAuth = &oc.LoggingLokiBasicAuth - } - var loggingLokiEndpoint *string - if cmd.Flags().Changed(LoggingLokiEndpointFlag) { - loggingLokiEndpoint = &oc.LoggingLokiEndpoint - } - var loggingGrafanaBaseURL *string - if cmd.Flags().Changed(LoggingGrafanaBaseURLFlag) { - loggingGrafanaBaseURL = &oc.LoggingGrafanaBaseURL - } - var loggingGrafanaDashboardURL *string - if cmd.Flags().Changed(LoggingGrafanaDashboardURLFlag) { - loggingGrafanaDashboardURL = &oc.LoggingGrafanaDashboardURL - } - var loggingGrafanaBearerToken *string - if cmd.Flags().Changed(LoggingGrafanaBearerTokenFlag) { - loggingGrafanaBearerToken = &oc.LoggingGrafanaBearerToken - } - - if testLogCollect != nil || loggingRunID != nil || loggingLogTargets != nil || loggingLokiEndpoint != nil || loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil { - tc.Logging = &ctf_config.LoggingConfig{} - tc.Logging.TestLogCollect = testLogCollect - tc.Logging.RunId = loggingRunID - if loggingLogTargets != nil { - tc.Logging.LogStream = &ctf_config.LogStreamConfig{ - LogTargets: loggingLogTargets, - } - } - if loggingLokiTenantID != nil || loggingLokiBasicAuth != nil || loggingLokiEndpoint != nil { - tc.Logging.Loki = &ctf_config.LokiConfig{ - TenantId: loggingLokiTenantID, - BasicAuth: loggingLokiBasicAuth, - Endpoint: loggingLokiEndpoint, - } - } - if loggingGrafanaBaseURL != nil || loggingGrafanaDashboardURL != nil || loggingGrafanaBearerToken != nil { - tc.Logging.Grafana = &ctf_config.GrafanaConfig{ - BaseUrl: loggingGrafanaBaseURL, - DashboardUrl: loggingGrafanaDashboardURL, - BearerToken: loggingGrafanaBearerToken, - } - } - } - - var privateEthereumNetworkExecutionLayer *string - if cmd.Flags().Changed(PrivateEthereumNetworkExecutionLayerFlag) { - privateEthereumNetworkExecutionLayer = &oc.PrivateEthereumNetworkExecutionLayer - } - var privateEthereumNetworkEthereumVersion *string - if cmd.Flags().Changed(PrivateEthereumNetworkEthereumVersionFlag) { - privateEthereumNetworkEthereumVersion = &oc.PrivateEthereumNetworkEthereumVersion - } - var privateEthereumNetworkCustomDockerImage *string - if cmd.Flags().Changed(PrivateEthereumNetworkCustomDockerImageFlag) { - privateEthereumNetworkCustomDockerImage = &oc.PrivateEthereumNetworkCustomDockerImages - } - if privateEthereumNetworkExecutionLayer != nil || privateEthereumNetworkEthereumVersion != nil || privateEthereumNetworkCustomDockerImage != nil { - var el ctf_config_types.ExecutionLayer - if privateEthereumNetworkExecutionLayer != nil { - el = ctf_config_types.ExecutionLayer(*privateEthereumNetworkExecutionLayer) - } - var ev ctf_config_types.EthereumVersion - if privateEthereumNetworkEthereumVersion != nil { - ev = ctf_config_types.EthereumVersion(*privateEthereumNetworkEthereumVersion) - } - var customImages map[ctf_config.ContainerType]string - if privateEthereumNetworkCustomDockerImage != nil { - customImages = map[ctf_config.ContainerType]string{"execution_layer": *privateEthereumNetworkCustomDockerImage} - } - tc.PrivateEthereumNetwork = &ctf_config.EthereumNetworkConfig{ - ExecutionLayer: &el, - EthereumVersion: &ev, - CustomDockerImages: customImages, - } - } - - configToml, err := toml.Marshal(tc) - if err != nil { - fmt.Fprintf(os.Stderr, "Error marshalling TestConfig to TOML: %v\n", err) - os.Exit(1) - } - - fmt.Fprintln(cmd.OutOrStdout(), string(configToml)) - }, -} diff --git a/integration-tests/citool/cmd/csv_export_cmd.go b/integration-tests/citool/cmd/csv_export_cmd.go deleted file mode 100644 index 8fe13440c81..00000000000 --- a/integration-tests/citool/cmd/csv_export_cmd.go +++ /dev/null @@ -1,96 +0,0 @@ -package cmd - -import ( - "encoding/csv" - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v3" -) - -var csvExportCmd = &cobra.Command{ - Use: "csvexport", - Short: "Export tests to CSV format", - Run: func(cmd *cobra.Command, _ []string) { - configFile, _ := cmd.Flags().GetString("file") - if err := exportConfigToCSV(configFile); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } - }, -} - -func init() { - csvExportCmd.Flags().StringP("file", "f", "", "Path to YML file") - err := csvExportCmd.MarkFlagRequired("file") - if err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func exportConfigToCSV(configFile string) error { - // Read the YAML file - bytes, err := os.ReadFile(configFile) - if err != nil { - return err - } - - // Unmarshal the YAML into the Config struct - var config Config - if err := yaml.Unmarshal(bytes, &config); err != nil { - return err - } - - // Create a CSV file - file, err := os.Create("output.csv") - if err != nil { - return err - } - defer file.Close() - - writer := csv.NewWriter(file) - defer writer.Flush() - - // Write CSV headers - headers := []string{"ID", "Test Path", "Test Env Type", "Runs On", "Test Cmd", "Test Config Override Required", "Test Secrets Required", "Remote Runner Memory", "Pyroscope Env", "Workflows", "Test Inputs"} - if err := writer.Write(headers); err != nil { - return err - } - - // Iterate over Tests and write data to CSV - for _, test := range config.Tests { - workflows := strings.Join(test.Workflows, ", ") // Combine workflows into a single CSV field - // Serialize TestInputs - testInputs := serializeMap(test.TestInputs) - - record := []string{ - test.ID, - test.Path, - test.TestEnvType, - test.RunsOn, - test.TestCmd, - fmt.Sprintf("%t", test.TestConfigOverrideRequired), - fmt.Sprintf("%t", test.TestSecretsRequired), - test.RemoteRunnerMemory, - test.PyroscopeEnv, - workflows, - testInputs, - } - if err := writer.Write(record); err != nil { - return err - } - } - - return nil -} - -func serializeMap(inputs map[string]string) string { - pairs := make([]string, 0, len(inputs)) - for key, value := range inputs { - pairs = append(pairs, fmt.Sprintf("%s=%s", key, value)) - } - return strings.Join(pairs, ", ") -} diff --git a/integration-tests/citool/cmd/filter_cmd.go b/integration-tests/citool/cmd/filter_cmd.go deleted file mode 100644 index c1d5f22357d..00000000000 --- a/integration-tests/citool/cmd/filter_cmd.go +++ /dev/null @@ -1,165 +0,0 @@ -package cmd - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "log" - "os" - "regexp" - "strings" - - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - - "github.com/smartcontractkit/chainlink-testing-framework/utils" -) - -// Filter tests based on workflow, test type, and test IDs. -func filterTests(allTests []CITestConf, workflow, testType, ids string, envresolve bool) []CITestConf { - workflowFilter := workflow - typeFilter := testType - idFilter := strings.Split(ids, ",") - - var filteredTests []CITestConf - - for _, test := range allTests { - workflowMatch := workflow == "" || contains(test.Workflows, workflowFilter) - typeMatch := testType == "" || test.TestEnvType == typeFilter - idMatch := ids == "*" || ids == "" || contains(idFilter, test.ID) - - if workflowMatch && typeMatch && idMatch { - test.IDSanitized = sanitizeTestID(test.ID) - filteredTests = append(filteredTests, test) - } - if envresolve { - for k, v := range test.TestInputs { - test.TestInputs[k] = utils.MustResolveEnvPlaceholder(v) - } - } - } - - return filteredTests -} - -func filterAndMergeTests(allTests []CITestConf, workflow, testType, base64Tests string, envresolve bool) ([]CITestConf, error) { - decodedBytes, err := base64.StdEncoding.DecodeString(base64Tests) - if err != nil { - return nil, err - } - var decodedTests []CITestConf - err = yaml.Unmarshal(decodedBytes, &decodedTests) - if err != nil { - return nil, err - } - - idFilter := make(map[string]CITestConf) - for _, dt := range decodedTests { - idFilter[dt.ID] = dt - } - - var filteredTests []CITestConf - for _, test := range allTests { - workflowMatch := workflow == "" || contains(test.Workflows, workflow) - typeMatch := testType == "" || test.TestEnvType == testType - - if decodedTest, exists := idFilter[test.ID]; exists && workflowMatch && typeMatch { - // Override test inputs from the base64 encoded tests - for k, v := range decodedTest.TestInputs { - if test.TestInputs == nil { - test.TestInputs = make(map[string]string) - } - test.TestInputs[k] = v - } - test.IDSanitized = sanitizeTestID(test.ID) - filteredTests = append(filteredTests, test) - } - if envresolve { - for k, v := range test.TestInputs { - test.TestInputs[k] = utils.MustResolveEnvPlaceholder(v) - } - } - } - - return filteredTests, nil -} - -func sanitizeTestID(id string) string { - // Define a regular expression that matches any character not a letter, digit, hyphen - re := regexp.MustCompile(`[^a-zA-Z0-9-_]+`) - // Replace all occurrences of disallowed characters with "_" - return re.ReplaceAllString(id, "_") -} - -// Utility function to check if a slice contains a string. -func contains(slice []string, element string) bool { - for _, s := range slice { - if s == element { - return true - } - } - return false -} - -// filterCmd represents the filter command -var filterCmd = &cobra.Command{ - Use: "filter", - Short: "Filter test configurations based on specified criteria", - Long: `Filters tests from a YAML configuration based on name, workflow, test type, and test IDs. -Example usage: -./e2e_tests_tool filter --file .github/e2e-tests.yml --workflow "Run Nightly E2E Tests" --test-env-type "docker" --test-ids "test1,test2"`, - Run: func(cmd *cobra.Command, _ []string) { - yamlFile, _ := cmd.Flags().GetString("file") - workflow, _ := cmd.Flags().GetString("workflow") - testType, _ := cmd.Flags().GetString("test-env-type") - testIDs, _ := cmd.Flags().GetString("test-ids") - testMap, _ := cmd.Flags().GetString("test-list") - envresolve, _ := cmd.Flags().GetBool("envresolve") - - data, err := os.ReadFile(yamlFile) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading YAML file: %v\n", err) - os.Exit(1) - } - - var config Config - err = yaml.Unmarshal(data, &config) - if err != nil { - fmt.Fprintf(os.Stderr, "Error parsing YAML file %s data: %v\n", yamlFile, err) - os.Exit(1) - } - - var filteredTests []CITestConf - if testMap == "" { - filteredTests = filterTests(config.Tests, workflow, testType, testIDs, envresolve) - } else { - filteredTests, err = filterAndMergeTests(config.Tests, workflow, testType, testMap, envresolve) - if err != nil { - log.Fatalf("Error filtering and merging tests: %v", err) - } - } - matrix := map[string][]CITestConf{"tests": filteredTests} - matrixJSON, err := json.Marshal(matrix) - if err != nil { - fmt.Fprintf(os.Stderr, "Error marshaling matrix to JSON: %v\n", err) - os.Exit(1) - } - - fmt.Printf("%s", matrixJSON) - }, -} - -func init() { - filterCmd.Flags().StringP("file", "f", "", "Path to the YAML file") - filterCmd.Flags().String("test-list", "", "Base64 encoded list of tests (YML objects) to filter by. Can include test_inputs for each test.") - filterCmd.Flags().StringP("test-ids", "i", "*", "Comma-separated list of test IDs to filter by") - filterCmd.Flags().StringP("test-env-type", "y", "", "Type of test to filter by") - filterCmd.Flags().StringP("workflow", "t", "", "Workflow filter") - filterCmd.Flags().Bool("envresolve", false, "Resolve environment variables in test inputs") - - err := filterCmd.MarkFlagRequired("file") - if err != nil { - fmt.Fprintf(os.Stderr, "Error marking flag as required: %v\n", err) - os.Exit(1) - } -} diff --git a/integration-tests/citool/cmd/filter_cmd_test.go b/integration-tests/citool/cmd/filter_cmd_test.go deleted file mode 100644 index ff6e9c981de..00000000000 --- a/integration-tests/citool/cmd/filter_cmd_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package cmd - -import ( - "testing" -) - -func TestFilterTestsByID(t *testing.T) { - tests := []CITestConf{ - {ID: "run_all_in_ocr_tests_go", TestEnvType: "docker"}, - {ID: "run_all_in_ocr2_tests_go", TestEnvType: "docker"}, - {ID: "run_all_in_ocr3_tests_go", TestEnvType: "k8s_remote_runner"}, - } - - cases := []struct { - description string - inputIDs string - expectedLen int - }{ - {"Filter by single ID", "run_all_in_ocr_tests_go", 1}, - {"Filter by multiple IDs", "run_all_in_ocr_tests_go,run_all_in_ocr2_tests_go", 2}, - {"Wildcard to include all", "*", 3}, - {"Empty ID string to include all", "", 3}, - } - - for _, c := range cases { - t.Run(c.description, func(t *testing.T) { - filtered := filterTests(tests, "", "", c.inputIDs, false) - if len(filtered) != c.expectedLen { - t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen) - } - }) - } -} - -func TestFilterTestsIntegration(t *testing.T) { - tests := []CITestConf{ - {ID: "run_all_in_ocr_tests_go", TestEnvType: "docker", Workflows: []string{"Run Nightly E2E Tests"}}, - {ID: "run_all_in_ocr2_tests_go", TestEnvType: "docker", Workflows: []string{"Run PR E2E Tests"}}, - {ID: "run_all_in_ocr3_tests_go", TestEnvType: "k8s_remote_runner", Workflows: []string{"Run PR E2E Tests"}}, - } - - cases := []struct { - description string - inputNames string - inputWorkflow string - inputTestType string - inputIDs string - expectedLen int - }{ - {"Filter by test type and ID", "", "", "docker", "run_all_in_ocr2_tests_go", 1}, - {"Filter by trigger and test type", "", "Run PR E2E Tests", "docker", "*", 1}, - {"No filters applied", "", "", "", "*", 3}, - {"Filter mismatching all criteria", "", "Run Nightly E2E Tests", "", "", 1}, - } - - for _, c := range cases { - t.Run(c.description, func(t *testing.T) { - filtered := filterTests(tests, c.inputWorkflow, c.inputTestType, c.inputIDs, false) - if len(filtered) != c.expectedLen { - t.Errorf("FilterTests(%s) returned %d tests, expected %d", c.description, len(filtered), c.expectedLen) - } - }) - } -} diff --git a/integration-tests/citool/cmd/root_cmd.go b/integration-tests/citool/cmd/root_cmd.go deleted file mode 100644 index fdb4efd5724..00000000000 --- a/integration-tests/citool/cmd/root_cmd.go +++ /dev/null @@ -1,32 +0,0 @@ -package cmd - -import ( - "os" - - "github.com/spf13/cobra" -) - -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "citool", - Short: "A tool to manage E2E tests on Github CI", -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) - } -} - -func init() { - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - - rootCmd.AddCommand(checkTestsCmd) - rootCmd.AddCommand(filterCmd) - rootCmd.AddCommand(csvExportCmd) - rootCmd.AddCommand(testConfigCmd) - testConfigCmd.AddCommand(createTestConfigCmd) -} diff --git a/integration-tests/citool/cmd/test_config_cmd.go b/integration-tests/citool/cmd/test_config_cmd.go deleted file mode 100644 index 0c0e272353b..00000000000 --- a/integration-tests/citool/cmd/test_config_cmd.go +++ /dev/null @@ -1,123 +0,0 @@ -package cmd - -import ( - "strings" - - "github.com/spf13/cobra" - - "github.com/smartcontractkit/chainlink-testing-framework/utils" -) - -var testConfigCmd = &cobra.Command{ - Use: "test-config", - Short: "Manage test config", -} - -// OverrideConfig holds the configuration data for overrides -type OverrideConfig struct { - ChainlinkImage string - ChainlinkVersion string - ChainlinkUpgradeImage string - ChainlinkUpgradeVersion string - ChainlinkPostgresVersion string - SelectedNetworks []string - PyroscopeEnabled bool - PyroscopeServerURL string - PyroscopeEnvironment string - PyroscopeKey string - LoggingTestLogCollect bool - LoggingRunID string - LoggingLogTargets []string - LoggingLokiTenantID string - LoggingLokiEndpoint string - LoggingLokiBasicAuth string - LoggingGrafanaBaseURL string - LoggingGrafanaDashboardURL string - LoggingGrafanaBearerToken string - PrivateEthereumNetworkExecutionLayer string - PrivateEthereumNetworkEthereumVersion string - PrivateEthereumNetworkCustomDockerImages string -} - -const ( - ChainlinkVersionFlag = "chainlink-version" - ChainlinkUpgradeVersionFlag = "chainlink-upgrade-version" - ChainlinkPostgresVersionFlag = "chainlink-postgres-version" - SelectedNetworksFlag = "selected-networks" - FromBase64ConfigFlag = "from-base64-config" - LoggingLokiBasicAuthFlag = "logging-loki-basic-auth" - LoggingLokiEndpointFlag = "logging-loki-endpoint" - LoggingRunIDFlag = "logging-run-id" - LoggingLokiTenantIDFlag = "logging-loki-tenant-id" - LoggingGrafanaBaseURLFlag = "logging-grafana-base-url" - LoggingGrafanaDashboardURLFlag = "logging-grafana-dashboard-url" - LoggingGrafanaBearerTokenFlag = "logging-grafana-bearer-token" - LoggingLogTargetsFlag = "logging-log-targets" - LoggingTestLogCollectFlag = "logging-test-log-collect" - PyroscopeEnabledFlag = "pyroscope-enabled" - PyroscopeServerURLFlag = "pyroscope-server-url" - PyroscopeKeyFlag = "pyroscope-key" - PyroscopeEnvironmentFlag = "pyroscope-environment" - PrivateEthereumNetworkExecutionLayerFlag = "private-ethereum-network-execution-layer" - PrivateEthereumNetworkEthereumVersionFlag = "private-ethereum-network-ethereum-version" - PrivateEthereumNetworkCustomDockerImageFlag = "private-ethereum-network-custom-docker-image" -) - -var oc OverrideConfig - -func init() { - cmds := []*cobra.Command{createTestConfigCmd} - for _, c := range cmds { - c.Flags().StringArrayVar(&oc.SelectedNetworks, SelectedNetworksFlag, nil, "Selected networks") - c.Flags().StringVar(&oc.ChainlinkVersion, ChainlinkVersionFlag, "", "Chainlink version") - c.Flags().StringVar(&oc.ChainlinkUpgradeVersion, ChainlinkUpgradeVersionFlag, "", "Chainlink upgrade version") - c.Flags().StringVar(&oc.ChainlinkPostgresVersion, ChainlinkPostgresVersionFlag, "", "Chainlink Postgres version") - c.Flags().BoolVar(&oc.PyroscopeEnabled, PyroscopeEnabledFlag, false, "Pyroscope enabled") - c.Flags().StringVar(&oc.PyroscopeServerURL, PyroscopeServerURLFlag, "", "Pyroscope server URL") - c.Flags().StringVar(&oc.PyroscopeKey, PyroscopeKeyFlag, "", "Pyroscope key") - c.Flags().StringVar(&oc.PyroscopeEnvironment, PyroscopeEnvironmentFlag, "", "Pyroscope environment") - c.Flags().BoolVar(&oc.LoggingTestLogCollect, LoggingTestLogCollectFlag, false, "Test log collect") - c.Flags().StringVar(&oc.LoggingRunID, LoggingRunIDFlag, "", "Run ID") - c.Flags().StringArrayVar(&oc.LoggingLogTargets, LoggingLogTargetsFlag, nil, "Logging.LogStream.LogTargets") - c.Flags().StringVar(&oc.LoggingLokiEndpoint, LoggingLokiEndpointFlag, "", "") - c.Flags().StringVar(&oc.LoggingLokiTenantID, LoggingLokiTenantIDFlag, "", "") - c.Flags().StringVar(&oc.LoggingLokiBasicAuth, LoggingLokiBasicAuthFlag, "", "") - c.Flags().StringVar(&oc.LoggingGrafanaBaseURL, LoggingGrafanaBaseURLFlag, "", "") - c.Flags().StringVar(&oc.LoggingGrafanaDashboardURL, LoggingGrafanaDashboardURLFlag, "", "") - c.Flags().StringVar(&oc.LoggingGrafanaBearerToken, LoggingGrafanaBearerTokenFlag, "", "") - c.Flags().StringVar(&oc.PrivateEthereumNetworkExecutionLayer, PrivateEthereumNetworkExecutionLayerFlag, "", "") - c.Flags().StringVar(&oc.PrivateEthereumNetworkEthereumVersion, PrivateEthereumNetworkEthereumVersionFlag, "", "") - c.Flags().StringVar(&oc.PrivateEthereumNetworkCustomDockerImages, PrivateEthereumNetworkCustomDockerImageFlag, "", "") - - c.PreRun = func(_ *cobra.Command, _ []string) { - // Resolve selected networks environment variable if set - if len(oc.SelectedNetworks) > 0 { - _, hasEnvVar := utils.LookupEnvVarName(oc.SelectedNetworks[0]) - if hasEnvVar { - selectedNetworks := utils.MustResolveEnvPlaceholder(oc.SelectedNetworks[0]) - oc.SelectedNetworks = strings.Split(selectedNetworks, ",") - } - } - - // Resolve all other environment variables - oc.ChainlinkImage = utils.MustResolveEnvPlaceholder(oc.ChainlinkImage) - oc.ChainlinkVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkVersion) - oc.ChainlinkUpgradeImage = utils.MustResolveEnvPlaceholder(oc.ChainlinkUpgradeImage) - oc.ChainlinkUpgradeVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkUpgradeVersion) - oc.ChainlinkPostgresVersion = utils.MustResolveEnvPlaceholder(oc.ChainlinkPostgresVersion) - oc.PyroscopeServerURL = utils.MustResolveEnvPlaceholder(oc.PyroscopeServerURL) - oc.PyroscopeKey = utils.MustResolveEnvPlaceholder(oc.PyroscopeKey) - oc.PyroscopeEnvironment = utils.MustResolveEnvPlaceholder(oc.PyroscopeEnvironment) - oc.LoggingRunID = utils.MustResolveEnvPlaceholder(oc.LoggingRunID) - oc.LoggingLokiTenantID = utils.MustResolveEnvPlaceholder(oc.LoggingLokiTenantID) - oc.LoggingLokiEndpoint = utils.MustResolveEnvPlaceholder(oc.LoggingLokiEndpoint) - oc.LoggingLokiBasicAuth = utils.MustResolveEnvPlaceholder(oc.LoggingLokiBasicAuth) - oc.LoggingGrafanaBaseURL = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaBaseURL) - oc.LoggingGrafanaDashboardURL = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaDashboardURL) - oc.LoggingGrafanaBearerToken = utils.MustResolveEnvPlaceholder(oc.LoggingGrafanaBearerToken) - oc.PrivateEthereumNetworkExecutionLayer = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkExecutionLayer) - oc.PrivateEthereumNetworkEthereumVersion = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkEthereumVersion) - oc.PrivateEthereumNetworkCustomDockerImages = utils.MustResolveEnvPlaceholder(oc.PrivateEthereumNetworkCustomDockerImages) - } - } -} diff --git a/integration-tests/citool/cmd/test_config_cmd_test.go b/integration-tests/citool/cmd/test_config_cmd_test.go deleted file mode 100644 index 79185e60822..00000000000 --- a/integration-tests/citool/cmd/test_config_cmd_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package cmd - -import ( - "bytes" - "testing" - - "github.com/pelletier/go-toml/v2" - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - - ctf_config "github.com/smartcontractkit/chainlink-testing-framework/config" - ctf_config_types "github.com/smartcontractkit/chainlink-testing-framework/config/types" -) - -func TestCreateTestConfigCmd(t *testing.T) { - tests := []struct { - name string - args []string - want interface{} - check func(t *testing.T, tc *ctf_config.TestConfig) - wantErr bool - }{ - { - name: "LoggingLogTargets", - args: []string{"create", "--logging-log-targets=target1", "--logging-log-targets=target2"}, - check: func(t *testing.T, tc *ctf_config.TestConfig) { - assert.NotNil(t, tc.Logging) - assert.NotNil(t, tc.Logging.LogStream) - assert.Equal(t, []string{"target1", "target2"}, tc.Logging.LogStream.LogTargets) - }, - }, - { - name: "PrivateEthereumNetworkExecutionLayerFlag", - args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0"}, - check: func(t *testing.T, tc *ctf_config.TestConfig) { - assert.NotNil(t, tc.PrivateEthereumNetwork) - assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer) - assert.Equal(t, ctf_config_types.ExecutionLayer("geth"), *tc.PrivateEthereumNetwork.ExecutionLayer) - assert.Equal(t, ctf_config_types.EthereumVersion("1.10.0"), *tc.PrivateEthereumNetwork.EthereumVersion) - }, - }, - { - name: "PrivateEthereumNetworkCustomDockerImageFlag", - args: []string{"create", "--private-ethereum-network-execution-layer=geth", "--private-ethereum-network-ethereum-version=1.10.0", "--private-ethereum-network-custom-docker-image=custom-image:v1.0"}, - check: func(t *testing.T, tc *ctf_config.TestConfig) { - assert.NotNil(t, tc.PrivateEthereumNetwork) - assert.NotNil(t, tc.PrivateEthereumNetwork.ExecutionLayer) - assert.Equal(t, map[ctf_config.ContainerType]string{"execution_layer": "custom-image:v1.0"}, tc.PrivateEthereumNetwork.CustomDockerImages) - }, - }, - } - - rootCmd := &cobra.Command{} - rootCmd.AddCommand(createTestConfigCmd) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rootCmd.SetArgs(tt.args) - var out bytes.Buffer - rootCmd.SetOutput(&out) - err := rootCmd.Execute() - if (err != nil) != tt.wantErr { - t.Fatalf("Execute() error = %v, wantErr %v", err, tt.wantErr) - } - var tc ctf_config.TestConfig - err = toml.Unmarshal(out.Bytes(), &tc) - if err != nil { - t.Fatalf("Failed to unmarshal output: %v", err) - } - if tt.check != nil { - tt.check(t, &tc) - } - }) - } -} diff --git a/integration-tests/citool/cmd/types.go b/integration-tests/citool/cmd/types.go deleted file mode 100644 index 3c347e9406b..00000000000 --- a/integration-tests/citool/cmd/types.go +++ /dev/null @@ -1,26 +0,0 @@ -package cmd - -type Test struct { - Name string - Path string -} - -// CITestConf defines the configuration for running a test in a CI environment, specifying details like test ID, path, type, runner settings, command, and associated workflows. -type CITestConf struct { - ID string `yaml:"id" json:"id"` - IDSanitized string `json:"id_sanitized"` - Path string `yaml:"path" json:"path"` - TestEnvType string `yaml:"test_env_type" json:"test_env_type"` - RunsOn string `yaml:"runs_on" json:"runs_on"` - TestCmd string `yaml:"test_cmd" json:"test_cmd"` - TestConfigOverrideRequired bool `yaml:"test_config_override_required" json:"testConfigOverrideRequired"` - TestSecretsRequired bool `yaml:"test_secrets_required" json:"testSecretsRequired"` - TestInputs map[string]string `yaml:"test_inputs" json:"test_inputs"` - RemoteRunnerMemory string `yaml:"remote_runner_memory" json:"remoteRunnerMemory"` - PyroscopeEnv string `yaml:"pyroscope_env" json:"pyroscopeEnv"` - Workflows []string `yaml:"workflows" json:"workflows"` -} - -type Config struct { - Tests []CITestConf `yaml:"runner-test-matrix"` -} diff --git a/integration-tests/citool/main.go b/integration-tests/citool/main.go deleted file mode 100644 index 4fa6cac56e5..00000000000 --- a/integration-tests/citool/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/smartcontractkit/chainlink/integration-tests/citool/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 8589f12840f..b4c1839c74a 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -54,8 +54,6 @@ require ( golang.org/x/sync v0.7.0 golang.org/x/text v0.16.0 gopkg.in/guregu/null.v4 v4.0.0 - gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.28.2 ) @@ -469,6 +467,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.28.2 // indirect k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/cli-runtime v0.28.2 // indirect