From 5a602f6f1930136271a9ccdad1bb04bf5dc44c63 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 14 Nov 2024 11:02:34 -0800 Subject: [PATCH 1/2] test: build test binaries --- .github/workflows/cache-test-binaries.yml | 82 +++++++++++++++++++ tools/bin/build-test-binaries | 24 ++++++ .../bin/build-test-binaries-with-concurrency | 55 +++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 .github/workflows/cache-test-binaries.yml create mode 100755 tools/bin/build-test-binaries create mode 100755 tools/bin/build-test-binaries-with-concurrency diff --git a/.github/workflows/cache-test-binaries.yml b/.github/workflows/cache-test-binaries.yml new file mode 100644 index 00000000000..300bd61704b --- /dev/null +++ b/.github/workflows/cache-test-binaries.yml @@ -0,0 +1,82 @@ +name: Test Binary Caching + +# Run on key branches to make sure integration is good, otherwise run on all PR's +on: + pull_request: + + +jobs: + + build-test-binaries: + runs-on: ubuntu22.04-32cores-128GB + env: + # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests + # when they should not be using it, while still allowing us to DRY up the setup + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + restore-build-cache-only: true + # piggy back on go_core_tests for caching + build-cache-version: 'go_core_tests' + + - name: Build Test Binaries + shell: bash + env: + OUTPUT_FILE: ./output.txt + USE_TEE: false + CL_DATABASE_URL: ${{ env.DB_URL }} + run: | + ./tools/bin/build-test-binaries + + - name: Debug testsout + shell: bash + run: | + ls -lah ./testsout + du -ch ./testsout + + build-test-binaries-concurrent: + runs-on: ubuntu22.04-32cores-128GB + env: + # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests + # when they should not be using it, while still allowing us to DRY up the setup + DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable + steps: + - name: Checkout the repo + uses: actions/checkout@v4.2.1 + + - name: Change Modtime of Files (cache optimization) + shell: bash + run: | + find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + restore-build-cache-only: true + # piggy back on go_core_tests for caching + build-cache-version: 'go_core_tests' + + - name: Build Test Binaries + shell: bash + env: + OUTPUT_FILE: ./output.txt + USE_TEE: false + CL_DATABASE_URL: ${{ env.DB_URL }} + run: | + ./tools/bin/build-test-binaries-with-concurrency + + - name: Debug testsout + shell: bash + run: | + ls -lah ./testsout + du -ch ./testsout diff --git a/tools/bin/build-test-binaries b/tools/bin/build-test-binaries new file mode 100755 index 00000000000..4821f5d9846 --- /dev/null +++ b/tools/bin/build-test-binaries @@ -0,0 +1,24 @@ +#!/bin/bash + +# Directory to store compiled test binaries +mkdir -p testsout + +TOOLS_PATH=${TOOLS_PATH:-"./tools"} +GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags) + +# Find all packages with tests recursively +testable_packages=$(go list ./... | grep -vE '/vendor|/testsout') + +# Compile test binaries for each package and handle naming conflicts +for pkg in $testable_packages; do + output_file="testsout/$(echo $pkg | tr '/' '-')-test" # Transform pkg path to a unique filename + + # Compile test binary + echo "Compiling test for package $pkg" + go test -c -o "$output_file" -ldflags "$GO_LDFLAGS" -tags integration -vet=off "$pkg" +done + +# for binary in testsout/*-test; do + # echo "Running $binary" + # $binary -test.run . || exit 1 # Exit on failure +# done \ No newline at end of file diff --git a/tools/bin/build-test-binaries-with-concurrency b/tools/bin/build-test-binaries-with-concurrency new file mode 100755 index 00000000000..b7563de093b --- /dev/null +++ b/tools/bin/build-test-binaries-with-concurrency @@ -0,0 +1,55 @@ +#!/bin/bash + +# Directory to store compiled test binaries +mkdir -p testsout + +TOOLS_PATH=${TOOLS_PATH:-"./tools"} +GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags) + +# Find all packages with tests recursively +testable_packages=$(go list ./... | grep -vE '/vendor|/testsout') + +# Set the concurrency limits +build_concurrency=4 +# run_concurrency=4 +build_jobs=0 +# run_jobs=0 + +# Compile test binaries for each package with controlled concurrency +for pkg in $testable_packages; do + output_file="testsout/$(echo $pkg | tr '/' '-')-test" # Transform pkg path to a unique filename + + # Compile test binary in the background + echo "Compiling test for package $pkg" + go test -c -o "$output_file" -ldflags "$GO_LDFLAGS" -tags integration -vet=off "$pkg" & + + # Increment the build job count + build_jobs=$((build_jobs + 1)) + + # If we've reached the build concurrency limit, wait for any job to finish + if [[ $build_jobs -ge $build_concurrency ]]; then + wait -n + build_jobs=$((build_jobs - 1)) + fi +done + +# Wait for all remaining build jobs to complete +wait + +# Run all compiled test binaries with controlled concurrency +# for binary in testsout/*-test; do + # echo "Running $binary" + # $binary -test.run . || exit 1 & + + # Increment the run job count + # run_jobs=$((run_jobs + 1)) + + # If we've reached the run concurrency limit, wait for any job to finish + # if [[ $run_jobs -ge $run_concurrency ]]; then + # wait -n + # run_jobs=$((run_jobs - 1)) + # fi +# done + +# Wait for all remaining run jobs to complete +wait From 54aaab946e2d8e2c7cef820e5ed529f0631cda91 Mon Sep 17 00:00:00 2001 From: Erik Burton Date: Thu, 14 Nov 2024 11:46:36 -0800 Subject: [PATCH 2/2] test: build concurrencies, hashes --- .github/workflows/cache-test-binaries.yml | 61 ++++++------------- .../bin/build-test-binaries-with-concurrency | 9 +-- 2 files changed, 23 insertions(+), 47 deletions(-) diff --git a/.github/workflows/cache-test-binaries.yml b/.github/workflows/cache-test-binaries.yml index 300bd61704b..cd0beb6ffcc 100644 --- a/.github/workflows/cache-test-binaries.yml +++ b/.github/workflows/cache-test-binaries.yml @@ -7,45 +7,17 @@ on: jobs: - build-test-binaries: - runs-on: ubuntu22.04-32cores-128GB - env: - # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests - # when they should not be using it, while still allowing us to DRY up the setup - DB_URL: postgresql://postgres:postgres@localhost:5432/chainlink_test?sslmode=disable - steps: - - name: Checkout the repo - uses: actions/checkout@v4.2.1 - - - name: Change Modtime of Files (cache optimization) - shell: bash - run: | - find . -type f,d -exec touch -r {} -d '1970-01-01T00:00:01' {} \; || true - - - name: Setup Go - uses: ./.github/actions/setup-go - with: - restore-build-cache-only: true - # piggy back on go_core_tests for caching - build-cache-version: 'go_core_tests' - - - name: Build Test Binaries - shell: bash - env: - OUTPUT_FILE: ./output.txt - USE_TEE: false - CL_DATABASE_URL: ${{ env.DB_URL }} - run: | - ./tools/bin/build-test-binaries - - - name: Debug testsout - shell: bash - run: | - ls -lah ./testsout - du -ch ./testsout - build-test-binaries-concurrent: runs-on: ubuntu22.04-32cores-128GB + strategy: + fail-fast: false + matrix: + type: + - concurrency: 4 + - concurrency: 8 + - concurrency: 16 + - concurrency: 32 + name: Build Test Binaries (${{ matrix.type.concurrency }}) env: # We explicitly have this env var not be "CL_DATABASE_URL" to avoid having it be used by core related tests # when they should not be using it, while still allowing us to DRY up the setup @@ -69,14 +41,21 @@ jobs: - name: Build Test Binaries shell: bash env: - OUTPUT_FILE: ./output.txt - USE_TEE: false CL_DATABASE_URL: ${{ env.DB_URL }} + BUILD_CONCURRENCY: ${{ matrix.type.concurrency }} run: | ./tools/bin/build-test-binaries-with-concurrency - name: Debug testsout shell: bash run: | - ls -lah ./testsout - du -ch ./testsout + # ls -lah ./testsout + # du -ch ./testsout + # Hash each file in the output directory and print the filename and hash + echo "Hashing compiled test binaries:" + for file in testsout/*; do + if [[ -f "$file" ]]; then + hash=$(sha256sum "$file" | awk '{ print $1 }') + echo "File: $file, SHA256 Hash: $hash" + fi + done diff --git a/tools/bin/build-test-binaries-with-concurrency b/tools/bin/build-test-binaries-with-concurrency index b7563de093b..c2641b49d1f 100755 --- a/tools/bin/build-test-binaries-with-concurrency +++ b/tools/bin/build-test-binaries-with-concurrency @@ -3,14 +3,11 @@ # Directory to store compiled test binaries mkdir -p testsout -TOOLS_PATH=${TOOLS_PATH:-"./tools"} -GO_LDFLAGS=$(bash ${TOOLS_PATH}/bin/ldflags) - # Find all packages with tests recursively testable_packages=$(go list ./... | grep -vE '/vendor|/testsout') # Set the concurrency limits -build_concurrency=4 +BUILD_CONCURRENCY=${BUILD_CONCURRENCY:-"4"} # run_concurrency=4 build_jobs=0 # run_jobs=0 @@ -21,13 +18,13 @@ for pkg in $testable_packages; do # Compile test binary in the background echo "Compiling test for package $pkg" - go test -c -o "$output_file" -ldflags "$GO_LDFLAGS" -tags integration -vet=off "$pkg" & + go test -c -o "$output_file" -vet=off "$pkg" & # Increment the build job count build_jobs=$((build_jobs + 1)) # If we've reached the build concurrency limit, wait for any job to finish - if [[ $build_jobs -ge $build_concurrency ]]; then + if [[ $build_jobs -ge $BUILD_CONCURRENCY ]]; then wait -n build_jobs=$((build_jobs - 1)) fi