Skip to content

feat(coverage): Enable coverage on any test change (rebased with uv) #273

feat(coverage): Enable coverage on any test change (rebased with uv)

feat(coverage): Enable coverage on any test change (rebased with uv) #273

Workflow file for this run

name: Evmone Coverage Report
on:
pull_request:
paths:
- 'tests/**' # This triggers the workflow for any changes in the tests folder
jobs:
evmone-coverage-diff:
runs-on: ubuntu-latest
strategy:
matrix:
driver: [retesteth, native]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Fetch github branches and detect introduces .py files
run: |
py_files=()
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
# Fetch changes when PR comes from remote repo
git fetch origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/origin/PR-${{ github.event.pull_request.number }}
gitdiff=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/)
else
# Fetch the base branch and the head branch
git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }}
gitdiff=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/)
fi
echo "git diff:"
echo "$gitdiff"
paths=$(echo "$gitdiff" | grep -oE '/[^[:space:]]+')
while IFS= read -r line; do
py_files+=("tests$line")
done <<< "$paths"
echo "Extracted file paths:"
for path in "${py_files[@]}"; do
echo "$path"
done
echo "Prepare the NEW_TESTS variable"
py_files_str=$(IFS=,; echo "${py_files[*]}")
echo "NEW_TESTS=$py_files_str" >> $GITHUB_ENV
echo "Detected new/changed .py files:"
source $GITHUB_ENV
files2=$(echo "$NEW_TESTS" | tr ',' '\n')
while IFS= read -r file; do
echo $file
done <<< "$files2"
- name: Log in to Docker Hub
uses: docker/login-action@v3
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
with:
username: winsvega
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Install deps
run: |
echo $(pwd)
echo ${{ github.workspace }}
- name: Set up uv
uses: ./.github/actions/setup-uv
- name: Set up Python
run: uv python install 3.10
- name: Install EEST
run: |
uv sync --no-progress
uv run python --version
# Required to fill .py tests
- name: Build EVMONE EVM
uses: ./.github/actions/build-evm-client/evmone
id: evm-builder2
with:
type: "main"
- name: Checkout ethereum/tests
uses: actions/checkout@v4
with:
repository: ethereum/tests
path: testpath
sparse-checkout: |
GeneralStateTests
EOFTests
- name: Checkout ethereum/legacytests
uses: actions/checkout@v4
with:
repository: ethereum/legacytests
path: legacytestpath
sparse-checkout: |
Cancun/GeneralStateTests
# This command diffs the file and filters in new lines
- name: Parse converted tests from converted-ethereum-tests.txt
run: |
echo "New lines introduced in converted-ethereum-tests.txt:"
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true)
if [ -z "$lines" ]; then
echo "No new lines in converted-ethereum-tests.txt, check updates instead:"
echo "converted_skip=true" >> $GITHUB_ENV
exit 0
else
echo "converted_skip=false" >> $GITHUB_ENV
fi
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
for file in $files; do
echo $file
done
echo "----------------"
echo "Discovered existing json tests that will be BASE files:"
BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
mkdir -p $BASE_TESTS_PATH
for file in $files; do
# Make sure each file exist at least in develop or legacy tests
file_found=0
file_path=${{ github.workspace }}/testpath/$file
if [ -e "$file_path" ]; then
file_found=1
cp $file_path $BASE_TESTS_PATH
echo $file_path
fi
# Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
if [[ "$file" == *"GeneralStateTests"* ]]; then
file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
base_name=$(basename "$file")
legacy_file_name="legacy_$base_name"
if [ -e "$file_path" ]; then
file_found=1
cp $file_path $BASE_TESTS_PATH/$legacy_file_name
echo $file_path
fi
fi
if [ $file_found -eq 0 ]; then
echo "Error: Failed to find the test file $file in test repo"
exit 1
fi
done
# This command diffs the .py scripts introduced by a PR
- name: Parse and fill introduced test sources
run: |
source $GITHUB_ENV
files=$(echo "$NEW_TESTS" | tr ',' '\n')
python3 -m venv ./venv/
source ./venv/bin/activate
# fill new tests
# using `|| true` here because if no tests found, pyspec fill returns error code
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
# Use a while loop with a here-string to avoid subshell issues
while IFS= read -r file; do
echo "Fill: $file"
uv run fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
done <<< "$files"
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests."
exit 1
fi
if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
echo "Disabling retesteth coverage check as EOF tests detected!"
echo "retesteth_skip=true" >> $GITHUB_ENV
exit 0
else
echo "retesteth_skip=false" >> $GITHUB_ENV
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
echo "Error: No filled JSON fixtures found in fixtures."
exit 1
fi
# Include basic evm operations into coverage by default
# As when we translate from yul/solidity some dup/push opcodes could become untouched
uv run fill tests/homestead/coverage/test_coverage.py --until=Cancun --evm-bin evmone-t8n
PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
mkdir -p $PATCH_TEST_PATH
find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
- name: Parse and fill introduced test sources from before the PR
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.converted_skip == 'true' }}
run: |
echo "--------------------"
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
# load introduces .py files
source $GITHUB_ENV
files=$(echo "$NEW_TESTS" | tr ',' '\n')
git checkout main
PREV_COMMIT=$(git rev-parse HEAD)
echo "Checkout head $PREV_COMMIT"
python3 -m venv ./venv/
source ./venv/bin/activate
rm -r fixtures
rm filloutput.log
rm filloutputEOF.log
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
# Use a while loop with a here-string to avoid subshell issues
while IFS= read -r file; do
echo "Fill: $file"
uv run fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
done <<< "$files"
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests from before the PR."
exit 1
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
mkdir -p $BASE_TEST_PATH
find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
for file in $BASE_TEST_PATH/*.json; do
if [ -e "$file" ]; then
mv "$file" "${file%.json}_$PREV_COMMIT.json"
fi
done
- name: Print tests that will be covered
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
run: |
echo "Original BASE tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
echo "--------------------"
echo "Ported PATCH tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
- name: Run coverage of the BASE tests
uses: addnab/docker-run-action@v3
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE
- name: Run coverage of the PATCH tests
uses: addnab/docker-run-action@v3
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH
- name: Run coverage DIFF of the PATCH tests compared to BASE tests
uses: addnab/docker-run-action@v3
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov
- name: Chmod coverage results
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
run: |
user=$(whoami)
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
- name: Upload coverage results
uses: actions/upload-artifact@v3
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
with:
name: coverage-diff-${{ matrix.driver }}
path: ${{ github.workspace }}/evmtest_coverage/coverage
- name: Verify coverage results
uses: addnab/docker-run-action@v3
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /check.sh