Skip to content

Commit

Permalink
remove retesteth driver, simplify the coverage script
Browse files Browse the repository at this point in the history
  • Loading branch information
winsvega committed Sep 30, 2024
1 parent bb54c96 commit a00c122
Showing 1 changed file with 22 additions and 34 deletions.
56 changes: 22 additions & 34 deletions .github/workflows/coverage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@ on:

jobs:
evmone-coverage-diff:
runs-on: ubuntu-24.04
strategy:
matrix:
driver: [retesteth, native]
runs-on: ubuntu-latest

steps:
- name: Checkout code
Expand Down Expand Up @@ -159,7 +156,7 @@ jobs:
# This command diffs the .py scripts introduced by a PR
- name: Parse and fill introduced test sources
if: steps.changed-tests.outputs.tests_any_changed == 'true'
if: steps.changed-tests.outputs.tests_any_changed == 'true'
env:
CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }}
run: |
Expand All @@ -171,24 +168,13 @@ jobs:
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
# Use a while loop with a here-string to avoid subshell issues
while IFS= read -r file; do
echo "Fill: $file"
uv run fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
done <<< "$files"
uv run fill $files --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill $files --fork=PragueEIP7692 --evm-bin evmone-t8n || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests."
exit 1
fi
if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
echo "Disabling retesteth coverage check as EOF tests detected!"
echo "retesteth_skip=true" >> $GITHUB_ENV
exit 0
else
echo "retesteth_skip=false" >> $GITHUB_ENV
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
Expand All @@ -207,14 +193,15 @@ jobs:
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
- name: Parse and fill introduced test sources from before the PR
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.converted_skip == 'true' }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && env.converted_skip == 'true' }}
env:
CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }}
run: |
echo "--------------------"
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
# load introduces .py files
source $GITHUB_ENV
files=$(echo "$NEW_TESTS" | tr ',' '\n')
files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')
git checkout main
PREV_COMMIT=$(git rev-parse HEAD)
Expand All @@ -226,18 +213,19 @@ jobs:
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
# Use a while loop with a here-string to avoid subshell issues
while IFS= read -r file; do
echo "Fill: $files"
uv run fill "$files" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill "$files" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) >> (tee -a filloutput.log filloutputEOF.log) 2>&1
done <<< "$files"
uv run fill $files --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(uv run fill $files --fork=PragueEIP7692 --evm-bin evmone-t8n || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests from before the PR."
exit 1
fi
if grep -q "ERROR collecting test session" filloutput.log; then
echo "Error: failed to generate .py tests from before the PR."
exit 1
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
Expand All @@ -253,7 +241,7 @@ jobs:
- name: Print tests that will be covered
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
run: |
echo "Original BASE tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
Expand All @@ -263,44 +251,44 @@ jobs:
- name: Run coverage of the BASE tests
uses: addnab/docker-run-action@v3
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE

- name: Run coverage of the PATCH tests
uses: addnab/docker-run-action@v3
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH

- name: Run coverage DIFF of the PATCH tests compared to BASE tests
uses: addnab/docker-run-action@v3
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov

- name: Chmod coverage results
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
run: |
user=$(whoami)
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
- name: Upload coverage results
uses: actions/upload-artifact@v3
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
with:
name: coverage-diff-${{ matrix.driver }}
path: ${{ github.workspace }}/evmtest_coverage/coverage

- name: Verify coverage results
uses: addnab/docker-run-action@v3
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && (env.retesteth_skip == 'false' || matrix.driver == 'native') }}
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
Expand Down

0 comments on commit a00c122

Please sign in to comment.