From 0a50bfd5e0011bb9cff3390d5f18d13942cd900e Mon Sep 17 00:00:00 2001 From: Dimitry Kh Date: Thu, 4 Jul 2024 13:38:56 +0200 Subject: [PATCH] coverage script disable retesteth on eof test exit code on fill errors, coverage test fix --- .github/workflows/coverage.yaml | 32 ++++++++++++++++++++-- converted-ethereum-tests.txt | 1 + tests/homestead/coverage/test_coverage.py | 33 ++++++++++++++++------- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index 7e6a3ae78c..bfe1ffe762 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -19,6 +19,13 @@ jobs: - name: Fetch target branch run: git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} + - name: Log in to Docker Hub + uses: docker/login-action@v3 + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + with: + username: winsvega + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Install deps run: | echo $(pwd) @@ -141,14 +148,28 @@ jobs: done # fill new tests + # using `|| true` here because if no tests found, pyspec fill returns error code mkdir -p fixtures/state_tests mkdir -p fixtures/eof_tests echo "$files" | while read line; do file=$(echo "$line" | cut -c 3-) - fill $file --until=Cancun --evm-bin evmone-t8n || true - fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true + fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1 + (fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1 done + if grep -q "FAILURES" filloutput.log; then + echo "Error: failed to generate .py tests." + exit 1 + fi + if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then + echo "Disabling retesteth coverage check as EOF tests detected!" + echo "retesteth_skip=true" >> $GITHUB_ENV + exit 0 + else + echo "retesteth_skip=false" >> $GITHUB_ENV + fi + + filesState=$(find fixtures/state_tests -type f -name "*.json") filesEOF=$(find fixtures/eof_tests -type f -name "*.json") if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then @@ -162,6 +183,7 @@ jobs: find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \; - name: Print tests that will be covered + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} run: | echo "Original BASE tests:" ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS @@ -171,6 +193,7 @@ jobs: - name: Run coverage of the BASE tests uses: addnab/docker-run-action@v3 + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} with: image: winsvega/evmone-coverage-script:latest options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests @@ -178,6 +201,7 @@ jobs: - name: Run coverage of the PATCH tests uses: addnab/docker-run-action@v3 + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} with: image: winsvega/evmone-coverage-script:latest options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests @@ -185,24 +209,28 @@ jobs: - name: Run coverage DIFF of the PATCH tests compared to BASE tests uses: addnab/docker-run-action@v3 + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} with: image: winsvega/evmone-coverage-script:latest options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov - name: Chmod coverage results + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} run: | user=$(whoami) sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage - name: Upload coverage results uses: actions/upload-artifact@v3 + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} with: name: coverage-diff path: ${{ github.workspace }}/evmtest_coverage/coverage - name: Verify coverage results uses: addnab/docker-run-action@v3 + if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }} with: image: winsvega/evmone-coverage-script:latest options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests diff --git a/converted-ethereum-tests.txt b/converted-ethereum-tests.txt index 75b1c0c362..f951cd45d0 100644 --- a/converted-ethereum-tests.txt +++ b/converted-ethereum-tests.txt @@ -1,3 +1,4 @@ +([#647](https://github.com/ethereum/execution-spec-tests/pull/647)) EOFTests/efValidation/EOF1_returncontract_invalid_.json EOFTests/efValidation/EOF1_returncontract_valid_.json diff --git a/tests/homestead/coverage/test_coverage.py b/tests/homestead/coverage/test_coverage.py index f0e73d5757..59dc0353cf 100644 --- a/tests/homestead/coverage/test_coverage.py +++ b/tests/homestead/coverage/test_coverage.py @@ -4,6 +4,7 @@ import pytest +from ethereum_test_forks import Cancun, Fork from ethereum_test_tools import Alloc, Environment, StateTestFiller, Transaction from ethereum_test_tools.vm.opcode import Opcodes as Op @@ -12,9 +13,10 @@ @pytest.mark.valid_from("Homestead") -def test_yul_coverage( +def test_coverage( state_test: StateTestFiller, pre: Alloc, + fork: Fork, ): """ This test covers gaps that result from transforming Yul code into @@ -49,13 +51,26 @@ def test_yul_coverage( code=Op.MSTORE(0, Op.CALL(Op.GAS, missed_coverage, 0, 0, 0, 0, 0)) + Op.RETURN(0, 32), ) - tx = Transaction( - sender=pre.fund_eoa(7_000_000_000_000_000_000), - gas_limit=100000, - to=address_to, - data=b"", - value=0, - protected=False, - ) + if fork >= Cancun: + tx = Transaction( + sender=pre.fund_eoa(7_000_000_000_000_000_000), + gas_limit=100000, + to=address_to, + data=b"", + value=0, + protected=False, + access_list=[], + max_fee_per_gas=10, + max_priority_fee_per_gas=5, + ) + else: + tx = Transaction( + sender=pre.fund_eoa(7_000_000_000_000_000_000), + gas_limit=100000, + to=address_to, + data=b"", + value=0, + protected=False, + ) state_test(env=Environment(), pre=pre, post={}, tx=tx)