feat(coverage): check patched tests coverage [TEST] #168
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Evmone Coverage Report | |
| on: | |
| pull_request: | |
| paths: | |
| - 'tests/**' # This triggers the workflow for any changes in the tests folder | |
| jobs: | |
| evmone-coverage-diff: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| driver: [retesteth, native] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Fetch target branch | |
| run: git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v3 | |
| if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} | |
| with: | |
| username: winsvega | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Install deps | |
| run: | | |
| echo $(pwd) | |
| echo ${{ github.workspace }} | |
| #install pyspec deps from root repo | |
| python3 --version | |
| pip install --upgrade pip | |
| python3 -m venv ./venv/ | |
| source ./venv/bin/activate | |
| pip install -e . | |
| solc-select use 0.8.25 --always-install | |
| # Required to fill .py tests | |
| - name: Build GO EVM | |
| uses: ./.github/actions/build-evm | |
| id: evm-builder | |
| with: | |
| type: 'main' | |
| - name: Build EVMONE EVM | |
| uses: ./.github/actions/build-evmone-evm | |
| id: evm-builder2 | |
| with: | |
| type: 'main' | |
| - name: Checkout ethereum/tests | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ethereum/tests | |
| path: testpath | |
| sparse-checkout: | | |
| GeneralStateTests | |
| EOFTests | |
| - name: Checkout ethereum/legacytests | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ethereum/legacytests | |
| path: legacytestpath | |
| sparse-checkout: | | |
| Cancun/GeneralStateTests | |
| # This command diffs the file and filters in new lines | |
| - name: Parse converted tests from converted-ethereum-tests.txt | |
| run: | | |
| echo "New lines introduced in converted-ethereum-tests.txt:" | |
| lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true) | |
| if [ -z "$lines" ]; then | |
| echo "No new lines in converted-ethereum-tests.txt, check updates instead:" | |
| echo "converted_skip=true" >> $GITHUB_ENV | |
| exit 0 | |
| else | |
| echo "converted_skip=false" >> $GITHUB_ENV | |
| fi | |
| files=$(echo "$lines" | grep -oP '(?<=\+).+\.json') | |
| for file in $files; do | |
| echo $file | |
| done | |
| echo "----------------" | |
| echo "Discovered existing json tests that will be BASE files:" | |
| BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| mkdir -p $BASE_TESTS_PATH | |
| for file in $files; do | |
| # Make sure each file exist at least in develop or legacy tests | |
| file_found=0 | |
| file_path=${{ github.workspace }}/testpath/$file | |
| if [ -e "$file_path" ]; then | |
| file_found=1 | |
| cp $file_path $BASE_TESTS_PATH | |
| echo $file_path | |
| fi | |
| # Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet) | |
| if [[ "$file" == *"GeneralStateTests"* ]]; then | |
| file_path=${{ github.workspace }}/legacytestpath/Cancun/$file | |
| base_name=$(basename "$file") | |
| legacy_file_name="legacy_$base_name" | |
| if [ -e "$file_path" ]; then | |
| file_found=1 | |
| cp $file_path $BASE_TESTS_PATH/$legacy_file_name | |
| echo $file_path | |
| fi | |
| fi | |
| if [ $file_found -eq 0 ]; then | |
| echo "Error: Failed to find the test file $file in test repo" | |
| exit 1 | |
| fi | |
| done | |
| # This command diffs the .py scripts introduced by a PR | |
| - name: Parse and fill introduced test sources | |
| run: | | |
| python3 -m venv ./venv/ | |
| source ./venv/bin/activate | |
| if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then | |
| # Fetch changes when PR comes from remote repo | |
| git fetch origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} | |
| git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/origin/PR-${{ github.event.pull_request.number }} | |
| files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$') | |
| else | |
| # Fetch the base branch and the head branch | |
| git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} | |
| git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }} | |
| # Perform the diff | |
| files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$') | |
| fi | |
| echo "Modified or new .py files in tests folder:" | |
| echo "$files" | while read line; do | |
| file=$(echo "$line" | cut -c 3-) | |
| echo $file | |
| done | |
| # fill new tests | |
| # using `|| true` here because if no tests found, pyspec fill returns error code | |
| FOUND_TEST=false | |
| mkdir -p fixtures/state_tests | |
| mkdir -p fixtures/eof_tests | |
| # Use a while loop with a here-string to avoid subshell issues | |
| while IFS= read -r line; do | |
| file=$(echo "$line" | cut -c 3-) | |
| if grep -q "def test_" "$file"; then | |
| FOUND_TEST=true | |
| fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1 | |
| (fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1 | |
| fi | |
| done <<< "$files" | |
| echo "Tests found: $FOUND_TEST" | |
| if [[ "$FOUND_TEST" == "false" ]]; then | |
| if [ "${{ env.converted_skip }}" == 'false' ]; then | |
| echo "Error converted-ethereum-tests.txt has new lines, but no test in .py files were found" | |
| exit 1 | |
| fi | |
| echo "No test detected in changed .py files, exiting the script" | |
| echo "coverage_skip=true" >> $GITHUB_ENV | |
| exit 0 | |
| else | |
| echo "coverage_skip=false" >> $GITHUB_ENV | |
| fi | |
| if grep -q "FAILURES" filloutput.log; then | |
| echo "Error: failed to generate .py tests." | |
| exit 1 | |
| fi | |
| if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then | |
| echo "Disabling retesteth coverage check as EOF tests detected!" | |
| echo "retesteth_skip=true" >> $GITHUB_ENV | |
| exit 0 | |
| else | |
| echo "retesteth_skip=false" >> $GITHUB_ENV | |
| fi | |
| filesState=$(find fixtures/state_tests -type f -name "*.json") | |
| filesEOF=$(find fixtures/eof_tests -type f -name "*.json") | |
| if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then | |
| echo "Error: No filled JSON fixtures found in fixtures." | |
| exit 1 | |
| fi | |
| PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS | |
| mkdir -p $PATCH_TEST_PATH | |
| find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \; | |
| find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \; | |
| - name: Parse and fill introduced test sources from before the PR | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.converted_skip == 'true' && env.coverage_skip == 'false'}} | |
| run: | | |
| echo "--------------------" | |
| echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:" | |
| if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then | |
| files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$') | |
| else | |
| files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$') | |
| fi | |
| echo "Modified or new .py files in tests folder:" | |
| echo "$files" | while read line; do | |
| file=$(echo "$line" | cut -c 3-) | |
| echo $file | |
| done | |
| git checkout main | |
| PREV_COMMIT=$(git rev-parse HEAD) | |
| echo "Checkout head $PREV_COMMIT" | |
| python3 -m venv ./venv/ | |
| source ./venv/bin/activate | |
| rm -r fixtures | |
| rm filloutput.log | |
| rm filloutputEOF.log | |
| mkdir -p fixtures/state_tests | |
| mkdir -p fixtures/eof_tests | |
| FOUND_TEST=false | |
| while IFS= read -r line; do | |
| file=$(echo "$line" | cut -c 3-) | |
| if grep -q "def test_" "$file"; then | |
| FOUND_TEST=true | |
| fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1 | |
| (fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1 | |
| fi | |
| done <<< "$files" | |
| echo "Tests found: $FOUND_TEST" | |
| if [[ "$FOUND_TEST" == "false" ]]; then | |
| echo "No test detected in .py files from before the PR, exiting the script" | |
| echo "coverage_skip=true" >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| if grep -q "FAILURES" filloutput.log; then | |
| echo "Error: failed to generate .py tests from before the PR." | |
| exit 1 | |
| fi | |
| filesState=$(find fixtures/state_tests -type f -name "*.json") | |
| filesEOF=$(find fixtures/eof_tests -type f -name "*.json") | |
| if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then | |
| echo "Error: No filled JSON fixtures found in fixtures from before the PR." | |
| exit 1 | |
| fi | |
| BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| mkdir -p $BASE_TEST_PATH | |
| find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \; | |
| find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \; | |
| for file in $BASE_TEST_PATH/*.json; do | |
| mv "$file" "${file%.json}_$PREV_COMMIT.json" | |
| done | |
| - name: Print tests that will be covered | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| run: | | |
| echo "Original BASE tests:" | |
| ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS | |
| echo "--------------------" | |
| echo "Ported PATCH tests:" | |
| ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS | |
| - name: Run coverage of the BASE tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE | |
| - name: Run coverage of the PATCH tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH | |
| - name: Run coverage DIFF of the PATCH tests compared to BASE tests | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov | |
| - name: Chmod coverage results | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| run: | | |
| user=$(whoami) | |
| sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage | |
| - name: Upload coverage results | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| with: | |
| name: coverage-diff | |
| path: ${{ github.workspace }}/evmtest_coverage/coverage | |
| - name: Verify coverage results | |
| uses: addnab/docker-run-action@v3 | |
| if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }} | |
| with: | |
| image: winsvega/evmone-coverage-script:latest | |
| options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests | |
| run: /check.sh |