coverage.yaml

name: Evmone Coverage Report

on:
  pull_request:
    paths:
      - 'tests/**' # This triggers the workflow for any changes in the tests folder
      - '!tests/prague/**'  # exclude changes in 'tests/prague'
      - '!tests/osaka/**'   # exclude changes in 'tests/osaka'

jobs:
  evmone-coverage-diff:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Debug GitHub context
        run: |
          echo "Git reference: ${{ github.ref }}"
          echo "Git head ref: ${{ github.head_ref }}"
          echo "Git base ref: ${{ github.base_ref }}"

      - name: Get all changed python files in tests/ and changes to coverted-ethereum-tests.txt
        id: changed-tests
        uses: tj-actions/changed-files@v45
        with:
          # TODO: non-test modules such as __init__.py or spec.py could effect coverage - in this case we should
          # fill all applicable tests (i.e., all the test_*.py files in or under the changed module's directory)
          include_all_old_new_renamed_files: true
          output_renamed_files_as_deleted_and_added: true
          files_yaml: |
            tests:
              - tests/**/test_*.py
              - '!tests/prague/**'
              - '!tests/osaka/**'
            converted_tests:
              - converted-ethereum-tests.txt

      - name: Exit workflow if there are no changed python files
        if: steps.changed-tests.outputs.tests_any_changed != 'true'
        run: |
          echo "No python files were changed in ./tests/ - no action necessary"
          exit 0

      - name: Report changed python test moudules
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        run: |
          echo "${{ toJson(steps.changed-tests.outputs) }}"
          echo "Changed python test modules: ${{ steps.changed-tests.outputs.tests_all_modified_files }}"

      - name: Debug GitHub context
        run: |
          echo "Git reference: ${{ github.ref }}"
          echo "Git head ref: ${{ github.head_ref }}"
          echo "Git base ref: ${{ github.base_ref }}"

      - name: Log in to Docker Hub
        if: ${{  steps.changed-tests.outputs.tests_any_changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository }}
        uses: docker/login-action@v3
        with:
          username: winsvega
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Install deps
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        run: |
          echo $(pwd)
          echo ${{ github.workspace }}

      - name: Set up uv
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        uses: ./.github/actions/setup-uv

      - name: Set up Python
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        run: uv python install 3.10

      - name: Install EEST
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        run: |
          uv sync --no-progress
          uv run python --version

      # Required to fill .py tests
      - name: Build EVMONE EVM
        uses: ./.github/actions/build-evm-client/evmone
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        id: evm-builder2
        with:
          type: "main"

      - name: Checkout ethereum/tests
        uses: actions/checkout@v4
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        with:
          repository: ethereum/tests
          path: testpath
          sparse-checkout: |
            GeneralStateTests
            EOFTests

      - name: Checkout ethereum/legacytests
        uses: actions/checkout@v4
        if: steps.changed-tests.outputs.tests_any_changed == 'true'      
        with:
          repository: ethereum/legacytests
          path: legacytestpath
          sparse-checkout: |
            Cancun/GeneralStateTests

      # This command diffs the file and filters in new lines
      - name: Parse converted tests from converted-ethereum-tests.txt
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        run: |
          echo "New lines introduced in converted-ethereum-tests.txt:"
          lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true)
          if [ -z "$lines" ]; then
              echo "No new lines in converted-ethereum-tests.txt, check updates instead:"
              echo "converted_skip=true" >> $GITHUB_ENV
              exit 0
          else
              echo "converted_skip=false" >> $GITHUB_ENV
          fi

          files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
          for file in $files; do
              echo $file
          done
          echo "----------------"
          echo "Discovered existing json tests that will be BASE files:"


          BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
          mkdir -p $BASE_TESTS_PATH
          for file in $files; do
              # Make sure each file exist at least in develop or legacy tests
              file_found=0
              file_path=${{ github.workspace }}/testpath/$file
              if [ -e "$file_path" ]; then
                file_found=1
                cp $file_path $BASE_TESTS_PATH
                echo $file_path
              fi

              # Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
              if [[ "$file" == *"GeneralStateTests"* ]]; then
                  file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
                  base_name=$(basename "$file")
                  legacy_file_name="legacy_$base_name"
                  if [ -e "$file_path" ]; then
                    file_found=1
                    cp $file_path $BASE_TESTS_PATH/$legacy_file_name
                    echo $file_path
                  fi
              fi

              if [ $file_found -eq 0 ]; then
                echo "Error: Failed to find the test file $file in test repo"
                exit 1
              fi
          done

      # This command diffs the .py scripts introduced by a PR
      - name: Parse and fill introduced test sources
        if: steps.changed-tests.outputs.tests_any_changed == 'true'
        env:
          CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }}
        run: |
          source $GITHUB_ENV
          files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')

          # fill new tests
          # using `|| true` here because if no tests found, pyspec fill returns error code
          mkdir -p fixtures/state_tests
          mkdir -p fixtures/eof_tests

	  echo "uv run fill $files --until=Cancun --evm-bin evmone-t8n >> filloutput.log 2>&1"
          uv run fill $files --until=Cancun --evm-bin evmone-t8n >> filloutput.log 2>&1

          if grep -q "FAILURES" filloutput.log; then
              echo "Error: failed to generate .py tests."
              exit 1
          fi

          filesState=$(find fixtures/state_tests -type f -name "*.json")
          filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
          if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
              echo "Error: No filled JSON fixtures found in fixtures."
              exit 1
          fi

          # Include basic evm operations into coverage by default
          # As when we translate from yul/solidity some dup/push opcodes could become untouched
          uv run fill tests/homestead/coverage/test_coverage.py --until=Cancun --evm-bin evmone-t8n

          PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
          mkdir -p $PATCH_TEST_PATH
          find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
          find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;

      - name: Parse and fill introduced test sources from before the PR
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && env.converted_skip == 'true' }}
        env:
          CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_modified_files }}
        run: |
            echo "--------------------"
            echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"

            source $GITHUB_ENV
            files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')

            git checkout main
            PREV_COMMIT=$(git rev-parse HEAD)
            echo "Checkout head $PREV_COMMIT"

            # Take only those files that exist in the filesystem (ignore newly created files)
            files_fixed=$(echo "$files" | tr ' ' '\n' | while read file; do
              if [ -f "$file" ]; then
                echo "$file"
              fi
            done | tr '\n' ' ')

            echo "Select files that were changed and exist on the main branch:"
            echo $files_fixed

            rm -r fixtures
            rm filloutput.log
            mkdir -p fixtures/state_tests
            mkdir -p fixtures/eof_tests

            if [ -n "$files_fixed" ]; then
                uv run fill $files_fixed --until=Cancun --evm-bin evmone-t8n >> filloutput.log 2>&1

                if grep -q "FAILURES" filloutput.log; then
                  echo "Error: failed to generate .py tests from before the PR."
                  exit 1
                fi

                if grep -q "ERROR collecting test session" filloutput.log; then
                  echo "Error: failed to generate .py tests from before the PR."
                  exit 1
                fi
            fi

            filesState=$(find fixtures/state_tests -type f -name "*.json")
            filesEOF=$(find fixtures/eof_tests -type f -name "*.json")

            BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
            mkdir -p $BASE_TEST_PATH
            find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
            find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
            for file in $BASE_TEST_PATH/*.json; do
                if [ -e "$file" ]; then
                    mv "$file" "${file%.json}_$PREV_COMMIT.json"
                fi
            done

      - name: Print tests that will be covered
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        run: |
          echo "Original BASE tests:"
          ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
          echo "--------------------"
          echo "Ported PATCH tests:"
          ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS

      - name: Run coverage of the BASE tests
        uses: addnab/docker-run-action@v3
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/BASE_TESTS --outputname=BASE

      - name: Run coverage of the PATCH tests
        uses: addnab/docker-run-action@v3
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/PATCH_TESTS --outputname=PATCH

      - name: Run coverage DIFF of the PATCH tests compared to BASE tests
        uses: addnab/docker-run-action@v3
        if: ${{  steps.changed-tests.outputs.tests_any_changed == 'true' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov

      - name: Chmod coverage results
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        run: |
          user=$(whoami)
          sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage

      - name: Upload coverage results
        uses: actions/upload-artifact@v3
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        with:
          name: coverage-diff-native
          path: ${{ github.workspace }}/evmtest_coverage/coverage

      - name: Verify coverage results
        uses: addnab/docker-run-action@v3
        if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /check.sh