coverage.yaml

name: Evmone Coverage Report

on:
  pull_request:
    paths:
      - 'converted-ethereum-tests.txt' # This triggers the workflow only for changes in file.txt

jobs:
  evmone-coverage-diff:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        driver: [retesteth, native]

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Fetch target branch
        run: git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v3
        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
        with:
          username: winsvega
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      - name: Install deps
        run: |
          echo $(pwd)
          echo ${{ github.workspace }}

          #install pyspec deps from root repo
          python3 --version
          pip install --upgrade pip
          python3 -m venv ./venv/
          source ./venv/bin/activate
          pip install -e .
          solc-select use 0.8.25 --always-install

      # Required to fill .py tests
      - name: Build GO EVM
        uses: ./.github/actions/build-evm-client/geth
        id: evm-builder
        with:
          type: 'main'

      - name: Build EVMONE EVM
        uses: ./.github/actions/build-evm-client/evmone
        id: evm-builder2
        with:
          type: 'main'

      - name: Checkout ethereum/tests
        uses: actions/checkout@v4
        with:
          repository: ethereum/tests
          path: testpath
          sparse-checkout: |
            GeneralStateTests
            EOFTests

      - name: Checkout ethereum/legacytests
        uses: actions/checkout@v4
        with:
          repository: ethereum/legacytests
          path: legacytestpath
          sparse-checkout: |
            Cancun/GeneralStateTests


      # This command diffs the file and filters in new lines
      - name: Parse converted tests from converted-ethereum-tests.txt
        run: |
          echo "New lines introduced in converted-ethereum-tests.txt:"
          lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++")
          files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')

          if [ -z "$files" ]; then
             echo "Error: No new JSON files found in converted-ethereum-tests.txt"
              exit 1
          fi

          for file in $files; do
              echo $file
          done
          echo "----------------"
          echo "Discovered existing json tests that will be BASE files:"


          BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
          mkdir -p $BASE_TESTS_PATH
          for file in $files; do
              # Make sure each file exist at least in develop or legacy tests
              file_found=0
              file_path=${{ github.workspace }}/testpath/$file
              if [ -e "$file_path" ]; then
                file_found=1
                cp $file_path $BASE_TESTS_PATH
                echo $file_path
              fi

              # Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
              if [[ "$file" == *"GeneralStateTests"* ]]; then
                  file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
                  base_name=$(basename "$file")
                  legacy_file_name="legacy_$base_name"
                  if [ -e "$file_path" ]; then
                    file_found=1
                    cp $file_path $BASE_TESTS_PATH/$legacy_file_name
                    echo $file_path
                  fi
              fi

              if [ $file_found -eq 0 ]; then
               echo "Error: Failed to find the test file $file in test repo"
                exit 1
              fi
          done
          

     # This command diffs the .py scripts introduced by a PR
      - name: Parse and fill introduced test sources
        run: |
          python3 -m venv ./venv/
          source ./venv/bin/activate

          if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
              # Fetch changes when PR comes from remote repo
              git fetch origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
              git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/origin/PR-${{ github.event.pull_request.number }}
              files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
          else
              # Fetch the base branch and the head branch
              git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
              git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }}

              # Perform the diff
              files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
          fi


          echo "Modified or new .py files in tests folder:"
          echo "$files" | while read line; do
            file=$(echo "$line" | cut -c 3-)
            echo $file
          done

          # fill new tests
          # using `|| true` here because if no tests found, pyspec fill returns error code
          mkdir -p fixtures/state_tests
          mkdir -p fixtures/eof_tests
          echo "$files" | while read line; do
            file=$(echo "$line" | cut -c 3-)
            fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
            (fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
          done

          if grep -q "FAILURES" filloutput.log; then
             echo "Error: failed to generate .py tests."
              exit 1
          fi
          if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
              echo "Disabling retesteth coverage check as EOF tests detected!"
              echo "retesteth_skip=true" >> $GITHUB_ENV
              exit 0
          else
              echo "retesteth_skip=false" >> $GITHUB_ENV
          fi


          filesState=$(find fixtures/state_tests -type f -name "*.json")
          filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
          if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
              echo "Error: No filled JSON fixtures found in fixtures."
               exit 1
          fi

          PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
          mkdir -p $PATCH_TEST_PATH
          find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
          find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;

      - name: Print tests that will be covered
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        run: |
          echo "Original BASE tests:"
          ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
          echo "--------------------"
          echo "Ported PATCH tests:"
          ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS

      - name: Run coverage of the BASE tests
        uses: addnab/docker-run-action@v3
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE

      - name: Run coverage of the PATCH tests
        uses: addnab/docker-run-action@v3
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH

      - name: Run coverage DIFF of the PATCH tests compared to BASE tests
        uses: addnab/docker-run-action@v3
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov

      - name: Chmod coverage results
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        run: |
          user=$(whoami)
          sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage

      - name: Upload coverage results
        uses: actions/upload-artifact@v3
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        with:
          name: coverage-diff
          path: ${{ github.workspace }}/evmtest_coverage/coverage

      - name: Verify coverage results
        uses: addnab/docker-run-action@v3
        if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
        with:
          image: winsvega/evmone-coverage-script:latest
          options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
          run: /check.sh