Skip to content

Commit

Permalink
coverage on the prs
Browse files Browse the repository at this point in the history
  • Loading branch information
@winsvega
winsvega committed Jul 16, 2024
1 parent f5797b5 commit 3407954
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 88 deletions.
128 changes: 109 additions & 19 deletions .github/workflows/coverage.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Evmone Coverage Report
on:
pull_request:
paths:
- 'converted-ethereum-tests.txt' # This triggers the workflow only for changes in file.txt
- 'tests/**' # This triggers the workflow for any changes in the tests folder

jobs:
evmone-coverage-diff:
Expand Down Expand Up @@ -74,14 +74,16 @@ jobs:
- name: Parse converted tests from converted-ethereum-tests.txt
run: |
echo "New lines introduced in converted-ethereum-tests.txt:"
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++")
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
if [ -z "$files" ]; then
echo "Error: No new JSON files found in converted-ethereum-tests.txt"
exit 1
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true)
if [ -z "$lines" ]; then
echo "No new lines in converted-ethereum-tests.txt, check updates instead:"
echo "converted_skip=true" >> $GITHUB_ENV
exit 0
else
echo "converted_skip=false" >> $GITHUB_ENV
fi
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
for file in $files; do
echo $file
done
Expand Down Expand Up @@ -118,7 +120,7 @@ jobs:
exit 1
fi
done
# This command diffs the .py scripts introduced by a PR
- name: Parse and fill introduced test sources
Expand Down Expand Up @@ -149,13 +151,32 @@ jobs:
# fill new tests
# using `|| true` here because if no tests found, pyspec fill returns error code
FOUND_TEST=false
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
echo "$files" | while read line; do
# Use a while loop with a here-string to avoid subshell issues
while IFS= read -r line; do
file=$(echo "$line" | cut -c 3-)
fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
done
if grep -q "def test_" "$file"; then
FOUND_TEST=true
fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
fi
done <<< "$files"
echo "Tests found: $FOUND_TEST"
if [[ "$FOUND_TEST" == "false" ]]; then
if [ "${{ env.converted_skip }}" == 'false' ]; then
echo "Error converted-ethereum-tests.txt has new lines, but no test in .py files were found"
exit 1
fi
echo "No test detected in changed .py files, exiting the script"
echo "coverage_skip=true" >> $GITHUB_ENV
exit 0
else
echo "coverage_skip=false" >> $GITHUB_ENV
fi
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests."
Expand All @@ -182,8 +203,77 @@ jobs:
find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
- name: Parse and fill introduced test sources from before the PR
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.converted_skip == 'true' && env.coverage_skip == 'false'}}
run: |
echo "--------------------"
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
else
files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
fi
echo "Modified or new .py files in tests folder:"
echo "$files" | while read line; do
file=$(echo "$line" | cut -c 3-)
echo $file
done
git checkout main
PREV_COMMIT=$(git rev-parse HEAD)
echo "Checkout head $PREV_COMMIT"
python3 -m venv ./venv/
source ./venv/bin/activate
rm -r fixtures
rm filloutput.log
rm filloutputEOF.log
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests

FOUND_TEST=false
while IFS= read -r line; do
file=$(echo "$line" | cut -c 3-)
if grep -q "def test_" "$file"; then
FOUND_TEST=true
fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
fi
done <<< "$files"

echo "Tests found: $FOUND_TEST"
if [[ "$FOUND_TEST" == "false" ]]; then
echo "No test detected in .py files from before the PR, exiting the script"
echo "coverage_skip=true" >> $GITHUB_ENV
exit 0
fi

if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests from before the PR."
exit 1
fi

filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
echo "Error: No filled JSON fixtures found in fixtures from before the PR."
exit 1
fi

BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
mkdir -p $BASE_TEST_PATH
find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
for file in $BASE_TEST_PATH/*.json; do
mv "$file" "${file%.json}_$PREV_COMMIT.json"
done


- name: Print tests that will be covered
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
run: |
echo "Original BASE tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
Expand All @@ -193,44 +283,44 @@ jobs:
- name: Run coverage of the BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE

- name: Run coverage of the PATCH tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH

- name: Run coverage DIFF of the PATCH tests compared to BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov

- name: Chmod coverage results
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
run: |
user=$(whoami)
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
- name: Upload coverage results
uses: actions/upload-artifact@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
name: coverage-diff
path: ${{ github.workspace }}/evmtest_coverage/coverage

- name: Verify coverage results
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
Expand Down
69 changes: 0 additions & 69 deletions tests/prague/eip7692_eof_v1/eip3540_eof_v1/test_example_valid_invalid.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,75 +65,6 @@
EOFException.MISSING_TERMINATOR,
id="headers_terminator_invalid",
),
pytest.param(
# Check that code that uses a new style relative jump succeeds
Container(
name="EOF1V0008",
sections=[
Section.Code(
code=Op.PUSH0
+ Op.RJUMPI[3]
+ Op.RJUMP[3]
+ Op.RJUMP[3]
+ Op.RJUMP[-6]
+ Op.STOP,
),
Section.Data("0x0bad60A7"),
],
),
"ef0001010004020001000E04000400008000015FE10003E00003E00003E0FFFA000bad60A7",
None,
id="rjump_valid",
),
pytest.param(
# Check that code that uses a new style conditional jump succeeds
Container(
name="EOF1V0011",
sections=[
Section.Code(
code=Op.PUSH1(1) + Op.RJUMPI[1] + Op.NOOP + Op.STOP,
),
Section.Data("0x0bad60A7"),
],
),
"ef0001010004020001000704000400008000016001E100015B000bad60A7",
None,
id="rjumpi_valid",
),
pytest.param(
# Sections that end with a legit terminating opcode are OK
Container(
name="EOF1V0014",
sections=[
Section.Code(
code=Op.PUSH0
+ Op.CALLDATALOAD
+ Op.RJUMPV[0, 3, 6, 9]
+ Op.JUMPF[1]
+ Op.JUMPF[2]
+ Op.JUMPF[3]
+ Op.CALLF[4]
+ Op.STOP,
),
Section.Code(
code=Op.PUSH0 + Op.PUSH0 + Op.RETURN,
),
Section.Code(
code=Op.PUSH0 + Op.PUSH0 + Op.REVERT,
),
Section.Code(code=Op.INVALID),
Section.Code(
code=Op.RETF,
code_outputs=0,
),
Section.Data("0x0bad60A7"),
],
),
"EF0001010014020005001900030003000100010400040000800001008000020080000200800000000"
"000005f35e2030000000300060009e50001e50002e50003e30004005f5ff35f5ffdfee40bad60a7",
None,
id="rjumpv_section_terminator_valid",
),
pytest.param(
# Check that jump tables work
Container(
Expand Down

0 comments on commit 3407954

Please sign in to comment.