-
Notifications
You must be signed in to change notification settings - Fork 88
237 lines (200 loc) · 9.3 KB
/
coverage.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
name: Evmone Coverage Report
on:
pull_request:
paths:
- 'converted-ethereum-tests.txt' # This triggers the workflow only for changes in file.txt
jobs:
evmone-coverage-diff:
runs-on: ubuntu-latest
strategy:
matrix:
driver: [retesteth, native]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Fetch target branch
run: git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
- name: Log in to Docker Hub
uses: docker/login-action@v3
if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
with:
username: winsvega
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Install deps
run: |
echo $(pwd)
echo ${{ github.workspace }}
#install pyspec deps from root repo
python3 --version
pip install --upgrade pip
python3 -m venv ./venv/
source ./venv/bin/activate
pip install -e .
solc-select use 0.8.25 --always-install
# Required to fill .py tests
- name: Build GO EVM
uses: ./.github/actions/build-evm
id: evm-builder
with:
type: 'main'
- name: Build EVMONE EVM
uses: ./.github/actions/build-evmone-evm
id: evm-builder2
with:
type: 'main'
- name: Checkout ethereum/tests
uses: actions/checkout@v4
with:
repository: ethereum/tests
path: testpath
sparse-checkout: |
GeneralStateTests
EOFTests
- name: Checkout ethereum/legacytests
uses: actions/checkout@v4
with:
repository: ethereum/legacytests
path: legacytestpath
sparse-checkout: |
Cancun/GeneralStateTests
# This command diffs the file and filters in new lines
- name: Parse converted tests from converted-ethereum-tests.txt
run: |
echo "New lines introduced in converted-ethereum-tests.txt:"
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++")
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
if [ -z "$files" ]; then
echo "Error: No new JSON files found in converted-ethereum-tests.txt"
exit 1
fi
for file in $files; do
echo $file
done
echo "----------------"
echo "Discovered existing json tests that will be BASE files:"
BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
mkdir -p $BASE_TESTS_PATH
for file in $files; do
# Make sure each file exist at least in develop or legacy tests
file_found=0
file_path=${{ github.workspace }}/testpath/$file
if [ -e "$file_path" ]; then
file_found=1
cp $file_path $BASE_TESTS_PATH
echo $file_path
fi
# Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
if [[ "$file" == *"GeneralStateTests"* ]]; then
file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
base_name=$(basename "$file")
legacy_file_name="legacy_$base_name"
if [ -e "$file_path" ]; then
file_found=1
cp $file_path $BASE_TESTS_PATH/$legacy_file_name
echo $file_path
fi
fi
if [ $file_found -eq 0 ]; then
echo "Error: Failed to find the test file $file in test repo"
exit 1
fi
done
# This command diffs the .py scripts introduced by a PR
- name: Parse and fill introduced test sources
run: |
python3 -m venv ./venv/
source ./venv/bin/activate
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
# Fetch changes when PR comes from remote repo
git fetch origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
git fetch origin +refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/origin/PR-${{ github.event.pull_request.number }}
files=$(git diff --name-status origin/${{ github.base_ref }}...origin/PR-${{ github.event.pull_request.number }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
else
# Fetch the base branch and the head branch
git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
git fetch origin ${{ github.head_ref }}:refs/remotes/origin/${{ github.head_ref }}
# Perform the diff
files=$(git diff --name-status origin/${{ github.base_ref }}...origin/${{ github.head_ref }} -- tests/ | grep -E '^[AM]' | grep '\.py$')
fi
echo "Modified or new .py files in tests folder:"
echo "$files" | while read line; do
file=$(echo "$line" | cut -c 3-)
echo $file
done
# fill new tests
# using `|| true` here because if no tests found, pyspec fill returns error code
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
echo "$files" | while read line; do
file=$(echo "$line" | cut -c 3-)
fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
done
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests."
exit 1
fi
if [ "${{ matrix.driver }}" = "retesteth" ] && grep -q "passed" filloutputEOF.log; then
echo "Disabling retesteth coverage check as EOF tests detected!"
echo "retesteth_skip=true" >> $GITHUB_ENV
exit 0
else
echo "retesteth_skip=false" >> $GITHUB_ENV
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
echo "Error: No filled JSON fixtures found in fixtures."
exit 1
fi
PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
mkdir -p $PATCH_TEST_PATH
find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
- name: Print tests that will be covered
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
run: |
echo "Original BASE tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
echo "--------------------"
echo "Ported PATCH tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
- name: Run coverage of the BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE
- name: Run coverage of the PATCH tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH
- name: Run coverage DIFF of the PATCH tests compared to BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov
- name: Chmod coverage results
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
run: |
user=$(whoami)
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
- name: Upload coverage results
uses: actions/upload-artifact@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
with:
name: coverage-diff
path: ${{ github.workspace }}/evmtest_coverage/coverage
- name: Verify coverage results
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /check.sh