Skip to content

Commit

Permalink
enable coverage on PRs
Browse files Browse the repository at this point in the history
  • Loading branch information
@winsvega
winsvega committed Jul 16, 2024
1 parent f5797b5 commit 43a75be
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 18 deletions.
94 changes: 77 additions & 17 deletions .github/workflows/coverage.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: Evmone Coverage Report
on:
pull_request:
paths:
- 'converted-ethereum-tests.txt' # This triggers the workflow only for changes in file.txt
- 'tests/**' # This triggers the workflow for any changes in the tests folder

jobs:
evmone-coverage-diff:
Expand Down Expand Up @@ -74,14 +74,16 @@ jobs:
- name: Parse converted tests from converted-ethereum-tests.txt
run: |
echo "New lines introduced in converted-ethereum-tests.txt:"
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++")
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
if [ -z "$files" ]; then
echo "Error: No new JSON files found in converted-ethereum-tests.txt"
exit 1
lines=$(git diff origin/${{ github.base_ref }} HEAD -- converted-ethereum-tests.txt | grep "^+" | grep -v "^+++" || true)
if [ -z "$lines" ]; then
echo "No new lines in converted-ethereum-tests.txt, check updates instead:"
echo "converted_skip=true" >> $GITHUB_ENV
exit 0
else
echo "converted_skip=false" >> $GITHUB_ENV
fi
files=$(echo "$lines" | grep -oP '(?<=\+).+\.json')
for file in $files; do
echo $file
done
Expand Down Expand Up @@ -118,7 +120,7 @@ jobs:
exit 1
fi
done
# This command diffs the .py scripts introduced by a PR
- name: Parse and fill introduced test sources
Expand Down Expand Up @@ -149,14 +151,30 @@ jobs:
# fill new tests
# using `|| true` here because if no tests found, pyspec fill returns error code
FOUND_TEST=false
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
echo "$files" | while read line; do
file=$(echo "$line" | cut -c 3-)
fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
if grep -q "def test_" $file; then
FOUND_TEST=true
fill $file --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill $file --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
fi
done
if [[ "$FOUND_TEST" == "false" ]]; then
if [ "${{ env.converted_skip }}" == 'false' ]; then
echo "Error converted-ethereum-tests.txt has new lines, but no test in .py files were found"
exit 1
fi
echo "No test detected in changed .py files, exiting the script"
echo "coverage_skip=true" >> $GITHUB_ENV
exit 0
else
echo "coverage_skip=false" >> $GITHUB_ENV
fi
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests."
exit 1
Expand All @@ -182,8 +200,50 @@ jobs:
find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
if [ "${{ env.converted_skip }}" == 'true' ]; then
echo "--------------------"
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
git checkout main
PREV_COMMIT=$(git rev-parse HEAD)
echo "Checkout head $PREV_COMMIT"
rm -r fixtures
rm filloutput.log
rm filloutputEOF.log
mkdir -p fixtures/state_tests
mkdir -p fixtures/eof_tests
echo "$files" | while read line; do
file=$(echo "$line" | cut -c 3-)
if [ ! -f "$file" ]; then
continue
fi
if grep -q "def test_" $file; then
fill "$file" --until=Cancun --evm-bin evmone-t8n || true >> filloutput.log 2>&1
(fill "$file" --fork=CancunEIP7692 --evm-bin evmone-t8n -k eof_test || true) > >(tee -a filloutput.log filloutputEOF.log) 2>&1
fi
done
if grep -q "FAILURES" filloutput.log; then
echo "Error: failed to generate .py tests from before the PR."
exit 1
fi
filesState=$(find fixtures/state_tests -type f -name "*.json")
filesEOF=$(find fixtures/eof_tests -type f -name "*.json")
if [ -z "$filesState" ] && [ -z "$filesEOF" ]; then
echo "Error: No filled JSON fixtures found in fixtures from before the PR."
exit 1
fi
BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
mkdir -p $BASE_TEST_PATH
find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
find fixtures/eof_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
fi
- name: Print tests that will be covered
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
run: |
echo "Original BASE tests:"
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
Expand All @@ -193,44 +253,44 @@ jobs:
- name: Run coverage of the BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/BASE_TESTS --outputname=BASE

- name: Run coverage of the PATCH tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=cover --driver=${{ matrix.driver }} --testpath=/tests/PATCH_TESTS --outputname=PATCH

- name: Run coverage DIFF of the PATCH tests compared to BASE tests
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov

- name: Chmod coverage results
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
run: |
user=$(whoami)
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
- name: Upload coverage results
uses: actions/upload-artifact@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
name: coverage-diff
path: ${{ github.workspace }}/evmtest_coverage/coverage

- name: Verify coverage results
uses: addnab/docker-run-action@v3
if: ${{ env.retesteth_skip == 'false' || matrix.driver == 'native' }}
if: ${{ (env.retesteth_skip == 'false' || matrix.driver == 'native') && env.coverage_skip == 'false' }}
with:
image: winsvega/evmone-coverage-script:latest
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
Expand Down
2 changes: 1 addition & 1 deletion converted-ethereum-tests.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
([#647](https://github.com/ethereum/execution-spec-tests/pull/647))
EOFTests/efValidation/EOF1_returncontract_invalid_.json
EOFTests/efValidation/EOF1_returncontract_invalid_.json
EOFTests/efValidation/EOF1_returncontract_valid_.json

([#440](https://github.com/ethereum/execution-spec-tests/pull/440))
Expand Down
1 change: 1 addition & 0 deletions tests/homestead/coverage/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
"""
Tests that fill coverage gaps when porting over from `ethereum/tests`.
Unrelated change
"""

0 comments on commit 43a75be

Please sign in to comment.