From e99873d97a3b27d55c9bb9dc982381242315645a Mon Sep 17 00:00:00 2001 From: Anton Pirker Date: Thu, 5 Sep 2024 14:13:13 +0200 Subject: [PATCH] Better test coverage reports (#3498) Our coverage reports are broken. This PR tries to fix them. - Sometimes the coverage report XML files contain references to files in `/tmp/...` (this can happen if dependencies write those files) so the first change is to omit those files. - We created our coverage reports with `coverage xml -i` where the `-i` means "ignore errors". This is why we never found out about problems generating coverage reports. Report generation fails now verbose (everywhere except in Python 3.6, because there are always some errors there because it can not parse python files with async code, but I guess those can be savely ignored) - For Python 3.6 we know have a special coverage config (`.coveragerc36`) because the option `exclude_also` was named `exclude_lines` in older coverage.py versions. --- .coveragerc36 | 14 +++++++++++ .github/workflows/test-integrations-ai.yml | 24 ++++++++++++++----- .../test-integrations-aws-lambda.yml | 12 +++++++--- .../test-integrations-cloud-computing.yml | 24 ++++++++++++++----- .../workflows/test-integrations-common.yml | 12 +++++++--- .../test-integrations-data-processing.yml | 24 ++++++++++++++----- .../workflows/test-integrations-databases.yml | 24 ++++++++++++++----- .../workflows/test-integrations-graphql.yml | 24 ++++++++++++++----- .../test-integrations-miscellaneous.yml | 24 ++++++++++++++----- .../test-integrations-networking.yml | 24 ++++++++++++++----- .../test-integrations-web-frameworks-1.yml | 24 ++++++++++++++----- .../test-integrations-web-frameworks-2.yml | 24 ++++++++++++++----- .gitignore | 4 +++- pyproject.toml | 15 +++++++++--- pytest.ini | 2 +- .../templates/test_group.jinja | 13 +++++++--- tox.ini | 4 +++- 17 files changed, 223 insertions(+), 69 deletions(-) create mode 100644 .coveragerc36 diff --git a/.coveragerc36 b/.coveragerc36 new file mode 100644 index 0000000000..722557bf6c --- /dev/null +++ b/.coveragerc36 @@ -0,0 +1,14 @@ +# This is the coverage.py config for Python 3.6 +# The config for newer Python versions is in pyproject.toml. + +[run] +branch = true +omit = + /tmp/* + */tests/* + */.venv/* + + +[report] +exclude_lines = + "if TYPE_CHECKING:", diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml index b3d96dfab3..c3c8f7a689 100644 --- a/.github/workflows/test-integrations-ai.yml +++ b/.github/workflows/test-integrations-ai.yml @@ -65,11 +65,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -127,11 +133,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-huggingface_hub" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-aws-lambda.yml b/.github/workflows/test-integrations-aws-lambda.yml index daab40a91d..10e319f8a2 100644 --- a/.github/workflows/test-integrations-aws-lambda.yml +++ b/.github/workflows/test-integrations-aws-lambda.yml @@ -84,11 +84,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-aws_lambda" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-cloud-computing.yml b/.github/workflows/test-integrations-cloud-computing.yml index 86ecab6f8e..94dd3473cd 100644 --- a/.github/workflows/test-integrations-cloud-computing.yml +++ b/.github/workflows/test-integrations-cloud-computing.yml @@ -61,11 +61,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-gcp-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -119,11 +125,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-gcp" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-common.yml b/.github/workflows/test-integrations-common.yml index 52baefd5b1..dbb3cb5d53 100644 --- a/.github/workflows/test-integrations-common.yml +++ b/.github/workflows/test-integrations-common.yml @@ -49,11 +49,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-common" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-data-processing.yml b/.github/workflows/test-integrations-data-processing.yml index 97fd913c44..6eb3a9f71f 100644 --- a/.github/workflows/test-integrations-data-processing.yml +++ b/.github/workflows/test-integrations-data-processing.yml @@ -79,11 +79,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-spark-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -155,11 +161,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-spark" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-databases.yml b/.github/workflows/test-integrations-databases.yml index d740912829..eca776d1c4 100644 --- a/.github/workflows/test-integrations-databases.yml +++ b/.github/workflows/test-integrations-databases.yml @@ -88,11 +88,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-sqlalchemy-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -173,11 +179,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-sqlalchemy" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-graphql.yml b/.github/workflows/test-integrations-graphql.yml index 6a499fa355..c89423327a 100644 --- a/.github/workflows/test-integrations-graphql.yml +++ b/.github/workflows/test-integrations-graphql.yml @@ -61,11 +61,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-strawberry-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -119,11 +125,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-strawberry" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-miscellaneous.yml b/.github/workflows/test-integrations-miscellaneous.yml index f5148fb2c8..492338c40e 100644 --- a/.github/workflows/test-integrations-miscellaneous.yml +++ b/.github/workflows/test-integrations-miscellaneous.yml @@ -65,11 +65,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-trytond-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -127,11 +133,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-trytond" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-networking.yml b/.github/workflows/test-integrations-networking.yml index 6a55ffadd8..fb55e708ae 100644 --- a/.github/workflows/test-integrations-networking.yml +++ b/.github/workflows/test-integrations-networking.yml @@ -61,11 +61,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-requests-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -119,11 +125,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-requests" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-web-frameworks-1.yml b/.github/workflows/test-integrations-web-frameworks-1.yml index 246248a700..01b391992d 100644 --- a/.github/workflows/test-integrations-web-frameworks-1.yml +++ b/.github/workflows/test-integrations-web-frameworks-1.yml @@ -79,11 +79,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-fastapi-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -155,11 +161,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-fastapi" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.github/workflows/test-integrations-web-frameworks-2.yml b/.github/workflows/test-integrations-web-frameworks-2.yml index cfc03a935a..310921a250 100644 --- a/.github/workflows/test-integrations-web-frameworks-2.yml +++ b/.github/workflows/test-integrations-web-frameworks-2.yml @@ -85,11 +85,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh "py${{ matrix.python-version }}-tornado-latest" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 @@ -167,11 +173,17 @@ jobs: run: | set -x # print commands that are executed ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-tornado" + - name: Generate coverage XML (Python 3.6) + if: ${{ !cancelled() && matrix.python-version == '3.6' }} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors - name: Generate coverage XML - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.python-version != '3.6' }} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: ${{ !cancelled() }} uses: codecov/codecov-action@v4.5.0 diff --git a/.gitignore b/.gitignore index cfd8070197..8c7a5f2174 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ *.db *.pid .python-version -.coverage* +.coverage +.coverage-sentry* +coverage.xml .junitxml* .DS_Store .tox diff --git a/pyproject.toml b/pyproject.toml index a2d2e0f7d0..7823c17a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,16 @@ extend-exclude = ''' | .*_pb2_grpc.py # exclude autogenerated Protocol Buffer files anywhere in the project ) ''' + +[tool.coverage.run] +branch = true +omit = [ + "/tmp/*", + "*/tests/*", + "*/.venv/*", +] + [tool.coverage.report] - exclude_also = [ - "if TYPE_CHECKING:", - ] \ No newline at end of file +exclude_also = [ + "if TYPE_CHECKING:", +] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index bece12f986..c03752b039 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -vvv -rfEs -s --durations=5 --cov=tests --cov=sentry_sdk --cov-branch --cov-report= --tb=short --junitxml=.junitxml +addopts = -vvv -rfEs -s --durations=5 --cov=./sentry_sdk --cov-branch --cov-report= --tb=short --junitxml=.junitxml asyncio_mode = strict markers = tests_internal_exceptions: Handle internal exceptions just as the SDK does, to test it. (Otherwise internal exceptions are recorded and reraised.) diff --git a/scripts/split-tox-gh-actions/templates/test_group.jinja b/scripts/split-tox-gh-actions/templates/test_group.jinja index 43d7081446..e63d6e0235 100644 --- a/scripts/split-tox-gh-actions/templates/test_group.jinja +++ b/scripts/split-tox-gh-actions/templates/test_group.jinja @@ -77,11 +77,18 @@ {% endif %} {% endfor %} + - name: Generate coverage XML (Python 3.6) + if: {% raw %}${{ !cancelled() && matrix.python-version == '3.6' }}{% endraw %} + run: | + export COVERAGE_RCFILE=.coveragerc36 + coverage combine .coverage-sentry-* + coverage xml --ignore-errors + - name: Generate coverage XML - if: {% raw %}${{ !cancelled() }}{% endraw %} + if: {% raw %}${{ !cancelled() && matrix.python-version != '3.6' }}{% endraw %} run: | - coverage combine .coverage* - coverage xml -i + coverage combine .coverage-sentry-* + coverage xml - name: Upload coverage to Codecov if: {% raw %}${{ !cancelled() }}{% endraw %} diff --git a/tox.ini b/tox.ini index dd1dbf1156..9c0092d7ba 100644 --- a/tox.ini +++ b/tox.ini @@ -683,7 +683,9 @@ deps = setenv = PYTHONDONTWRITEBYTECODE=1 OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES - COVERAGE_FILE=.coverage-{envname} + COVERAGE_FILE=.coverage-sentry-{envname} + py3.6: COVERAGE_RCFILE=.coveragerc36 + django: DJANGO_SETTINGS_MODULE=tests.integrations.django.myapp.settings common: TESTPATH=tests