Better test coverage reports (#3498)

Our coverage reports are broken. This PR tries to fix them. - Sometimes the coverage report XML files contain references to files in `/tmp/...` (this can happen if dependencies write those files) so the first change is to omit those files. - We created our coverage reports with `coverage xml -i` where the `-i` means "ignore errors". This is why we never found out about problems generating coverage reports. Report generation fails now verbose (everywhere except in Python 3.6, because there are always some errors there because it can not parse python files with async code, but I guess those can be savely ignored) - For Python 3.6 we know have a special coverage config (`.coveragerc36`) because the option `exclude_also` was named `exclude_lines` in older coverage.py versions.
getsentry · Sep 5, 2024 · e99873d · e99873d
1 parent 0fb9606
commit e99873d
Show file tree

Hide file tree

Showing 17 changed files with 223 additions and 69 deletions.
diff --git a/.coveragerc36 b/.coveragerc36
@@ -0,0 +1,14 @@
+# This is the coverage.py config for Python 3.6
+# The config for newer Python versions is in pyproject.toml.
+
+[run]
+branch = true
+omit = 
+    /tmp/*
+    */tests/*
+    */.venv/*
+
+
+[report]
+exclude_lines = 
+    "if TYPE_CHECKING:",
diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml
@@ -65,11 +65,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -127,11 +133,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-huggingface_hub"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-aws-lambda.yml b/.github/workflows/test-integrations-aws-lambda.yml
@@ -84,11 +84,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-aws_lambda"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-cloud-computing.yml b/.github/workflows/test-integrations-cloud-computing.yml
@@ -61,11 +61,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-gcp-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -119,11 +125,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-gcp"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-common.yml b/.github/workflows/test-integrations-common.yml
@@ -49,11 +49,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-common"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-data-processing.yml b/.github/workflows/test-integrations-data-processing.yml
@@ -79,11 +79,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-spark-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -155,11 +161,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-spark"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-databases.yml b/.github/workflows/test-integrations-databases.yml
@@ -88,11 +88,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-sqlalchemy-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -173,11 +179,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-sqlalchemy"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-graphql.yml b/.github/workflows/test-integrations-graphql.yml
@@ -61,11 +61,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-strawberry-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -119,11 +125,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-strawberry"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-miscellaneous.yml b/.github/workflows/test-integrations-miscellaneous.yml
@@ -65,11 +65,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-trytond-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -127,11 +133,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-trytond"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-networking.yml b/.github/workflows/test-integrations-networking.yml
@@ -61,11 +61,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-requests-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -119,11 +125,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-requests"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0

diff --git a/.github/workflows/test-integrations-web-frameworks-1.yml b/.github/workflows/test-integrations-web-frameworks-1.yml
@@ -79,11 +79,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-fastapi-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0
@@ -155,11 +161,17 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-fastapi"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }} 
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
       - name: Generate coverage XML
-        if: ${{ !cancelled() }}
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }} 
         run: |
-          coverage combine .coverage*
-          coverage xml -i
+          coverage combine .coverage-sentry-*
+          coverage xml
       - name: Upload coverage to Codecov
         if: ${{ !cancelled() }}
         uses: codecov/codecov-action@v4.5.0