From c7d2a03fc45efd5dd2e28e3e108dc98a53157dd7 Mon Sep 17 00:00:00 2001 From: Mike Fiedler Date: Thu, 22 Feb 2024 18:24:56 -0500 Subject: [PATCH 1/7] wip xdist across all cores Problem with coverage measurements Signed-off-by: Mike Fiedler --- pyproject.toml | 1 + requirements/tests.in | 1 + requirements/tests.txt | 9 +++++++++ tests/conftest.py | 4 ++-- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c93378a5ab24..3c61ce78a04e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ addopts = [ "--disable-socket", "--allow-hosts=localhost,::1,notdatadog,stripe", "--durations=20", + "--numprocesses=auto", ] cache_dir = "dev/.pytest_cache" norecursedirs = ['build', 'dist', 'node_modules', '*.egg-info', '.state requirements'] diff --git a/requirements/tests.in b/requirements/tests.in index 7460ee8adf5b..8f57969a550c 100644 --- a/requirements/tests.in +++ b/requirements/tests.in @@ -7,6 +7,7 @@ pytest-icdiff pytest-postgresql>=3.1.3,<7.0.0 pytest-randomly pytest-socket +pytest-xdist pytz responses>=0.5.1 webtest diff --git a/requirements/tests.txt b/requirements/tests.txt index f0b259662c18..743e98483cd0 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -158,6 +158,10 @@ coverage==7.5.4 \ --hash=sha256:e564c2cf45d2f44a9da56f4e3a26b2236504a496eb4cb0ca7221cd4cc7a9aca9 \ --hash=sha256:ed550e7442f278af76d9d65af48069f1fb84c9f745ae249c1a183c1e9d1b025c # via -r requirements/tests.in +execnet==2.0.2 \ + --hash=sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41 \ + --hash=sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af + # via pytest-xdist factory-boy==3.3.0 \ --hash=sha256:a2cdbdb63228177aa4f1c52f4b6d83fab2b8623bf602c7dedd7eb83c0f69c04c \ --hash=sha256:bc76d97d1a65bbd9842a6d722882098eb549ec8ee1081f9fb2e8ff29f0c300f1 @@ -238,6 +242,7 @@ pytest==8.2.2 \ # pytest-postgresql # pytest-randomly # pytest-socket + # pytest-xdist pytest-icdiff==0.9 \ --hash=sha256:13aede616202e57fcc882568b64589002ef85438046f012ac30a8d959dac8b75 \ --hash=sha256:efee0da3bd1b24ef2d923751c5c547fbb8df0a46795553fba08ef57c3ca03d82 @@ -254,6 +259,10 @@ pytest-socket==0.7.0 \ --hash=sha256:71ab048cbbcb085c15a4423b73b619a8b35d6a307f46f78ea46be51b1b7e11b3 \ --hash=sha256:7e0f4642177d55d317bbd58fc68c6bd9048d6eadb2d46a89307fa9221336ce45 # via -r requirements/tests.in +pytest-xdist==3.5.0 \ + --hash=sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a \ + --hash=sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24 + # via -r requirements/tests.in python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 diff --git a/tests/conftest.py b/tests/conftest.py index 2f5fa57c445b..c12bded74f63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -249,12 +249,12 @@ def cli(): @pytest.fixture(scope="session") -def database(request): +def database(request, worker_id): config = get_config(request) pg_host = config.get("host") pg_port = config.get("port") or os.environ.get("PGPORT", 5432) pg_user = config.get("user") - pg_db = config.get("db", "tests") + pg_db = f"tests-{worker_id}" pg_version = config.get("version", 16.1) janitor = DatabaseJanitor( From 8209142bdfd3ecccbb3cfa176e8820673efd2fc5 Mon Sep 17 00:00:00 2001 From: Mike Fiedler Date: Tue, 9 Apr 2024 15:57:31 -0400 Subject: [PATCH 2/7] trying parallel Signed-off-by: Mike Fiedler --- bin/tests | 1 + pyproject.toml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/bin/tests b/bin/tests index ce9533a755ab..387e0170d423 100755 --- a/bin/tests +++ b/bin/tests @@ -38,5 +38,6 @@ mkdir -p warehouse/static/dist/ # Actually run our tests. python -m coverage run -m pytest --strict-markers $COMMAND_ARGS +python -m coverage combine python -m coverage html --show-contexts python -m coverage report -m --fail-under 100 diff --git a/pyproject.toml b/pyproject.toml index 3c61ce78a04e..99f3eb4e4bce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ omit = [ # And again, tiny shim code. "warehouse/celery.py", ] +# We run multiple processes in parallel, so we need to make sure that we don't +# write to the same file at the same time. +parallel = true [tool.coverage.report] exclude_lines = [ From 72426db9e9ff49f82b664df937e8334f28640971 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 1 Jul 2024 17:05:13 -0400 Subject: [PATCH 3/7] Try to fix coverage --- .coveragerc | 30 ++++++++++++++++++++++++++++++ bin/tests | 2 +- docker-compose.yml | 2 ++ sitecustomize.py | 5 +++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .coveragerc create mode 100644 sitecustomize.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000000..29c01c9b1246 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,30 @@ +[run] +# We run multiple processes in parallel, so we need to make sure that we don't +# write to the same file at the same time. +parallel = true + +branch = true +dynamic_context = test_function +source = warehouse +omit = + # We don't want to get coverage information for our migrations. + warehouse/migrations/* + + # We don't want to actually cover our __main__.py file because it is hard + # to cover and it really just acts as a tiny shim to a function. + warehouse/__main__.py + + # Again, tiny shim code that we don't actually need to test and trying to + # do so would just get in the way. + warehouse/wsgi.py + + # And again, tiny shim code. + warehouse/celery.py + +[report] +exclude_lines = + pragma: no cover + class \\w+\\(Interface\\): + if (typing\\.)?TYPE_CHECKING: + +skip_covered = true diff --git a/bin/tests b/bin/tests index 387e0170d423..6d9f8b963c28 100755 --- a/bin/tests +++ b/bin/tests @@ -6,7 +6,7 @@ set -e # correct. export LC_ALL="${ENCODING:-en_US.UTF-8}" export LANG="${ENCODING:-en_US.UTF-8}" - +export COVERAGE_PROCESS_START="$(find .. -type f -name '.coveragerc' -exec realpath {} \;)" COMMAND_ARGS="$@" # Test the postgres connection diff --git a/docker-compose.yml b/docker-compose.yml index e96214a9eb5b..17b6db35ead3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,6 +87,8 @@ services: - ./htmlcov:/opt/warehouse/src/htmlcov:z - ./setup.cfg:/opt/warehouse/src/setup.cfg:z - ./pyproject.toml:/opt/warehouse/src/pyproject.toml:z + - ./.coveragerc:/opt/warehouse/src/.coveragerc:z + - ./sitecustomize.py:/opt/warehouse/src/sitecustomize.py:z - packages:/var/opt/warehouse/packages - packages-archive:/var/opt/warehouse/packages-archive - sponsorlogos:/var/opt/warehouse/sponsorlogos diff --git a/sitecustomize.py b/sitecustomize.py new file mode 100644 index 000000000000..764a7c76211c --- /dev/null +++ b/sitecustomize.py @@ -0,0 +1,5 @@ +try: + import coverage + coverage.process_startup() +except ImportError: + pass From d5d4892b25652caf9b558d77cc5b04e9a95f85d9 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 1 Jul 2024 17:15:17 -0400 Subject: [PATCH 4/7] Fix coverage --- .coveragerc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 29c01c9b1246..0485db1522ea 100644 --- a/.coveragerc +++ b/.coveragerc @@ -22,9 +22,8 @@ omit = warehouse/celery.py [report] -exclude_lines = - pragma: no cover - class \\w+\\(Interface\\): - if (typing\\.)?TYPE_CHECKING: +exclude_also = + class \w+\(Interface\): + if (typing\.)?TYPE_CHECKING: skip_covered = true From fb8a839cee126da2a43be366e4c0a62f174be791 Mon Sep 17 00:00:00 2001 From: Alexis Date: Mon, 1 Jul 2024 18:53:55 -0400 Subject: [PATCH 5/7] Remove .coveragerc and define COVERAGE_PROCESS_START using `pyproject.toml` --- .coveragerc | 29 ----------------------------- bin/tests | 2 +- docker-compose.yml | 1 - 3 files changed, 1 insertion(+), 31 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 0485db1522ea..000000000000 --- a/.coveragerc +++ /dev/null @@ -1,29 +0,0 @@ -[run] -# We run multiple processes in parallel, so we need to make sure that we don't -# write to the same file at the same time. -parallel = true - -branch = true -dynamic_context = test_function -source = warehouse -omit = - # We don't want to get coverage information for our migrations. - warehouse/migrations/* - - # We don't want to actually cover our __main__.py file because it is hard - # to cover and it really just acts as a tiny shim to a function. - warehouse/__main__.py - - # Again, tiny shim code that we don't actually need to test and trying to - # do so would just get in the way. - warehouse/wsgi.py - - # And again, tiny shim code. - warehouse/celery.py - -[report] -exclude_also = - class \w+\(Interface\): - if (typing\.)?TYPE_CHECKING: - -skip_covered = true diff --git a/bin/tests b/bin/tests index 6d9f8b963c28..c05763edff65 100755 --- a/bin/tests +++ b/bin/tests @@ -6,7 +6,7 @@ set -e # correct. export LC_ALL="${ENCODING:-en_US.UTF-8}" export LANG="${ENCODING:-en_US.UTF-8}" -export COVERAGE_PROCESS_START="$(find .. -type f -name '.coveragerc' -exec realpath {} \;)" +export COVERAGE_PROCESS_START="$(pwd)/pyproject.toml" COMMAND_ARGS="$@" # Test the postgres connection diff --git a/docker-compose.yml b/docker-compose.yml index 17b6db35ead3..8105f1d24c0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,6 @@ services: - ./htmlcov:/opt/warehouse/src/htmlcov:z - ./setup.cfg:/opt/warehouse/src/setup.cfg:z - ./pyproject.toml:/opt/warehouse/src/pyproject.toml:z - - ./.coveragerc:/opt/warehouse/src/.coveragerc:z - ./sitecustomize.py:/opt/warehouse/src/sitecustomize.py:z - packages:/var/opt/warehouse/packages - packages-archive:/var/opt/warehouse/packages-archive From 3906d4fb1007b2e8426acbe2077648efa2286567 Mon Sep 17 00:00:00 2001 From: Alexis Date: Tue, 2 Jul 2024 09:30:09 +0200 Subject: [PATCH 6/7] Add comments to describe the usage of sitecustomize.py --- docker-compose.yml | 2 ++ sitecustomize.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 8105f1d24c0a..deae656d2912 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,6 +87,8 @@ services: - ./htmlcov:/opt/warehouse/src/htmlcov:z - ./setup.cfg:/opt/warehouse/src/setup.cfg:z - ./pyproject.toml:/opt/warehouse/src/pyproject.toml:z + # Site customization shim to enable multiprocess coverage collection in tests. + # See: https://coverage.readthedocs.io/en/latest/subprocess.html - ./sitecustomize.py:/opt/warehouse/src/sitecustomize.py:z - packages:/var/opt/warehouse/packages - packages-archive:/var/opt/warehouse/packages-archive diff --git a/sitecustomize.py b/sitecustomize.py index 764a7c76211c..bcee1a264894 100644 --- a/sitecustomize.py +++ b/sitecustomize.py @@ -1,3 +1,6 @@ +# Site customization shim to enable multiprocess coverage collection in tests. +# See: https://coverage.readthedocs.io/en/latest/subprocess.html + try: import coverage coverage.process_startup() From fc8a9fbdb14bb2434771ad0acd2a6a6ef0b0f4a3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 2 Jul 2024 10:12:44 -0400 Subject: [PATCH 7/7] sitecustomize: fix licenses Signed-off-by: William Woodruff --- sitecustomize.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sitecustomize.py b/sitecustomize.py index bcee1a264894..2fa0b1ddf31f 100644 --- a/sitecustomize.py +++ b/sitecustomize.py @@ -1,8 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Site customization shim to enable multiprocess coverage collection in tests. # See: https://coverage.readthedocs.io/en/latest/subprocess.html try: import coverage + coverage.process_startup() except ImportError: pass