From 28e1b5ab2f4d573a91705cdbb025e57023d264b1 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 11:15:26 -0400 Subject: [PATCH 01/23] Use .pth file to import distutils from setuptools --- MANIFEST.in | 1 + _distutils_importer/__init__.py | 6 ++++ .../distutils/__init__.py | 3 ++ distutils_precedence.pth | 1 + setup.py | 32 +++++++++++++++++++ 5 files changed, 43 insertions(+) create mode 100644 _distutils_importer/__init__.py create mode 100644 _distutils_importer/distutils-shim-package/distutils/__init__.py create mode 100644 distutils_precedence.pth diff --git a/MANIFEST.in b/MANIFEST.in index 128ae280ec..be83a7f336 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,4 +13,5 @@ include launcher.c include msvc-build-launcher.cmd include pytest.ini include tox.ini +include distutils_precedence.pth exclude pyproject.toml # Temporary workaround for #1644. diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py new file mode 100644 index 0000000000..54a825fed6 --- /dev/null +++ b/_distutils_importer/__init__.py @@ -0,0 +1,6 @@ +import sys + +here = os.path.dirname(__file__) +NEW_DISTUTILS_LOCATION = os.path.join(here, 'distutils-shim-package') + +sys.path.insert(0, NEW_DISTUTILS_LOCATION) diff --git a/_distutils_importer/distutils-shim-package/distutils/__init__.py b/_distutils_importer/distutils-shim-package/distutils/__init__.py new file mode 100644 index 0000000000..de098c72f7 --- /dev/null +++ b/_distutils_importer/distutils-shim-package/distutils/__init__.py @@ -0,0 +1,3 @@ +import setuptools.distutils_patch + +from distutils import * diff --git a/distutils_precedence.pth b/distutils_precedence.pth new file mode 100644 index 0000000000..2d9b996abc --- /dev/null +++ b/distutils_precedence.pth @@ -0,0 +1 @@ +import os; (os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') == 'local' and __import__('_distutils_importer')) diff --git a/setup.py b/setup.py index 1fe18bd13c..daab3e8796 100755 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import sys import setuptools +from setuptools.command.install import install here = os.path.dirname(__file__) @@ -49,6 +50,7 @@ def _gen_console_scripts(): package_data = dict( setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'], + _distutils_importer=['distutils-shim-package/distutils/__init__.py'], ) force_windows_specific_files = ( @@ -81,8 +83,38 @@ def pypi_link(pkg_filename): return '/'.join(parts) +class install_with_pth(install): + """ + Custom install command to install a .pth file for distutils patching. + + This is necessary because there's no standard way to install a `.pth` file + alongside your package (and there probably shouldn't be one), but we need + to do this in order to give precedence higher precedence to our version of + `distutils` than the standard library. + """ + + def initialize_options(self): + install.initialize_options(self) + + name = 'distutils_precedence' + with open(os.path.join(here, name + '.pth'), 'rt') as f: + contents = f.read() + + self.extra_path = (name, contents) + + def finalize_options(self): + install.finalize_options(self) + + install_suffix = os.path.relpath(self.install_lib, + self.install_libbase) + + if install_suffix == self.extra_path[1]: + self.install_lib = self.install_libbase + + setup_params = dict( src_root=None, + cmdclass={'install': install_with_pth}, package_data=package_data, entry_points={ "distutils.commands": [ From 03b36b5dc594bbe239d0ad66dc43ea7d1832072c Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 12:02:41 -0400 Subject: [PATCH 02/23] Remove warnings With the new `.pth` file, these warnings are no longer necessary. --- setuptools/distutils_patch.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/setuptools/distutils_patch.py b/setuptools/distutils_patch.py index 33f1e7f961..d01a1a1b10 100644 --- a/setuptools/distutils_patch.py +++ b/setuptools/distutils_patch.py @@ -12,26 +12,10 @@ import warnings -is_pypy = '__pypy__' in sys.builtin_module_names - - -def warn_distutils_present(): - if 'distutils' not in sys.modules: - return - if is_pypy and sys.version_info < (3, 7): - # PyPy for 3.6 unconditionally imports distutils, so bypass the warning - # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 - return - warnings.warn( - "Distutils was imported before Setuptools. This usage is discouraged " - "and may exhibit undesirable behaviors or errors. Please use " - "Setuptools' objects directly or at least import Setuptools first.") - - def clear_distutils(): if 'distutils' not in sys.modules: return - warnings.warn("Setuptools is replacing distutils.") + mods = [name for name in sys.modules if re.match(r'distutils\b', name)] for name in mods: del sys.modules[name] @@ -56,6 +40,5 @@ def ensure_local_distutils(): assert '_distutils' in core.__file__, core.__file__ -warn_distutils_present() if enabled(): ensure_local_distutils() From 89e9d3c8910c3f419eb9f1c2758a748c6938655b Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 14:43:23 -0400 Subject: [PATCH 03/23] Adjust distutils shim when removing _distutils_importer --- _distutils_importer/__init__.py | 17 ++++++++++++++--- setuptools/sandbox.py | 19 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py index 54a825fed6..498c4ac176 100644 --- a/_distutils_importer/__init__.py +++ b/_distutils_importer/__init__.py @@ -1,6 +1,17 @@ import sys -here = os.path.dirname(__file__) -NEW_DISTUTILS_LOCATION = os.path.join(here, 'distutils-shim-package') +_HERE = os.path.dirname(__file__) +NEW_DISTUTILS_LOCATION = os.path.join(_HERE, 'distutils-shim-package') -sys.path.insert(0, NEW_DISTUTILS_LOCATION) +def add_shim(): + if NEW_DISTUTILS_LOCATION not in sys.path: + sys.path.insert(0, NEW_DISTUTILS_LOCATION) + +def remove_shim(): + try: + sys.path.remove(NEW_DISTUTILS_LOCATION) + except ValueError: + pass + + +add_shim() diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 93ae8eb4d9..342a713f5c 100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -185,8 +185,8 @@ def setup_context(setup_dir): temp_dir = os.path.join(setup_dir, 'temp') with save_pkg_resources_state(): with save_modules(): - hide_setuptools() with save_path(): + hide_setuptools() with save_argv(): with override_temp(temp_dir): with pushd(setup_dir): @@ -195,6 +195,15 @@ def setup_context(setup_dir): yield +_MODULES_TO_HIDE = { + 'setuptools', + 'distutils', + 'pkg_resources', + 'Cython', + '_distutils_importer', +} + + def _needs_hiding(mod_name): """ >>> _needs_hiding('setuptools') @@ -212,8 +221,8 @@ def _needs_hiding(mod_name): >>> _needs_hiding('Cython') True """ - pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)') - return bool(pattern.match(mod_name)) + base_module = mod_name.split('.', 1)[0] + return base_module in _MODULES_TO_HIDE def hide_setuptools(): @@ -223,6 +232,10 @@ def hide_setuptools(): necessary to avoid issues such as #315 where setuptools upgrading itself would fail to find a function declared in the metadata. """ + _distutils_importer = sys.modules.get('_distutils_importer', None) + if _distutils_importer is not None: + _distutils_importer.remove_shim() + modules = filter(_needs_hiding, sys.modules) _clear_modules(modules) From 642604f82c01175f2ad285800d969ff521495af0 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 14:44:12 -0400 Subject: [PATCH 04/23] Clean up setuptools/__init__.py imports This puts non-distutils imports first and removes one unused import. --- setuptools/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 8388251115..d9740403d9 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,17 +1,16 @@ """Extensions to the 'distutils' for large or complex distributions""" -import os +from fnmatch import fnmatchcase import functools +import os +import re # Disabled for now due to: #2228, #2230 import setuptools.distutils_patch # noqa: F401 import distutils.core -import distutils.filelist -import re from distutils.errors import DistutilsOptionError from distutils.util import convert_path -from fnmatch import fnmatchcase from ._deprecation_warning import SetuptoolsDeprecationWarning From 370839b417f6bafe783fa040646d80bdf673fac4 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 15:25:32 -0400 Subject: [PATCH 05/23] Use import hook instead of sys.path manipulation --- _distutils_importer/__init__.py | 32 ++++++++++++++++--- .../distutils/__init__.py | 3 -- setup.py | 1 - 3 files changed, 27 insertions(+), 9 deletions(-) delete mode 100644 _distutils_importer/distutils-shim-package/distutils/__init__.py diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py index 498c4ac176..323ae2037c 100644 --- a/_distutils_importer/__init__.py +++ b/_distutils_importer/__init__.py @@ -1,15 +1,37 @@ import sys -_HERE = os.path.dirname(__file__) -NEW_DISTUTILS_LOCATION = os.path.join(_HERE, 'distutils-shim-package') + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + if path is not None or fullname != "distutils": + return None + + return self.get_distutils_spec() + + def get_distutils_spec(self): + import importlib + + class DistutilsLoader(importlib.util.abc.Loader): + + def create_module(self, spec): + return importlib.import_module('._distutils', 'setuptools') + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader('distutils', DistutilsLoader()) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + def add_shim(): - if NEW_DISTUTILS_LOCATION not in sys.path: - sys.path.insert(0, NEW_DISTUTILS_LOCATION) + sys.meta_path.insert(0, DISTUTILS_FINDER) + def remove_shim(): try: - sys.path.remove(NEW_DISTUTILS_LOCATION) + sys.path.remove(DISTUTILS_FINDER) except ValueError: pass diff --git a/_distutils_importer/distutils-shim-package/distutils/__init__.py b/_distutils_importer/distutils-shim-package/distutils/__init__.py deleted file mode 100644 index de098c72f7..0000000000 --- a/_distutils_importer/distutils-shim-package/distutils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import setuptools.distutils_patch - -from distutils import * diff --git a/setup.py b/setup.py index daab3e8796..cba37d3e59 100755 --- a/setup.py +++ b/setup.py @@ -50,7 +50,6 @@ def _gen_console_scripts(): package_data = dict( setuptools=['script (dev).tmpl', 'script.tmpl', 'site-patch.py'], - _distutils_importer=['distutils-shim-package/distutils/__init__.py'], ) force_windows_specific_files = ( From 85a0a9026d1b40448d1757ca6cd75e5cc2c50fc6 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Mon, 13 Jul 2020 15:36:39 -0400 Subject: [PATCH 06/23] Revert "Remove warnings" This reverts commit 30b883f0b8071a3b1472c884574f38ce0128e457. --- setuptools/distutils_patch.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/setuptools/distutils_patch.py b/setuptools/distutils_patch.py index d01a1a1b10..33f1e7f961 100644 --- a/setuptools/distutils_patch.py +++ b/setuptools/distutils_patch.py @@ -12,10 +12,26 @@ import warnings -def clear_distutils(): +is_pypy = '__pypy__' in sys.builtin_module_names + + +def warn_distutils_present(): if 'distutils' not in sys.modules: return + if is_pypy and sys.version_info < (3, 7): + # PyPy for 3.6 unconditionally imports distutils, so bypass the warning + # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 + return + warnings.warn( + "Distutils was imported before Setuptools. This usage is discouraged " + "and may exhibit undesirable behaviors or errors. Please use " + "Setuptools' objects directly or at least import Setuptools first.") + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + warnings.warn("Setuptools is replacing distutils.") mods = [name for name in sys.modules if re.match(r'distutils\b', name)] for name in mods: del sys.modules[name] @@ -40,5 +56,6 @@ def ensure_local_distutils(): assert '_distutils' in core.__file__, core.__file__ +warn_distutils_present() if enabled(): ensure_local_distutils() From e371422476f51a83d27d70dc45bbfba1544aad55 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 19 Jul 2020 21:36:33 -0400 Subject: [PATCH 07/23] Consolidate distutils importing hacks into _distutils_importer package. Generate distutils-precedence.pth inline. --- MANIFEST.in | 1 - _distutils_importer/__init__.py | 19 ++++++++++++++++--- _distutils_importer/install.py | 5 +++++ .../override.py | 18 ++---------------- conftest.py | 2 +- distutils_precedence.pth | 1 - setup.py | 8 ++------ setuptools/__init__.py | 3 +-- 8 files changed, 27 insertions(+), 30 deletions(-) create mode 100644 _distutils_importer/install.py rename setuptools/distutils_patch.py => _distutils_importer/override.py (79%) delete mode 100644 distutils_precedence.pth diff --git a/MANIFEST.in b/MANIFEST.in index be83a7f336..128ae280ec 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,5 +13,4 @@ include launcher.c include msvc-build-launcher.cmd include pytest.ini include tox.ini -include distutils_precedence.pth exclude pyproject.toml # Temporary workaround for #1644. diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py index 323ae2037c..ffa4caead5 100644 --- a/_distutils_importer/__init__.py +++ b/_distutils_importer/__init__.py @@ -1,4 +1,20 @@ +""" +Ensure that the local copy of distutils is preferred over stdlib. + +See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 +for more motivation. +""" + import sys +import os + + +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') + return which == 'local' class DistutilsMetaFinder: @@ -34,6 +50,3 @@ def remove_shim(): sys.path.remove(DISTUTILS_FINDER) except ValueError: pass - - -add_shim() diff --git a/_distutils_importer/install.py b/_distutils_importer/install.py new file mode 100644 index 0000000000..73f13b2993 --- /dev/null +++ b/_distutils_importer/install.py @@ -0,0 +1,5 @@ +from . import enabled, add_shim + + +if enabled(): + add_shim() diff --git a/setuptools/distutils_patch.py b/_distutils_importer/override.py similarity index 79% rename from setuptools/distutils_patch.py rename to _distutils_importer/override.py index 33f1e7f961..917384852d 100644 --- a/setuptools/distutils_patch.py +++ b/_distutils_importer/override.py @@ -1,16 +1,10 @@ -""" -Ensure that the local copy of distutils is preferred over stdlib. - -See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 -for more motivation. -""" - import sys import re -import os import importlib import warnings +from . import enabled + is_pypy = '__pypy__' in sys.builtin_module_names @@ -37,14 +31,6 @@ def clear_distutils(): del sys.modules[name] -def enabled(): - """ - Allow selection of distutils by environment variable. - """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') - return which == 'local' - - def ensure_local_distutils(): clear_distutils() distutils = importlib.import_module('setuptools._distutils') diff --git a/conftest.py b/conftest.py index 6013e1870f..868bf5bed2 100644 --- a/conftest.py +++ b/conftest.py @@ -15,7 +15,7 @@ def pytest_addoption(parser): 'tests/manual_test.py', 'setuptools/tests/mod_with_constant.py', 'setuptools/_distutils', - 'setuptools/distutils_patch.py', + '_distutils_importer', ] diff --git a/distutils_precedence.pth b/distutils_precedence.pth deleted file mode 100644 index 2d9b996abc..0000000000 --- a/distutils_precedence.pth +++ /dev/null @@ -1 +0,0 @@ -import os; (os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') == 'local' and __import__('_distutils_importer')) diff --git a/setup.py b/setup.py index cba37d3e59..a6e1abc455 100755 --- a/setup.py +++ b/setup.py @@ -94,12 +94,8 @@ class install_with_pth(install): def initialize_options(self): install.initialize_options(self) - - name = 'distutils_precedence' - with open(os.path.join(here, name + '.pth'), 'rt') as f: - contents = f.read() - - self.extra_path = (name, contents) + self.extra_path = ( + 'distutils-precedence', 'import _distutils_importer.install') def finalize_options(self): install.finalize_options(self) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index d9740403d9..80b287b4b4 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -5,8 +5,7 @@ import os import re -# Disabled for now due to: #2228, #2230 -import setuptools.distutils_patch # noqa: F401 +import _distutils_importer.override # noqa: F401 import distutils.core from distutils.errors import DistutilsOptionError From 2986be4e55d6c8113344d5e184d40c6c3945a2bb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 19 Jul 2020 21:49:24 -0400 Subject: [PATCH 08/23] Fix AttributeError when `importlib.util` was not otherwise imported. --- _distutils_importer/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py index ffa4caead5..06674eb889 100644 --- a/_distutils_importer/__init__.py +++ b/_distutils_importer/__init__.py @@ -25,7 +25,7 @@ def find_spec(self, fullname, path, target=None): return self.get_distutils_spec() def get_distutils_spec(self): - import importlib + import importlib.util class DistutilsLoader(importlib.util.abc.Loader): From 1e53a2c14e7e0f788c9df2a542ac10f6b2f511d7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 19 Jul 2020 21:53:39 -0400 Subject: [PATCH 09/23] Move docstring closer to relevant context --- _distutils_importer/__init__.py | 7 ------- _distutils_importer/override.py | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/_distutils_importer/__init__.py b/_distutils_importer/__init__.py index 06674eb889..3ad701002d 100644 --- a/_distutils_importer/__init__.py +++ b/_distutils_importer/__init__.py @@ -1,10 +1,3 @@ -""" -Ensure that the local copy of distutils is preferred over stdlib. - -See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 -for more motivation. -""" - import sys import os diff --git a/_distutils_importer/override.py b/_distutils_importer/override.py index 917384852d..523139bb6e 100644 --- a/_distutils_importer/override.py +++ b/_distutils_importer/override.py @@ -1,3 +1,10 @@ +""" +Ensure that the local copy of distutils is preferred over stdlib. + +See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 +for more motivation. +""" + import sys import re import importlib From dcc71f773576c19a3658735879893515b056ece5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:35:02 -0400 Subject: [PATCH 10/23] Rename _distutils_importer to _distutils_hack, as it supplies more than just an importer. --- {_distutils_importer => _distutils_hack}/__init__.py | 0 {_distutils_importer => _distutils_hack}/install.py | 0 {_distutils_importer => _distutils_hack}/override.py | 0 conftest.py | 2 +- setup.py | 2 +- setuptools/__init__.py | 2 +- setuptools/sandbox.py | 8 ++++---- 7 files changed, 7 insertions(+), 7 deletions(-) rename {_distutils_importer => _distutils_hack}/__init__.py (100%) rename {_distutils_importer => _distutils_hack}/install.py (100%) rename {_distutils_importer => _distutils_hack}/override.py (100%) diff --git a/_distutils_importer/__init__.py b/_distutils_hack/__init__.py similarity index 100% rename from _distutils_importer/__init__.py rename to _distutils_hack/__init__.py diff --git a/_distutils_importer/install.py b/_distutils_hack/install.py similarity index 100% rename from _distutils_importer/install.py rename to _distutils_hack/install.py diff --git a/_distutils_importer/override.py b/_distutils_hack/override.py similarity index 100% rename from _distutils_importer/override.py rename to _distutils_hack/override.py diff --git a/conftest.py b/conftest.py index 868bf5bed2..25537f56e5 100644 --- a/conftest.py +++ b/conftest.py @@ -15,7 +15,7 @@ def pytest_addoption(parser): 'tests/manual_test.py', 'setuptools/tests/mod_with_constant.py', 'setuptools/_distutils', - '_distutils_importer', + '_distutils_hack', ] diff --git a/setup.py b/setup.py index a6e1abc455..2d8bdf85d8 100755 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ class install_with_pth(install): def initialize_options(self): install.initialize_options(self) self.extra_path = ( - 'distutils-precedence', 'import _distutils_importer.install') + 'distutils-precedence', 'import _distutils_hack.install') def finalize_options(self): install.finalize_options(self) diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 80b287b4b4..99094230d3 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -5,7 +5,7 @@ import os import re -import _distutils_importer.override # noqa: F401 +import _distutils_hack.override # noqa: F401 import distutils.core from distutils.errors import DistutilsOptionError diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 342a713f5c..24a360808a 100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -200,7 +200,7 @@ def setup_context(setup_dir): 'distutils', 'pkg_resources', 'Cython', - '_distutils_importer', + '_distutils_hack', } @@ -232,9 +232,9 @@ def hide_setuptools(): necessary to avoid issues such as #315 where setuptools upgrading itself would fail to find a function declared in the metadata. """ - _distutils_importer = sys.modules.get('_distutils_importer', None) - if _distutils_importer is not None: - _distutils_importer.remove_shim() + _distutils_hack = sys.modules.get('_distutils_hack', None) + if _distutils_hack is not None: + _distutils_hack.remove_shim() modules = filter(_needs_hiding, sys.modules) _clear_modules(modules) From 384a51657c94271d29c437415080f25f7df4103b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:38:13 -0400 Subject: [PATCH 11/23] Extract pth name and contents into class variables. --- setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 2d8bdf85d8..edb2bbcff6 100755 --- a/setup.py +++ b/setup.py @@ -92,18 +92,21 @@ class install_with_pth(install): `distutils` than the standard library. """ + _pth_name = 'distutils-precedence' + _pth_contents = 'import _distutils_hack.install' + def initialize_options(self): install.initialize_options(self) - self.extra_path = ( - 'distutils-precedence', 'import _distutils_hack.install') + self.extra_path = self._pth_name, self._pth_contents def finalize_options(self): install.finalize_options(self) + # undo secondary effect of `extra_path` adding to `install_lib` install_suffix = os.path.relpath(self.install_lib, self.install_libbase) - if install_suffix == self.extra_path[1]: + if install_suffix == self._pth_contents: self.install_lib = self.install_libbase From f3b177e9c2b77104ddebaec7b581e2aee73a1184 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:42:15 -0400 Subject: [PATCH 12/23] Update docstring to use imperative voice, provide a bit more context, and advise against copying of the behavior. --- setup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index edb2bbcff6..45ddb149a5 100755 --- a/setup.py +++ b/setup.py @@ -86,10 +86,13 @@ class install_with_pth(install): """ Custom install command to install a .pth file for distutils patching. - This is necessary because there's no standard way to install a `.pth` file - alongside your package (and there probably shouldn't be one), but we need - to do this in order to give precedence higher precedence to our version of - `distutils` than the standard library. + This hack is necessary because there's no standard way to install behavior + on startup (and it's debatable if there should be one). This hack (ab)uses + the `extra_path` behavior in Setuptools to install a `.pth` file with + implicit behavior on startup to give higher precedence to the local version + of `distutils` over the version from the standard library. + + Please do not replicate this behavior. """ _pth_name = 'distutils-precedence' From 268ef5f553f29977f708c256ee398c9e29cb4da7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:43:08 -0400 Subject: [PATCH 13/23] Remove hanging indent --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 45ddb149a5..6290a74698 100755 --- a/setup.py +++ b/setup.py @@ -106,8 +106,10 @@ def finalize_options(self): install.finalize_options(self) # undo secondary effect of `extra_path` adding to `install_lib` - install_suffix = os.path.relpath(self.install_lib, - self.install_libbase) + install_suffix = os.path.relpath( + self.install_lib, + self.install_libbase, + ) if install_suffix == self._pth_contents: self.install_lib = self.install_libbase From 5642e413fb6c75434f109be943bdb09ea9e7ade2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:44:51 -0400 Subject: [PATCH 14/23] Extract function for restoring install lib to encapsulate behavior. --- setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 6290a74698..6f6601fedf 100755 --- a/setup.py +++ b/setup.py @@ -104,14 +104,15 @@ def initialize_options(self): def finalize_options(self): install.finalize_options(self) + self._restore_install_lib() - # undo secondary effect of `extra_path` adding to `install_lib` - install_suffix = os.path.relpath( - self.install_lib, - self.install_libbase, - ) + def _restore_install_lib(self): + """ + Undo secondary effect of `extra_path` adding to `install_lib` + """ + suffix = os.path.relpath(self.install_lib, self.install_libbase) - if install_suffix == self._pth_contents: + if suffix == self._pth_contents: self.install_lib = self.install_libbase From 83a85a71cd779b1b1b3a44e21cc198264650da46 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:51:40 -0400 Subject: [PATCH 15/23] Restore early opt-in/opt-out for pth behavior. --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6f6601fedf..2e44225e41 100755 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ import os import sys +import textwrap import setuptools from setuptools.command.install import install @@ -96,7 +97,11 @@ class install_with_pth(install): """ _pth_name = 'distutils-precedence' - _pth_contents = 'import _distutils_hack.install' + _pth_contents = textwrap.dedent(""" + import os + enabled = os.environ.get('SETUPTOOLS_USE_DISTUTILS') == 'local' + enabled and __import__('_distutils_hack.install') + """).lstrip().replace('\n', '; ') def initialize_options(self): install.initialize_options(self) From 1251a231ad75fa649da700645690eb3c0a348f08 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:53:31 -0400 Subject: [PATCH 16/23] Replace install behavior on import with direct invocation (now that 'enabled' logic is duplicated in pth file). --- _distutils_hack/install.py | 5 ----- setup.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 _distutils_hack/install.py diff --git a/_distutils_hack/install.py b/_distutils_hack/install.py deleted file mode 100644 index 73f13b2993..0000000000 --- a/_distutils_hack/install.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import enabled, add_shim - - -if enabled(): - add_shim() diff --git a/setup.py b/setup.py index 2e44225e41..37953051b0 100755 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ class install_with_pth(install): _pth_contents = textwrap.dedent(""" import os enabled = os.environ.get('SETUPTOOLS_USE_DISTUTILS') == 'local' - enabled and __import__('_distutils_hack.install') + enabled and __import__('_distutils_hack').add_shim() """).lstrip().replace('\n', '; ') def initialize_options(self): From 6c6d69e0213c4012caa36f0087f1fe54bac62c89 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 10:59:11 -0400 Subject: [PATCH 17/23] Move all but a small shim in override into _distutils_hack --- _distutils_hack/__init__.py | 51 ++++++++++++++++++++++++++++++++++ _distutils_hack/override.py | 55 +------------------------------------ 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index 3ad701002d..a8638344f0 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -1,5 +1,11 @@ import sys import os +import re +import importlib +import warnings + + +is_pypy = '__pypy__' in sys.builtin_module_names def enabled(): @@ -10,6 +16,51 @@ def enabled(): return which == 'local' +def warn_distutils_present(): + if 'distutils' not in sys.modules: + return + if is_pypy and sys.version_info < (3, 7): + # PyPy for 3.6 unconditionally imports distutils, so bypass the warning + # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 + return + warnings.warn( + "Distutils was imported before Setuptools. This usage is discouraged " + "and may exhibit undesirable behaviors or errors. Please use " + "Setuptools' objects directly or at least import Setuptools first.") + + +def clear_distutils(): + if 'distutils' not in sys.modules: + return + warnings.warn("Setuptools is replacing distutils.") + mods = [name for name in sys.modules if re.match(r'distutils\b', name)] + for name in mods: + del sys.modules[name] + + +def ensure_local_distutils(): + clear_distutils() + distutils = importlib.import_module('setuptools._distutils') + distutils.__name__ = 'distutils' + sys.modules['distutils'] = distutils + + # sanity check that submodules load as expected + core = importlib.import_module('distutils.core') + assert '_distutils' in core.__file__, core.__file__ + + +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + warn_distutils_present() + if enabled(): + ensure_local_distutils() + + class DistutilsMetaFinder: def find_spec(self, fullname, path, target=None): if path is not None or fullname != "distutils": diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py index 523139bb6e..2cc433a4a5 100644 --- a/_distutils_hack/override.py +++ b/_distutils_hack/override.py @@ -1,54 +1 @@ -""" -Ensure that the local copy of distutils is preferred over stdlib. - -See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 -for more motivation. -""" - -import sys -import re -import importlib -import warnings - -from . import enabled - - -is_pypy = '__pypy__' in sys.builtin_module_names - - -def warn_distutils_present(): - if 'distutils' not in sys.modules: - return - if is_pypy and sys.version_info < (3, 7): - # PyPy for 3.6 unconditionally imports distutils, so bypass the warning - # https://foss.heptapod.net/pypy/pypy/-/blob/be829135bc0d758997b3566062999ee8b23872b4/lib-python/3/site.py#L250 - return - warnings.warn( - "Distutils was imported before Setuptools. This usage is discouraged " - "and may exhibit undesirable behaviors or errors. Please use " - "Setuptools' objects directly or at least import Setuptools first.") - - -def clear_distutils(): - if 'distutils' not in sys.modules: - return - warnings.warn("Setuptools is replacing distutils.") - mods = [name for name in sys.modules if re.match(r'distutils\b', name)] - for name in mods: - del sys.modules[name] - - -def ensure_local_distutils(): - clear_distutils() - distutils = importlib.import_module('setuptools._distutils') - distutils.__name__ = 'distutils' - sys.modules['distutils'] = distutils - - # sanity check that submodules load as expected - core = importlib.import_module('distutils.core') - assert '_distutils' in core.__file__, core.__file__ - - -warn_distutils_present() -if enabled(): - ensure_local_distutils() +__import__('_distutils_hack').do_override() From 8db806d30d7591828528ac937e8f3b334e957ed3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 Jul 2020 20:19:45 -0400 Subject: [PATCH 18/23] remove shim should by symmetric to add_shim --- _distutils_hack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index a8638344f0..a10af2cc94 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -91,6 +91,6 @@ def add_shim(): def remove_shim(): try: - sys.path.remove(DISTUTILS_FINDER) + sys.meta_path.remove(DISTUTILS_FINDER) except ValueError: pass From ebfe95fcbb12c22f58b045c6f10bd899a21a53d8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 5 Aug 2020 21:49:08 -0400 Subject: [PATCH 19/23] Update changelog. --- changelog.d/2259.change.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2259.change.rst diff --git a/changelog.d/2259.change.rst b/changelog.d/2259.change.rst new file mode 100644 index 0000000000..43701ec23b --- /dev/null +++ b/changelog.d/2259.change.rst @@ -0,0 +1 @@ +Setuptools now provides a .pth file (except for editable installs of setuptools) to the target environment to ensure that when enabled, the setuptools-provided distutils is preferred before setuptools has been imported (and even if setuptools is never imported). Honors the SETUPTOOLS_USE_DISTUTILS environment variable. From 48a17a56ecfc77fb60780e3cfa75390f6bb10b15 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 8 Aug 2020 12:22:05 -0400 Subject: [PATCH 20/23] As discovered in bpo-41509, relpath can strip spaces, so match that expectation. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37953051b0..5d98c029c5 100755 --- a/setup.py +++ b/setup.py @@ -117,7 +117,7 @@ def _restore_install_lib(self): """ suffix = os.path.relpath(self.install_lib, self.install_libbase) - if suffix == self._pth_contents: + if suffix.strip() == self._pth_contents.strip(): self.install_lib = self.install_libbase From 6b70fb201d6a81448de6ca6f71d7091b9a26096c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 8 Aug 2020 13:38:56 -0400 Subject: [PATCH 21/23] Restore location of 'enabled' --- _distutils_hack/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/_distutils_hack/__init__.py b/_distutils_hack/__init__.py index a10af2cc94..71fa7ce1b9 100644 --- a/_distutils_hack/__init__.py +++ b/_distutils_hack/__init__.py @@ -8,14 +8,6 @@ is_pypy = '__pypy__' in sys.builtin_module_names -def enabled(): - """ - Allow selection of distutils by environment variable. - """ - which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') - return which == 'local' - - def warn_distutils_present(): if 'distutils' not in sys.modules: return @@ -38,6 +30,14 @@ def clear_distutils(): del sys.modules[name] +def enabled(): + """ + Allow selection of distutils by environment variable. + """ + which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'stdlib') + return which == 'local' + + def ensure_local_distutils(): clear_distutils() distutils = importlib.import_module('setuptools._distutils') From 521987da809e63ee51a63aa45dbe372d40deb8f7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 8 Aug 2020 19:50:24 -0400 Subject: [PATCH 22/23] Remove expected fail. --- setuptools/tests/test_distutils_adoption.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index 7f28a217d6..bb8e34d558 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -57,7 +57,10 @@ def test_distutils_local_with_setuptools(venv): assert venv.name in loc.split(os.sep) -@pytest.mark.xfail(reason="#2259") def test_distutils_local(venv): + """ + Even without importing, the setuptools-local copy of distutils is + preferred. + """ env = dict(SETUPTOOLS_USE_DISTUTILS='local') assert venv.name in find_distutils(venv, env=env).split(os.sep) From 7cf009a7e39270e1e1d13d913e0c352fb00534c0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 8 Aug 2020 20:58:57 -0400 Subject: [PATCH 23/23] Expect test to fail on PyPy due to implicit import during startup. --- setuptools/tests/test_distutils_adoption.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index bb8e34d558..daccc473c2 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -9,6 +9,9 @@ import path +IS_PYPY = '__pypy__' in sys.builtin_module_names + + class VirtualEnv(jaraco.envs.VirtualEnv): name = '.env' @@ -57,6 +60,7 @@ def test_distutils_local_with_setuptools(venv): assert venv.name in loc.split(os.sep) +@pytest.mark.xfail('IS_PYPY', reason='pypy imports distutils on startup') def test_distutils_local(venv): """ Even without importing, the setuptools-local copy of distutils is