diff --git a/setuptools/distutils_patch.py b/_distutils_hack/__init__.py similarity index 57% rename from setuptools/distutils_patch.py rename to _distutils_hack/__init__.py index 33f1e7f961..71fa7ce1b9 100644 --- a/setuptools/distutils_patch.py +++ b/_distutils_hack/__init__.py @@ -1,13 +1,6 @@ -""" -Ensure that the local copy of distutils is preferred over stdlib. - -See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 -for more motivation. -""" - import sys -import re import os +import re import importlib import warnings @@ -56,6 +49,48 @@ def ensure_local_distutils(): assert '_distutils' in core.__file__, core.__file__ -warn_distutils_present() -if enabled(): - ensure_local_distutils() +def do_override(): + """ + Ensure that the local copy of distutils is preferred over stdlib. + + See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401 + for more motivation. + """ + warn_distutils_present() + if enabled(): + ensure_local_distutils() + + +class DistutilsMetaFinder: + def find_spec(self, fullname, path, target=None): + if path is not None or fullname != "distutils": + return None + + return self.get_distutils_spec() + + def get_distutils_spec(self): + import importlib.util + + class DistutilsLoader(importlib.util.abc.Loader): + + def create_module(self, spec): + return importlib.import_module('._distutils', 'setuptools') + + def exec_module(self, module): + pass + + return importlib.util.spec_from_loader('distutils', DistutilsLoader()) + + +DISTUTILS_FINDER = DistutilsMetaFinder() + + +def add_shim(): + sys.meta_path.insert(0, DISTUTILS_FINDER) + + +def remove_shim(): + try: + sys.meta_path.remove(DISTUTILS_FINDER) + except ValueError: + pass diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py new file mode 100644 index 0000000000..2cc433a4a5 --- /dev/null +++ b/_distutils_hack/override.py @@ -0,0 +1 @@ +__import__('_distutils_hack').do_override() diff --git a/changelog.d/2259.change.rst b/changelog.d/2259.change.rst new file mode 100644 index 0000000000..43701ec23b --- /dev/null +++ b/changelog.d/2259.change.rst @@ -0,0 +1 @@ +Setuptools now provides a .pth file (except for editable installs of setuptools) to the target environment to ensure that when enabled, the setuptools-provided distutils is preferred before setuptools has been imported (and even if setuptools is never imported). Honors the SETUPTOOLS_USE_DISTUTILS environment variable. diff --git a/conftest.py b/conftest.py index 6013e1870f..25537f56e5 100644 --- a/conftest.py +++ b/conftest.py @@ -15,7 +15,7 @@ def pytest_addoption(parser): 'tests/manual_test.py', 'setuptools/tests/mod_with_constant.py', 'setuptools/_distutils', - 'setuptools/distutils_patch.py', + '_distutils_hack', ] diff --git a/setup.py b/setup.py index 1fe18bd13c..5d98c029c5 100755 --- a/setup.py +++ b/setup.py @@ -5,8 +5,10 @@ import os import sys +import textwrap import setuptools +from setuptools.command.install import install here = os.path.dirname(__file__) @@ -81,8 +83,47 @@ def pypi_link(pkg_filename): return '/'.join(parts) +class install_with_pth(install): + """ + Custom install command to install a .pth file for distutils patching. + + This hack is necessary because there's no standard way to install behavior + on startup (and it's debatable if there should be one). This hack (ab)uses + the `extra_path` behavior in Setuptools to install a `.pth` file with + implicit behavior on startup to give higher precedence to the local version + of `distutils` over the version from the standard library. + + Please do not replicate this behavior. + """ + + _pth_name = 'distutils-precedence' + _pth_contents = textwrap.dedent(""" + import os + enabled = os.environ.get('SETUPTOOLS_USE_DISTUTILS') == 'local' + enabled and __import__('_distutils_hack').add_shim() + """).lstrip().replace('\n', '; ') + + def initialize_options(self): + install.initialize_options(self) + self.extra_path = self._pth_name, self._pth_contents + + def finalize_options(self): + install.finalize_options(self) + self._restore_install_lib() + + def _restore_install_lib(self): + """ + Undo secondary effect of `extra_path` adding to `install_lib` + """ + suffix = os.path.relpath(self.install_lib, self.install_libbase) + + if suffix.strip() == self._pth_contents.strip(): + self.install_lib = self.install_libbase + + setup_params = dict( src_root=None, + cmdclass={'install': install_with_pth}, package_data=package_data, entry_points={ "distutils.commands": [ diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 8388251115..99094230d3 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,17 +1,15 @@ """Extensions to the 'distutils' for large or complex distributions""" -import os +from fnmatch import fnmatchcase import functools +import os +import re -# Disabled for now due to: #2228, #2230 -import setuptools.distutils_patch # noqa: F401 +import _distutils_hack.override # noqa: F401 import distutils.core -import distutils.filelist -import re from distutils.errors import DistutilsOptionError from distutils.util import convert_path -from fnmatch import fnmatchcase from ._deprecation_warning import SetuptoolsDeprecationWarning diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 93ae8eb4d9..24a360808a 100644 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -185,8 +185,8 @@ def setup_context(setup_dir): temp_dir = os.path.join(setup_dir, 'temp') with save_pkg_resources_state(): with save_modules(): - hide_setuptools() with save_path(): + hide_setuptools() with save_argv(): with override_temp(temp_dir): with pushd(setup_dir): @@ -195,6 +195,15 @@ def setup_context(setup_dir): yield +_MODULES_TO_HIDE = { + 'setuptools', + 'distutils', + 'pkg_resources', + 'Cython', + '_distutils_hack', +} + + def _needs_hiding(mod_name): """ >>> _needs_hiding('setuptools') @@ -212,8 +221,8 @@ def _needs_hiding(mod_name): >>> _needs_hiding('Cython') True """ - pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)') - return bool(pattern.match(mod_name)) + base_module = mod_name.split('.', 1)[0] + return base_module in _MODULES_TO_HIDE def hide_setuptools(): @@ -223,6 +232,10 @@ def hide_setuptools(): necessary to avoid issues such as #315 where setuptools upgrading itself would fail to find a function declared in the metadata. """ + _distutils_hack = sys.modules.get('_distutils_hack', None) + if _distutils_hack is not None: + _distutils_hack.remove_shim() + modules = filter(_needs_hiding, sys.modules) _clear_modules(modules) diff --git a/setuptools/tests/test_distutils_adoption.py b/setuptools/tests/test_distutils_adoption.py index 7f28a217d6..daccc473c2 100644 --- a/setuptools/tests/test_distutils_adoption.py +++ b/setuptools/tests/test_distutils_adoption.py @@ -9,6 +9,9 @@ import path +IS_PYPY = '__pypy__' in sys.builtin_module_names + + class VirtualEnv(jaraco.envs.VirtualEnv): name = '.env' @@ -57,7 +60,11 @@ def test_distutils_local_with_setuptools(venv): assert venv.name in loc.split(os.sep) -@pytest.mark.xfail(reason="#2259") +@pytest.mark.xfail('IS_PYPY', reason='pypy imports distutils on startup') def test_distutils_local(venv): + """ + Even without importing, the setuptools-local copy of distutils is + preferred. + """ env = dict(SETUPTOOLS_USE_DISTUTILS='local') assert venv.name in find_distutils(venv, env=env).split(os.sep)