diff --git a/_distutils_hack/override.py b/_distutils_hack/override.py
deleted file mode 100644
index 2cc433a4a5..0000000000
--- a/_distutils_hack/override.py
+++ /dev/null
@@ -1 +0,0 @@
-__import__('_distutils_hack').do_override()
diff --git a/setup.py b/setup.py
index d15189db0f..a0a2dbe57d 100755
--- a/setup.py
+++ b/setup.py
@@ -51,11 +51,15 @@ class install_with_pth(install):
     """
 
     _pth_name = 'distutils-precedence'
+    # NB: this abuses the fact that site.py directly execs this with a `sitedir` local that contains the current sitedir it's working on; __file__ is no good here
     _pth_contents = textwrap.dedent("""
         import os
+        from importlib.util import spec_from_file_location, module_from_spec
         var = 'SETUPTOOLS_USE_DISTUTILS'
         enabled = os.environ.get(var, 'local') == 'local'
-        enabled and __import__('_distutils_hack').ensure_shim()
+        mod = module_from_spec(spec_from_file_location('_shimmod', os.path.join(sitedir, 'setuptools', '_distutils_shim.py'))) if enabled else None
+        mod and mod.__spec__.loader.exec_module(mod)
+        mod and mod.ensure_shim()
         """).lstrip().replace('\n', '; ')
 
     def initialize_options(self):
diff --git a/setuptools/__init__.py b/setuptools/__init__.py
index 9d6f0bc0dd..c912993237 100644
--- a/setuptools/__init__.py
+++ b/setuptools/__init__.py
@@ -5,7 +5,8 @@
 import os
 import re
 
-import _distutils_hack.override  # noqa: F401
+from ._distutils_shim import do_override  # noqa: F401
+do_override()
 
 import distutils.core
 from distutils.errors import DistutilsOptionError
diff --git a/_distutils_hack/__init__.py b/setuptools/_distutils_shim.py
similarity index 78%
rename from _distutils_hack/__init__.py
rename to setuptools/_distutils_shim.py
index 85a51370b6..e22b9439c9 100644
--- a/_distutils_hack/__init__.py
+++ b/setuptools/_distutils_shim.py
@@ -74,6 +74,9 @@ def do_override():
 
 
 class DistutilsMetaFinder:
+    # for forensic purposes, since we may not be able to get at the module object easily
+    _fromfile = __file__
+
     def find_spec(self, fullname, path, target=None):
         if path is not None:
             return
@@ -86,8 +89,16 @@ def spec_for_distutils(self):
         import importlib.abc
         import importlib.util
 
-        class DistutilsLoader(importlib.abc.Loader):
+        st = importlib.import_module('setuptools')
+        if os.path.dirname(st.__file__) != os.path.dirname(__file__):
+            # no-op; the setuptools we imported was from a foreign site-packages dir, remove self from meta_path
+            remove_shim()
+            return None
+        else:
+            # we're a sibling of the setuptools that actually got imported; kill any other shims
+            remove_foreign_shims()
 
+        class DistutilsLoader(importlib.abc.Loader):
             def create_module(self, spec):
                 return importlib.import_module('setuptools._distutils')
 
@@ -130,7 +141,8 @@ def frame_file_is_setup(frame):
 
 
 def ensure_shim():
-    DISTUTILS_FINDER in sys.meta_path or add_shim()
+    if not (any(s for s in sys.meta_path if type(s).__name__ == 'DistutilsMetaFinder' and getattr(s, '_fromfile', None) == DistutilsMetaFinder._fromfile)):
+        add_shim()
 
 
 @contextlib.contextmanager
@@ -151,3 +163,15 @@ def remove_shim():
         sys.meta_path.remove(DISTUTILS_FINDER)
     except ValueError:
         pass
+
+
+def remove_foreign_shims():
+    """
+    Remove setuptools metapath shims that might've been installed by other installations on sys.path
+    """
+    try:
+        foreign_shims = [l for l in sys.meta_path if type(l).__name__ == 'DistutilsMetaFinder' and l is not DISTUTILS_FINDER]
+        for shim in foreign_shims:
+            sys.meta_path.remove(shim)
+    except Exception:
+        pass