distutils: pass -rpath to linker on macOS >=10.5

Fix -R option of build_ext for macOS (darwin)

Resolves this old bug against distutils that expired due to PIP 632:
https://bugs.python.org/issue36353

Applies patch originally submitted to CPython:
python/cpython#12418

Contributor: Toon Verstraelen <Toon.Verstraelen@UGent.be>
Signed-off-by: Alexei Colin <acolin@isi.edu>

distutils/spawn.py

@@ -15,11 +15,6 @@
 from distutils import log
 
 
-if sys.platform == 'darwin':
-    _cfg_target = None
-    _cfg_target_split = None
-
-
 def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None):
     """Run another program, specified as a command list 'cmd', in a new process.
 
@@ -52,28 +47,10 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None):
     env = env if env is not None else dict(os.environ)
 
     if sys.platform == 'darwin':
-        global _cfg_target, _cfg_target_split
-        if _cfg_target is None:
-            from distutils import sysconfig
-            _cfg_target = sysconfig.get_config_var(
-                                  'MACOSX_DEPLOYMENT_TARGET') or ''
-            if _cfg_target:
-                _cfg_target_split = [int(x) for x in _cfg_target.split('.')]
-        if _cfg_target:
-            # Ensure that the deployment target of the build process is not
-            # less than 10.3 if the interpreter was built for 10.3 or later.
-            # This ensures extension modules are built with correct
-            # compatibility values, specifically LDSHARED which can use
-            # '-undefined dynamic_lookup' which only works on >= 10.3.
-            cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
-            cur_target_split = [int(x) for x in cur_target.split('.')]
-            if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]:
-                my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
-                          'now "%s" but "%s" during configure;'
-                          'must use 10.3 or later'
-                                % (cur_target, _cfg_target))
-                raise DistutilsPlatformError(my_msg)
-            env.update(MACOSX_DEPLOYMENT_TARGET=cur_target)
+        from distutils.util import MACOSX_VERSION_VAR, get_macosx_ver
+        macosx_ver = get_macosx_ver()
+        if macosx_ver:
+            env[MACOSX_VERSION_VAR] = macosx_ver
 
     try:
         proc = subprocess.Popen(cmd, env=env)

distutils/tests/test_unixccompiler.py

@@ -1,12 +1,15 @@
 """Tests for distutils.unixccompiler."""
+import os
 import sys
 import unittest
 from test.support import run_unittest
 
 from .py38compat import EnvironmentVarGuard
 
 from distutils import sysconfig
+from distutils.errors import DistutilsPlatformError
 from distutils.unixccompiler import UnixCCompiler
+from distutils.util import _clear_cached_macosx_ver
 
 class UnixCCompilerTestCase(unittest.TestCase):
 
@@ -26,18 +29,90 @@ def tearDown(self):
 
     @unittest.skipIf(sys.platform == 'win32', "can't test on Windows")
     def test_runtime_libdir_option(self):
-        # Issue#5900
+        # Issue #5900; GitHub Issue #37
         #
         # Ensure RUNPATH is added to extension modules with RPATH if
         # GNU ld is used
 
         # darwin
         sys.platform = 'darwin'
-        self.assertEqual(self.cc.rpath_foo(), '-L/foo')
+        darwin_ver_var = 'MACOSX_DEPLOYMENT_TARGET'
+        darwin_rpath_flag = '-Wl,-rpath,/foo'
+        darwin_lib_flag = '-L/foo'
+
+        # (macOS version from syscfg, macOS version from env var) -> flag
+        # Version value of None generates two tests: as None and as empty string
+        # Expected flag value of None means an mismatch exception is expected
+        darwin_test_cases = [
+            ((None    , None    ), darwin_lib_flag),
+            ((None    , '11'    ), darwin_rpath_flag),
+            (('10'    , None    ), darwin_lib_flag),
+            (('10.3'  , None    ), darwin_lib_flag),
+            (('10.3.1', None    ), darwin_lib_flag),
+            (('10.5'  , None    ), darwin_rpath_flag),
+            (('10.5.1', None    ), darwin_rpath_flag),
+            (('10.3'  , '10.3'  ), darwin_lib_flag),
+            (('10.3'  , '10.5'  ), darwin_rpath_flag),
+            (('10.5'  , '10.3'  ), darwin_lib_flag),
+            (('10.5'  , '11'    ), darwin_rpath_flag),
+            (('10.4'  , '10'    ), None),
+        ]
+
+        def make_darwin_gcv(syscfg_macosx_ver):
+            def gcv(var):
+                if var == darwin_ver_var:
+                    return syscfg_macosx_ver
+                return "xxx"
+            return gcv
+
+        def do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag):
+            env = os.environ
+            msg = "macOS version = (sysconfig=%r, env=%r)" % \
+                    (syscfg_macosx_ver, env_macosx_ver)
+
+            # Save
+            old_gcv = sysconfig.get_config_var
+            old_env_macosx_ver = env.get(darwin_ver_var)
+
+            # Setup environment
+            _clear_cached_macosx_ver()
+            sysconfig.get_config_var = make_darwin_gcv(syscfg_macosx_ver)
+            if env_macosx_ver is not None:
+                env[darwin_ver_var] = env_macosx_ver
+            elif darwin_ver_var in env:
+                env.pop(darwin_ver_var)
+
+            # Run the test
+            if expected_flag is not None:
+                self.assertEqual(self.cc.rpath_foo(), expected_flag, msg=msg)
+            else:
+                with self.assertRaisesRegex(DistutilsPlatformError,
+                        darwin_ver_var + r' mismatch', msg=msg):
+                    self.cc.rpath_foo()
+
+            # Restore
+            if old_env_macosx_ver is not None:
+                env[darwin_ver_var] = old_env_macosx_ver
+            elif darwin_ver_var in env:
+                env.pop(darwin_ver_var)
+            sysconfig.get_config_var = old_gcv
+            _clear_cached_macosx_ver()
+
+        for macosx_vers, expected_flag in darwin_test_cases:
+            syscfg_macosx_ver, env_macosx_ver = macosx_vers
+            do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag)
+            # Bonus test cases with None interpreted as empty string
+            if syscfg_macosx_ver is None:
+                do_darwin_test("", env_macosx_ver, expected_flag)
+            if env_macosx_ver is None:
+                do_darwin_test(syscfg_macosx_ver, "", expected_flag)
+            if syscfg_macosx_ver is None and env_macosx_ver is None:
+                do_darwin_test("", "", expected_flag)
+
+        old_gcv = sysconfig.get_config_var
 
         # hp-ux
         sys.platform = 'hp-ux'
-        old_gcv = sysconfig.get_config_var
         def gcv(v):
             return 'xxx'
         sysconfig.get_config_var = gcv

distutils/unixccompiler.py

@@ -233,8 +233,12 @@ def runtime_library_dir_option(self, dir):
         # we use this hack.
         compiler = os.path.basename(sysconfig.get_config_var("CC"))
         if sys.platform[:6] == "darwin":
-            # MacOSX's linker doesn't understand the -R flag at all
-            return "-L" + dir
+            from distutils.util import get_macosx_ver, as_version
+            macosx_ver = get_macosx_ver()
+            if macosx_ver and as_version(macosx_ver) >= [10, 5]:
+                return "-Wl,-rpath," + dir
+            else: # no support for -rpath on earlier macOS versions
+                return "-L" + dir
         elif sys.platform[:7] == "freebsd":
             return "-Wl,-rpath=" + dir
         elif sys.platform[:5] == "hp-ux":

distutils/util.py

@@ -108,6 +108,60 @@ def get_platform():
     else:
         return get_host_platform()
 
+
+if sys.platform == 'darwin':
+    _syscfg_macosx_ver = None # cache the version pulled from sysconfig
+MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET'
+
+def _clear_cached_macosx_ver():
+    """For testing only. Do not call."""
+    global _syscfg_macosx_ver
+    _syscfg_macosx_ver = None
+
+def get_macosx_ver_from_syscfg():
+    """Get the version of macOS latched in the Python interpreter configuration.
+    Returns the version as a string or None if can't obtain one. Cached."""
+    global _syscfg_macosx_ver
+    if _syscfg_macosx_ver is None:
+        from distutils import sysconfig
+        ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or ''
+        if ver:
+            _syscfg_macosx_ver = ver
+    return _syscfg_macosx_ver
+
+def get_macosx_ver():
+    """Return the version of macOS for which we are building.
+
+    The target version defaults to the version in sysconfig latched at time
+    the Python interpreter was built, unless overriden by an environment
+    variable. If neither source has a value, then None is returned"""
+
+    syscfg_ver = get_macosx_ver_from_syscfg()
+    env_ver = os.environ.get(MACOSX_VERSION_VAR)
+
+    if env_ver:
+        # Validate overriden version against sysconfig version, if have both.
+        # Ensure that the deployment target of the build process is not less
+        # than 10.3 if the interpreter was built for 10.3 or later.  This
+        # ensures extension modules are built with correct compatibility
+        # values, specifically LDSHARED which can use
+        # '-undefined dynamic_lookup' which only works on >= 10.3.
+        if syscfg_ver and \
+            as_version(syscfg_ver) >= [10, 3] and as_version(env_ver) < [10, 3]:
+            my_msg = ('$' + MACOSX_VERSION_VAR + ' mismatch: '
+                      'now "%s" but "%s" during configure; '
+                      'must use 10.3 or later'
+                      % (env_ver, syscfg_ver))
+            raise DistutilsPlatformError(my_msg)
+        return env_ver
+    return syscfg_ver
+
+
+def as_version(s):
+    """Convert a dot-separated string into a list of numbers for comparisons"""
+    return [int(n) for n in s.split('.')]
+
+
 def convert_path (pathname):
     """Return 'pathname' as a name that will work on the native filesystem,
     i.e. split it on '/' and put it back together again using the current