From cb75204a70964118ec22369e0eac1ebd61a64be3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Mon, 4 Oct 2021 16:43:24 +0200 Subject: [PATCH 1/6] undistutils: the easy part - get_platform() -> cmd.plat_name - distutils.version -> packaging.version - distutils.sysconfig.get_config_var -> sysconfig.get_config_var --- buildutils/misc.py | 11 +++-------- setup.py | 35 ++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/buildutils/misc.py b/buildutils/misc.py index b7b40a205..0cab8e3ec 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -67,7 +67,7 @@ def get_output_error(cmd, **kwargs): return result.returncode, so, se -def locate_vcredist_dir(): +def locate_vcredist_dir(plat): """Locate vcredist directory and add it to $PATH Adding it to $PATH is required to run @@ -75,16 +75,11 @@ def locate_vcredist_dir(): """ from setuptools import msvc - try: - from setuptools._distutils.util import get_platform - except ImportError: - from distutils.util import get_platform - - vcvars = msvc.msvc14_get_vc_env(get_platform()) + vcvars = msvc.msvc14_get_vc_env(plat) try: vcruntime = vcvars["py_vcruntime_redist"] except KeyError: - warn(f"platform={get_platform()}, vcvars=") + warn(f"platform={plat}, vcvars=") pprint(vcvars, stream=sys.stderr) warn( diff --git a/setup.py b/setup.py index e5ad7b9a2..015ec7f8f 100755 --- a/setup.py +++ b/setup.py @@ -35,17 +35,17 @@ except ImportError: cffi = None +from packaging.version import Version as V from setuptools import setup, Command from setuptools.command.bdist_egg import bdist_egg from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist from setuptools.extension import Extension -import distutils.util from distutils.ccompiler import get_default_compiler from distutils.ccompiler import new_compiler -from distutils.sysconfig import customize_compiler, get_config_var -from distutils.version import LooseVersion as V +from distutils.sysconfig import customize_compiler +from sysconfig import get_config_var from glob import glob from os.path import splitext, basename, join as pjoin @@ -136,10 +136,7 @@ os.environ['ZMQ_DRAFT_API'] = '1' -if sys.platform.startswith('win'): - # ensure vcredist is on PATH - locate_vcredist_dir() -else: +if not sys.platform.startswith('win'): cxx_flags = os.getenv("CXXFLAGS", "") if "-std" not in cxx_flags: cxx_flags = "-std=c++11 " + cxx_flags @@ -159,7 +156,7 @@ # --- compiler settings ------------------------------------------------- -def bundled_settings(debug): +def bundled_settings(cmd): """settings for linking extensions against bundled libzmq""" settings = {} settings['libraries'] = [] @@ -172,13 +169,13 @@ def bundled_settings(debug): settings['libraries'].append('pthread') elif sys.platform.startswith('win'): # link against libzmq in build dir: - plat = distutils.util.get_platform() + plat = cmd.plat_name temp = 'temp.%s-%i.%i' % (plat, sys.version_info[0], sys.version_info[1]) if hasattr(sys, 'gettotalrefcount'): temp += '-pydebug' # Python 3.5 adds EXT_SUFFIX to libs - ext_suffix = distutils.sysconfig.get_config_var("EXT_SUFFIX") + ext_suffix = get_config_var("EXT_SUFFIX") suffix = os.path.splitext(ext_suffix)[0] if debug: @@ -232,7 +229,7 @@ def check_pkgconfig(): def _add_rpath(settings, path): """Add rpath to settings - Implemented here because distutils runtime_library_dirs doesn't do anything on darwin + Implemented here because setuptools runtime_library_dirs doesn't do anything on darwin """ if sys.platform == 'darwin': settings['extra_link_args'].extend(['-Wl,-rpath', '-Wl,%s' % path]) @@ -375,6 +372,10 @@ def finalize_options(self): self.config = discover_settings(self.build_base) if self.zmq is not None: merge(self.config, config_from_prefix(self.zmq)) + + # ensure vcredist is on PATH + if sys.platform.startswith("win"): + locate_vcredist_dir(self.plat_name) self.init_settings_from_config() def save_config(self, name, cfg): @@ -393,7 +394,7 @@ def init_settings_from_config(self): cfg = self.config if cfg['libzmq_extension']: - settings = bundled_settings(self.debug) + settings = bundled_settings(self) else: settings = settings_from_prefix(cfg['zmq_prefix']) @@ -779,7 +780,7 @@ def run(self): except LibZMQVersionError as e: info("\nBad libzmq version: %s\n" % e) except Exception as e: - # print the error as distutils would if we let it raise: + # print the error as setuptools would if we let it raise: info("\nerror: %s\n" % e) else: self.finish_run() @@ -796,7 +797,7 @@ def run(self): except LibZMQVersionError as e: info("\nBad libzmq version: %s\n" % e) except Exception as e: - # print the error as distutils would if we let it raise: + # print the error as setuptools would if we let it raise: info("\nerror: %s\n" % e) else: # if we get here the second run succeeded, so we need to update compiler @@ -861,7 +862,7 @@ def fetch_libzmq_src(self): class TestCommand(Command): - """Custom distutils command to run the test suite.""" + """Custom setuptools command to run the test suite.""" description = ( "Test PyZMQ (must have been built inplace: `setup.py build_ext --inplace`)" @@ -946,7 +947,7 @@ def finalize_options(self): class CleanCommand(Command): - """Custom distutils command to clean the .so and .pyc files.""" + """Custom setuptools command to clean the .so and .pyc files.""" user_options = [ ('all', 'a', "remove all build output, not just temporary by-products") @@ -1251,7 +1252,7 @@ def run(self): suffix = '.pyx' class CythonCommand(build_ext_cython): - """Custom distutils command subclassed from Cython.Distutils.build_ext + """Custom setuptools command subclassed from Cython.Distutils.build_ext to compile pyx->c, and stop there. All this does is override the C-compile method build_extension() with a no-op.""" From 61416502564fda1c22f4cc8b0d07c191e3168e4c Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Nov 2021 11:06:38 +0100 Subject: [PATCH 2/6] isort setup.py isort --py 36 --tc -m 3 setup.py --- setup.py | 60 ++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/setup.py b/setup.py index 015ec7f8f..ab822487d 100755 --- a/setup.py +++ b/setup.py @@ -16,66 +16,58 @@ # pyzmq-static: # ----------------------------------------------------------------------------- -from __future__ import with_statement, print_function - -from contextlib import contextmanager import copy +import errno import io import os +import platform import shutil import subprocess import sys import time -import errno -import platform +from contextlib import contextmanager +from distutils.ccompiler import get_default_compiler, new_compiler +from distutils.sysconfig import customize_compiler +from glob import glob +from os.path import basename +from os.path import join as pjoin +from os.path import splitext +from subprocess import PIPE, CalledProcessError, Popen, check_call +from sysconfig import get_config_var from traceback import print_exc -try: - import cffi -except ImportError: - cffi = None - from packaging.version import Version as V -from setuptools import setup, Command +from setuptools import Command, setup from setuptools.command.bdist_egg import bdist_egg from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist from setuptools.extension import Extension -from distutils.ccompiler import get_default_compiler -from distutils.ccompiler import new_compiler -from distutils.sysconfig import customize_compiler -from sysconfig import get_config_var - -from glob import glob -from os.path import splitext, basename, join as pjoin - -from subprocess import Popen, PIPE, check_call, CalledProcessError # local script imports: sys.path.insert(0, os.path.dirname(__file__)) from buildutils import ( - discover_settings, - v_str, - save_config, - detect_zmq, - merge, + bundled_version, + compile_and_forget, config_from_prefix, - info, - warn, - fatal, + customize_mingw, debug, + detect_zmq, + discover_settings, + fatal, + fetch_libzmq, + fetch_libzmq_dll, + info, line, localpath, locate_vcredist_dir, - fetch_libzmq, - fetch_libzmq_dll, - stage_platform_hpp, - bundled_version, - customize_mingw, - compile_and_forget, + merge, patch_lib_paths, + save_config, + stage_platform_hpp, + v_str, + warn, ) # ----------------------------------------------------------------------------- From e50222b44c7f7cb60908d7a0adf49435413bb41b Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Nov 2021 11:36:15 +0100 Subject: [PATCH 3/6] buildutils: pass around compiler objects instead of instantiating new ones with `distutils.ccompiler` which is deprecated with no replacement --- buildutils/detect.py | 22 ++++++++-------------- buildutils/misc.py | 11 ++--------- setup.py | 38 +++++++++++++++++++------------------- 3 files changed, 29 insertions(+), 42 deletions(-) diff --git a/buildutils/detect.py b/buildutils/detect.py index 359dc30f5..605c4c4a5 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -11,13 +11,12 @@ # the file COPYING.BSD, distributed as part of this software. # ----------------------------------------------------------------------------- +import copy import shutil import sys import os import logging import platform -from distutils import ccompiler -from distutils.sysconfig import customize_compiler from .misc import get_compiler, get_output_error from .msg import info @@ -30,9 +29,8 @@ # ----------------------------------------------------------------------------- -def test_compilation(cfile, compiler=None, **compiler_attrs): +def test_compilation(cfile, compiler, **compiler_attrs): """Test simple compilation with given settings""" - cc = get_compiler(compiler, **compiler_attrs) efile, ext = os.path.splitext(cfile) @@ -58,12 +56,12 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): extra_link = compiler_attrs.get('extra_link_args', []) lpreargs.extend(extra_link) - objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra) - cc.link_executable(objs, efile, extra_preargs=lpreargs) + objs = compiler.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra) + compiler.link_executable(objs, efile, extra_preargs=lpreargs) return efile -def compile_and_forget(basedir, src, compiler=None, **compiler_attrs): +def compile_and_forget(basedir, src, compiler, **compiler_attrs): """Make sure src compiles and links successfully. The resulting binary is deleted without being run. @@ -73,13 +71,12 @@ def compile_and_forget(basedir, src, compiler=None, **compiler_attrs): cfile = pjoin(basedir, os.path.basename(src)) shutil.copy(src, cfile) try: - cc = get_compiler(compiler, **compiler_attrs) - efile = test_compilation(cfile, compiler=cc, **compiler_attrs) + efile = test_compilation(cfile, compiler=compiler, **compiler_attrs) finally: shutil.rmtree(basedir) -def detect_zmq(basedir, compiler=None, **compiler_attrs): +def detect_zmq(basedir, compiler, **compiler_attrs): """Compile, link & execute a test program, in empty directory `basedir`. The C compiler will be updated with any keywords given via setattr. @@ -111,14 +108,11 @@ def detect_zmq(basedir, compiler=None, **compiler_attrs): # check if we need to link against Realtime Extensions library if sys.platform.startswith('linux'): - cc = ccompiler.new_compiler(compiler=compiler) - customize_compiler(cc) - cc.output_dir = basedir info("Checking for timer_create") info( "** Errors about missing timer_create are a normal part of this process **" ) - if not cc.has_function('timer_create'): + if not compiler.has_function('timer_create'): compiler_attrs['libraries'].append('rt') info( "** The above error about timer_create is normal and not a problem! **" diff --git a/buildutils/misc.py b/buildutils/misc.py index 0cab8e3ec..14b61aa68 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -3,11 +3,10 @@ # Copyright (c) PyZMQ Developers # Distributed under the terms of the Modified BSD License. +import copy import os import sys import logging -from distutils import ccompiler -from distutils.sysconfig import customize_compiler from pipes import quote from pprint import pprint from subprocess import Popen, PIPE @@ -36,13 +35,7 @@ def customize_mingw(cc): def get_compiler(compiler, **compiler_attrs): """get and customize a compiler""" - if compiler is None or isinstance(compiler, str): - cc = ccompiler.new_compiler(compiler=compiler) - customize_compiler(cc) - if cc.compiler_type == 'mingw32': - customize_mingw(cc) - else: - cc = compiler + cc = copy.deepcopy(compiler) for name, val in compiler_attrs.items(): setattr(cc, name, val) diff --git a/setup.py b/setup.py index ab822487d..c4b24d04c 100755 --- a/setup.py +++ b/setup.py @@ -26,8 +26,6 @@ import sys import time from contextlib import contextmanager -from distutils.ccompiler import get_default_compiler, new_compiler -from distutils.sysconfig import customize_compiler from glob import glob from os.path import basename from os.path import join as pjoin @@ -170,7 +168,7 @@ def bundled_settings(cmd): ext_suffix = get_config_var("EXT_SUFFIX") suffix = os.path.splitext(ext_suffix)[0] - if debug: + if cmd.debug: release = 'Debug' else: release = 'Release' @@ -368,7 +366,8 @@ def finalize_options(self): # ensure vcredist is on PATH if sys.platform.startswith("win"): locate_vcredist_dir(self.plat_name) - self.init_settings_from_config() + # need a dummy extension for run to set up a compiler + self.extensions = [Extension("fake", ["unused.c"])] def save_config(self, name, cfg): """write config to JSON""" @@ -399,7 +398,10 @@ def init_settings_from_config(self): pass try: compile_and_forget( - self.tempdir, pjoin('buildutils', 'check_sys_un.c'), **minus_zmq + self.tempdir, + pjoin('buildutils', 'check_sys_un.c'), + compiler=self.compiler, + **minus_zmq, ) except Exception as e: warn("No sys/un.h, IPC_PATH_MAX_LEN will be undefined: %s" % e) @@ -462,13 +464,7 @@ def erase_tempdir(self): @property def compiler_type(self): - compiler = self.compiler - if compiler is None: - return get_default_compiler() - elif isinstance(compiler, str): - return compiler - else: - return compiler.compiler_type + return self.compiler.compiler_type @property def cross_compiling(self): @@ -628,9 +624,9 @@ def bundle_libzmq_extension(self): # When compiling the C++ code inside of libzmq itself, we want to # avoid "warning C4530: C++ exception handler used, but unwind # semantics are not enabled. Specify /EHsc". - if self.compiler_type == 'msvc': + if self.compiler.compiler_type == 'msvc': libzmq.extra_compile_args.append('/EHsc') - elif self.compiler_type == 'mingw32': + elif self.compiler.compiler_type == 'mingw32': libzmq.define_macros.append(('ZMQ_HAVE_MINGW32', 1)) # And things like sockets come from libraries that must be named. @@ -640,13 +636,10 @@ def bundle_libzmq_extension(self): libzmq.include_dirs.append(bundledir) # check if we need to link against Realtime Extensions library - cc = new_compiler(compiler=self.compiler_type) - customize_compiler(cc) - cc.output_dir = self.build_temp if not sys.platform.startswith(('darwin', 'freebsd')): line() info("checking for timer_create") - if not cc.has_function('timer_create'): + if not self.compiler.has_function('timer_create'): info("no timer_create, linking librt") libzmq.libraries.append('rt') else: @@ -727,7 +720,7 @@ def test_build(self, prefix, settings): info("Configure: Autodetecting ZMQ settings...") info(" Custom ZMQ dir: %s" % prefix) try: - detected = detect_zmq(self.tempdir, compiler=self.compiler_type, **settings) + detected = detect_zmq(self.tempdir, compiler=self.compiler, **settings) finally: self.erase_tempdir() @@ -739,7 +732,14 @@ def finish_run(self): self.save_config('config', self.config) line() + def build_extensions(self): + """Need an empty build_extensions so that .run() gives us a configured compiler""" + def run(self): + # super().run() is what sets up self.compiler + super().run() + self.init_settings_from_config() + cfg = self.config if cfg['libzmq_extension']: From bec362447c0bd5002cf813894a2fec3e8ee8ce5d Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Nov 2021 11:48:17 +0100 Subject: [PATCH 4/6] get build_temp instead of reconstructing it when linking bundled libmzq on Windows, access the build dir directly --- setup.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/setup.py b/setup.py index c4b24d04c..f2641f3a5 100755 --- a/setup.py +++ b/setup.py @@ -159,22 +159,12 @@ def bundled_settings(cmd): settings['libraries'].append('pthread') elif sys.platform.startswith('win'): # link against libzmq in build dir: - plat = cmd.plat_name - temp = 'temp.%s-%i.%i' % (plat, sys.version_info[0], sys.version_info[1]) - if hasattr(sys, 'gettotalrefcount'): - temp += '-pydebug' - # Python 3.5 adds EXT_SUFFIX to libs ext_suffix = get_config_var("EXT_SUFFIX") suffix = os.path.splitext(ext_suffix)[0] - if cmd.debug: - release = 'Debug' - else: - release = 'Release' - settings['libraries'].append(libzmq_name + suffix) - settings['library_dirs'].append(pjoin('build', temp, release, 'buildutils')) + settings['library_dirs'].append(pjoin(cmd.build_temp, 'buildutils')) return settings From bfe554325c128a8e514b445e868b33d2ebe7e4c6 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Nov 2021 12:14:30 +0100 Subject: [PATCH 5/6] avoid global option in pip it doesn't work! --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf4f3cde8..11f3f607b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -137,6 +137,11 @@ jobs: sudo apt-get -y remove libzmq5 || true # workaround https://github.com/actions/virtual-environments/issues/3317 sudo apt-get -y install libzmq3-dev libsodium-dev + - name: set $ZMQ_PREFIX + if: matrix.zmq + run: | + echo "ZMQ_PREFIX=${{ matrix.zmq }}" >> "$GITHUB_ENV" + - name: install libzmq-dev if: matrix.zmq == 'head' run: | @@ -154,7 +159,7 @@ jobs: - name: build pyzmq run: | - pip install -v -e . --global-option=--zmq=${{ matrix.zmq }} + pip install -v -e . - name: import zmq run: | From df94062e5134a65a6224e62987b0a0a96fc1eef3 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 5 Nov 2021 12:18:02 +0100 Subject: [PATCH 6/6] EXT_SUFFIX from sysconfig is wrong on most versions of Python use distutils on Python < 3.9.2 --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f2641f3a5..d04a2b540 100755 --- a/setup.py +++ b/setup.py @@ -159,9 +159,15 @@ def bundled_settings(cmd): settings['libraries'].append('pthread') elif sys.platform.startswith('win'): # link against libzmq in build dir: - # Python 3.5 adds EXT_SUFFIX to libs - ext_suffix = get_config_var("EXT_SUFFIX") + if sys.version_info < (3, 9, 2): + # bpo-39825: EXT_SUFFIX is wrong from sysconfig prior to 3.9.2 / 3.8.7 + import distutils.sysconfig + + ext_suffix = distutils.sysconfig.get_config_var("EXT_SUFFIX") + else: + ext_suffix = get_config_var("EXT_SUFFIX") suffix = os.path.splitext(ext_suffix)[0] + print(f"DEBUG EXT_SUFFIX {suffix!r} {ext_suffix!r}") settings['libraries'].append(libzmq_name + suffix) settings['library_dirs'].append(pjoin(cmd.build_temp, 'buildutils'))