From d9eb9d534e852a55a73448683abd3effaf6b73cc Mon Sep 17 00:00:00 2001 From: mayeut Date: Fri, 17 May 2024 21:44:08 +0200 Subject: [PATCH 1/4] feat: add support for free-threaded (no-gil) Python 3.13 --- bin/update_pythons.py | 29 ++++++++++++++----- cibuildwheel/linux.py | 4 +++ cibuildwheel/macos.py | 4 +++ cibuildwheel/resources/build-platforms.toml | 13 +++++++++ .../resources/constraints-python313.txt | 2 +- cibuildwheel/resources/constraints.in | 3 +- cibuildwheel/util.py | 9 ++++-- cibuildwheel/windows.py | 26 ++++++++++++----- test/test_abi_variants.py | 16 +++++++--- test/utils.py | 2 ++ 10 files changed, 85 insertions(+), 23 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 05a68b0d4..ecf8e3ef2 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -58,7 +58,7 @@ class ConfigMacOS(TypedDict): class WindowsVersions: - def __init__(self, arch_str: ArchStr) -> None: + def __init__(self, arch_str: ArchStr, free_threaded: bool) -> None: response = requests.get("https://api.nuget.org/v3/index.json") response.raise_for_status() api_info = response.json() @@ -72,7 +72,11 @@ def __init__(self, arch_str: ArchStr) -> None: self.arch_str = arch_str self.arch = ARCH_DICT[arch_str] + self.free_threaded = free_threaded + package = PACKAGE_DICT[arch_str] + if free_threaded: + package = f"{package}-freethreaded" response = requests.get(f"{endpoint}{package}/index.json") response.raise_for_status() @@ -92,8 +96,9 @@ def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: if not versions: return None + flags = "t" if self.free_threaded else "" version = versions[0] - identifier = f"cp{version.major}{version.minor}-{self.arch}" + identifier = f"cp{version.major}{version.minor}{flags}-{self.arch}" return ConfigWinCP( identifier=identifier, version=self.version_dict[version], @@ -233,9 +238,12 @@ def update_version_macos( class AllVersions: def __init__(self) -> None: - self.windows_32 = WindowsVersions("32") - self.windows_64 = WindowsVersions("64") - self.windows_arm64 = WindowsVersions("ARM64") + self.windows_32 = WindowsVersions("32", False) + self.windows_t_32 = WindowsVersions("32", True) + self.windows_64 = WindowsVersions("64", False) + self.windows_t_64 = WindowsVersions("64", True) + self.windows_arm64 = WindowsVersions("ARM64", False) + self.windows_t_arm64 = WindowsVersions("ARM64", True) self.windows_pypy_64 = PyPyVersions("64") self.macos_cpython = CPythonVersions() @@ -259,14 +267,19 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.macos_pypy.update_version_macos(spec) elif "macosx_arm64" in identifier: config_update = self.macos_pypy_arm64.update_version_macos(spec) - elif "win32" in identifier: - if identifier.startswith("cp"): - config_update = self.windows_32.update_version_windows(spec) + elif "t-win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_32.update_version_windows(spec) + elif "win32" in identifier and identifier.startswith("cp"): + config_update = self.windows_32.update_version_windows(spec) + elif "t-win_amd64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_64.update_version_windows(spec) elif "win_amd64" in identifier: if identifier.startswith("cp"): config_update = self.windows_64.update_version_windows(spec) elif identifier.startswith("pp"): config_update = self.windows_pypy_64.update_version_windows(spec) + elif "t-win_arm64" in identifier and identifier.startswith("cp"): + config_update = self.windows_t_arm64.update_version_windows(spec) elif "win_arm64" in identifier and identifier.startswith("cp"): config_update = self.windows_arm64.update_version_windows(spec) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 55e61f2d3..1fc686126 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -337,6 +337,10 @@ def build_in_container( virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version == "3.13": + container.call(["pip", "install", "pip>=24.1b1"], env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6b219a102..3df239618 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -575,6 +575,10 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call_with_arch("which", "python", env=virtualenv_env) + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version == "3.13": + call("python", "-m", "pip", "install", "pip>=24.1b1", env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f95d329cc..2f0393e11 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -8,6 +8,7 @@ python_configurations = [ { identifier = "cp311-manylinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -16,6 +17,7 @@ python_configurations = [ { identifier = "cp311-manylinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, @@ -28,6 +30,7 @@ python_configurations = [ { identifier = "cp311-manylinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -36,6 +39,7 @@ python_configurations = [ { identifier = "cp311-manylinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -44,6 +48,7 @@ python_configurations = [ { identifier = "cp311-manylinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-manylinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-manylinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/pp38-pypy38_pp73" }, { identifier = "pp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/pp39-pypy39_pp73" }, @@ -60,6 +65,7 @@ python_configurations = [ { identifier = "cp311-musllinux_x86_64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_x86_64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_x86_64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -68,6 +74,7 @@ python_configurations = [ { identifier = "cp311-musllinux_i686", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_i686", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_i686", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -76,6 +83,7 @@ python_configurations = [ { identifier = "cp311-musllinux_aarch64", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_aarch64", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_aarch64", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -84,6 +92,7 @@ python_configurations = [ { identifier = "cp311-musllinux_ppc64le", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_ppc64le", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_ppc64le", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, { identifier = "cp36-musllinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-musllinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-musllinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, @@ -92,6 +101,7 @@ python_configurations = [ { identifier = "cp311-musllinux_s390x", version = "3.11", path_str = "/opt/python/cp311-cp311" }, { identifier = "cp312-musllinux_s390x", version = "3.12", path_str = "/opt/python/cp312-cp312" }, { identifier = "cp313-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313" }, + { identifier = "cp313t-musllinux_s390x", version = "3.13", path_str = "/opt/python/cp313-cp313t" }, ] [macos] @@ -142,12 +152,15 @@ python_configurations = [ { identifier = "cp312-win32", version = "3.12.3", arch = "32" }, { identifier = "cp312-win_amd64", version = "3.12.3", arch = "64" }, { identifier = "cp313-win32", version = "3.13.0-b1", arch = "32" }, + { identifier = "cp313t-win32", version = "3.13.0-b1", arch = "32" }, { identifier = "cp313-win_amd64", version = "3.13.0-b1", arch = "64" }, + { identifier = "cp313t-win_amd64", version = "3.13.0-b1", arch = "64" }, { identifier = "cp39-win_arm64", version = "3.9.10", arch = "ARM64" }, { identifier = "cp310-win_arm64", version = "3.10.11", arch = "ARM64" }, { identifier = "cp311-win_arm64", version = "3.11.9", arch = "ARM64" }, { identifier = "cp312-win_arm64", version = "3.12.3", arch = "ARM64" }, { identifier = "cp313-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, + { identifier = "cp313t-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-win64.zip" }, { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-win64.zip" }, { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" }, diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 474fa6de0..5f4f1bf97 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -16,7 +16,7 @@ packaging==24.0 # via # build # delocate -pip==24.0 +pip==24.1b1 # via -r cibuildwheel/resources/constraints.in platformdirs==4.2.2 # via virtualenv diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in index 50bfabb6e..4f01a4b8c 100644 --- a/cibuildwheel/resources/constraints.in +++ b/cibuildwheel/resources/constraints.in @@ -1,4 +1,5 @@ -pip +pip>=24.1b1 ; python_version >= '3.13' +pip ; python_version < '3.13' build delocate virtualenv diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 66be99ca7..e704911cb 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -238,13 +238,15 @@ class BuildSelector: requires_python: SpecifierSet | None = None # a pattern that skips prerelease versions, when include_prereleases is False. - PRERELEASE_SKIP: ClassVar[str] = "cp313-*" + PRERELEASE_SKIP: ClassVar[str] = "cp313-* cp313t-*" prerelease_pythons: bool = False def __call__(self, build_id: str) -> bool: # Filter build selectors by python_requires if set if self.requires_python is not None: py_ver_str = build_id.split("-")[0] + if py_ver_str.endswith("t"): + py_ver_str = py_ver_str[:-1] major = int(py_ver_str[2]) minor = int(py_ver_str[3:]) version = Version(f"{major}.{minor}.99") @@ -645,10 +647,13 @@ def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: """ interpreter, platform = identifier.split("-") + free_threaded = interpreter.endswith("t") + if free_threaded: + interpreter = interpreter[:-1] for wheel in wheels: _, _, _, tags = parse_wheel_filename(wheel.name) for tag in tags: - if tag.abi == "abi3": + if tag.abi == "abi3" and not free_threaded: # ABI3 wheels must start with cp3 for impl and tag if not (interpreter.startswith("cp3") and tag.interpreter.startswith("cp3")): continue diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e661716d4..a8e82b65e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -44,7 +44,9 @@ ) -def get_nuget_args(version: str, arch: str, output_directory: Path) -> list[str]: +def get_nuget_args( + version: str, arch: str, free_threaded: bool, output_directory: Path +) -> list[str]: package_name = { "32": "pythonx86", "64": "python", @@ -53,6 +55,8 @@ def get_nuget_args(version: str, arch: str, output_directory: Path) -> list[str] "x86": "pythonx86", "AMD64": "python", }[arch] + if free_threaded: + package_name = f"{package_name}-freethreaded" return [ package_name, "-Version", @@ -106,11 +110,12 @@ def _ensure_nuget() -> Path: return nuget -def install_cpython(version: str, arch: str) -> Path: +def install_cpython(version: str, arch: str, free_threaded: bool) -> Path: base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" - nuget_args = get_nuget_args(version, arch, base_output_dir) + nuget_args = get_nuget_args(version, arch, free_threaded, base_output_dir) installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" - with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"): + free_threaded_str = "-freethreaded" if free_threaded else "" + with FileLock(str(base_output_dir) + f"-{version}{free_threaded_str}-{arch}.lock"): if not installation_path.exists(): nuget = _ensure_nuget() call(nuget, "install", *nuget_args) @@ -224,18 +229,21 @@ def setup_python( log.step(f"Installing Python {implementation_id}...") if implementation_id.startswith("cp"): native_arch = platform_module.machine() + free_threaded = implementation_id.endswith("t") if python_configuration.arch == "ARM64" != native_arch: # To cross-compile for ARM64, we need a native CPython to run the # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') # for any extension modules. python_libs_base = install_cpython( - python_configuration.version, python_configuration.arch + python_configuration.version, python_configuration.arch, free_threaded ) python_libs_base = python_libs_base.parent / "libs" log.step(f"Installing native Python {native_arch} for cross-compilation...") - base_python = install_cpython(python_configuration.version, native_arch) + base_python = install_cpython(python_configuration.version, native_arch, free_threaded) else: - base_python = install_cpython(python_configuration.version, python_configuration.arch) + base_python = install_cpython( + python_configuration.version, python_configuration.arch, free_threaded + ) elif implementation_id.startswith("pp"): assert python_configuration.url is not None base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) @@ -521,6 +529,10 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call("where", "python", env=virtualenv_env) + # TODO remove me once virtualenv provides pip>=24.1b1 + if config.version.startswith("3.13."): + call("python", "-m", "pip", "install", "--pre", "-U", "pip", env=virtualenv_env) + if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index d203c06fe..4446ed56c 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -13,9 +13,14 @@ limited_api_project = test_projects.new_c_project( setup_py_add=textwrap.dedent( r""" + import sysconfig + + IS_CPYTHON = sys.implementation.name == "cpython" + Py_GIL_DISABLED = sysconfig.get_config_var("Py_GIL_DISABLED") + CAN_USE_ABI3 = IS_CPYTHON and not Py_GIL_DISABLED cmdclass = {} extension_kwargs = {} - if sys.version_info[:2] >= (3, 8): + if CAN_USE_ABI3 and sys.version_info[:2] >= (3, 8): from wheel.bdist_wheel import bdist_wheel as _bdist_wheel class bdist_wheel_abi3(_bdist_wheel): @@ -47,7 +52,8 @@ def test_abi3(tmp_path): actual_wheels = utils.cibuildwheel_run( project_dir, add_env={ - "CIBW_SKIP": "pp* ", # PyPy does not have a Py_LIMITED_API equivalent + # free_threaded and PyPy do not have a Py_LIMITED_API equivalent, just build one of those + "CIBW_BUILD": "cp3?-* cp31?-* cp313t-* pp310-*" }, ) @@ -55,7 +61,9 @@ def test_abi3(tmp_path): expected_wheels = [ w.replace("cp38-cp38", "cp38-abi3") for w in utils.expected_wheels("spam", "0.1.0") - if "-pp" not in w and "-cp39" not in w and "-cp31" not in w + if ("-pp310" in w or "-pp" not in w) + and "-cp39" not in w + and ("-cp313t" in w or "-cp31" not in w) ] assert set(actual_wheels) == set(expected_wheels) @@ -177,7 +185,7 @@ def test_abi_none(tmp_path, capfd): "CIBW_TEST_REQUIRES": "pytest", "CIBW_TEST_COMMAND": "pytest {project}/test", # limit the number of builds for test performance reasons - "CIBW_BUILD": "cp38-* cp310-* pp39-*", + "CIBW_BUILD": "cp38-* cp310-* cp313t-* pp310-*", }, ) diff --git a/test/utils.py b/test/utils.py index f6f610c41..4bc02f47d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -176,6 +176,8 @@ def expected_wheels( "cp312-cp312", "cp313-cp313", ] + if platform != "macos": + python_abi_tags.append("cp313-cp313t") if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]: python_abi_tags += [ From 46f76a774a86a1fc8f37a8f40fe0c6cb42c4f055 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 19 May 2024 13:53:37 +0200 Subject: [PATCH 2/4] feat: add option to opt-in free-threaded builds --- bin/generate_schema.py | 5 +++ cibuildwheel/options.py | 12 +++++-- .../resources/cibuildwheel.schema.json | 6 ++++ cibuildwheel/resources/defaults.toml | 1 + cibuildwheel/util.py | 7 ++++ test/utils.py | 5 +++ unit_test/build_selector_test.py | 9 +++++ unit_test/options_test.py | 33 +++++++++++++++++++ 8 files changed, 75 insertions(+), 3 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 419418970..231edd909 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -104,6 +104,10 @@ description: Set environment variables on the host to pass-through to the container during the build. type: string_array + free-threaded-support: + type: boolean + default: false + description: The project supports free-threaded builds of Python (PEP703) manylinux-aarch64-image: type: string description: Specify alternative manylinux / musllinux container images @@ -248,6 +252,7 @@ del non_global_options["build"] del non_global_options["skip"] del non_global_options["test-skip"] +del non_global_options["free-threaded-support"] overrides["items"]["properties"]["select"]["oneOf"] = string_array overrides["items"]["properties"] |= non_global_options.copy() diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index e5b4458e8..da2d68d65 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -117,7 +117,7 @@ def architectures(self) -> set[Architecture]: return self.globals.architectures -Setting = Union[Mapping[str, str], Sequence[str], str, int] +Setting = Union[Mapping[str, str], Sequence[str], str, int, bool] @dataclasses.dataclass(frozen=True) @@ -196,7 +196,7 @@ def _resolve_cascade( if value is None: continue - if ignore_empty and not value: + if ignore_empty and not value and value is not False: continue value_string = _stringify_setting(value, list_sep, table_format) @@ -258,7 +258,7 @@ def _stringify_setting( raise ConfigOptionError(msg) return list_sep.join(setting) - if isinstance(setting, int): + if isinstance(setting, (bool, int)): return str(setting) return setting @@ -516,6 +516,10 @@ def globals(self) -> GlobalOptions: skip_config = self.reader.get("skip", env_plat=False, list_sep=" ") test_skip = self.reader.get("test-skip", env_plat=False, list_sep=" ") + free_threaded_support = strtobool( + self.reader.get("free-threaded-support", env_plat=False, ignore_empty=True) + ) + prerelease_pythons = args.prerelease_pythons or strtobool( self.env.get("CIBW_PRERELEASE_PYTHONS", "0") ) @@ -536,12 +540,14 @@ def globals(self) -> GlobalOptions: skip_config = "" architectures = Architecture.all_archs(self.platform) prerelease_pythons = True + free_threaded_support = True build_selector = BuildSelector( build_config=build_config, skip_config=skip_config, requires_python=requires_python, prerelease_pythons=prerelease_pythons, + free_threaded_support=free_threaded_support, ) test_selector = TestSelector(skip_config=test_skip) diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 2681aca5d..d0ba3ff60 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -255,6 +255,12 @@ ], "title": "CIBW_ENVIRONMENT_PASS" }, + "free-threaded-support": { + "type": "boolean", + "default": false, + "description": "The project supports free-threaded builds of Python (PEP703)", + "title": "CIBW_FREE_THREADED_SUPPORT" + }, "manylinux-aarch64-image": { "type": "string", "description": "Specify alternative manylinux / musllinux container images", diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index b3ddeb95a..85e82f2f7 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -2,6 +2,7 @@ build = "*" skip = "" test-skip = "" +free-threaded-support = false archs = ["auto"] build-frontend = "default" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index e704911cb..72c937d71 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -241,6 +241,8 @@ class BuildSelector: PRERELEASE_SKIP: ClassVar[str] = "cp313-* cp313t-*" prerelease_pythons: bool = False + free_threaded_support: bool = False + def __call__(self, build_id: str) -> bool: # Filter build selectors by python_requires if set if self.requires_python is not None: @@ -257,6 +259,10 @@ def __call__(self, build_id: str) -> bool: if not self.prerelease_pythons and selector_matches(self.PRERELEASE_SKIP, build_id): return False + # filter out free threaded pythons if self.free_threaded_support is False + if not self.free_threaded_support and selector_matches("*t-*", build_id): + return False + should_build = selector_matches(self.build_config, build_id) should_skip = selector_matches(self.skip_config, build_id) @@ -268,6 +274,7 @@ def options_summary(self) -> Any: "skip_config": self.skip_config, "requires_python": str(self.requires_python), "prerelease_pythons": self.prerelease_pythons, + "free_threaded_support": self.free_threaded_support, } diff --git a/test/utils.py b/test/utils.py index 4bc02f47d..08e2bc93a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,6 +37,9 @@ def cibuildwheel_get_build_identifiers(project_path, env=None, *, prerelease_pyt cmd = [sys.executable, "-m", "cibuildwheel", "--print-build-identifiers", str(project_path)] if prerelease_pythons: cmd.append("--prerelease-pythons") + if env is None: + env = os.environ.copy() + env.setdefault("CIBW_FREE_THREADED_SUPPORT", "1") cmd_output = subprocess.run( cmd, @@ -94,6 +97,8 @@ def cibuildwheel_run( _update_pip_cache_dir(env) + env.setdefault("CIBW_FREE_THREADED_SUPPORT", "1") + with TemporaryDirectory() as tmp_output_dir: subprocess.run( [ diff --git a/unit_test/build_selector_test.py b/unit_test/build_selector_test.py index 7d7eeb5fb..6066c82e8 100644 --- a/unit_test/build_selector_test.py +++ b/unit_test/build_selector_test.py @@ -50,6 +50,7 @@ def test_build_filter_pre(): assert build_selector("cp313-manylinux_x86_64") assert build_selector("cp37-win_amd64") assert build_selector("cp313-win_amd64") + assert not build_selector("cp313t-manylinux_x86_64") def test_skip(): @@ -144,6 +145,14 @@ def test_build_limited_python_patch(): assert build_selector("cp37-manylinux_x86_64") +def test_build_free_threaded_python(): + build_selector = BuildSelector( + build_config="*", skip_config="", prerelease_pythons=True, free_threaded_support=True + ) + + assert build_selector("cp313t-manylinux_x86_64") + + def test_testing_selector(): # local import to avoid pytest trying to collect this as a test class! from cibuildwheel.util import TestSelector diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 288f95caa..1b482c752 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -415,3 +415,36 @@ def test_override_inherit_environment_with_references(tmp_path: Path): ) == { "PATH": "/opt/local/bin:/opt/bin:/usr/bin:/bin", } + + +@pytest.mark.parametrize( + ("toml_assignment", "env", "expected_result"), + [ + ("", {}, False), + ("free-threaded-support = true", {}, True), + ("free-threaded-support = false", {}, False), + ("", {"CIBW_FREE_THREADED_SUPPORT": "0"}, False), + ("", {"CIBW_FREE_THREADED_SUPPORT": "1"}, True), + ("free-threaded-support = false", {"CIBW_FREE_THREADED_SUPPORT": "1"}, True), + ("free-threaded-support = true", {"CIBW_FREE_THREADED_SUPPORT": "0"}, False), + ("free-threaded-support = true", {"CIBW_FREE_THREADED_SUPPORT": ""}, True), + ("free-threaded-support = false", {"CIBW_FREE_THREADED_SUPPORT": ""}, False), + ], +) +def test_free_threaded_support( + tmp_path: Path, toml_assignment: str, env: dict[str, str], expected_result: bool +): + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + options = Options(platform="linux", command_line_arguments=args, env=env) + assert options.globals.build_selector.free_threaded_support is expected_result From 608755164c9e4e96e101d63116fa58e1d7202592 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 19 May 2024 14:01:12 +0200 Subject: [PATCH 3/4] chore: pass a PythonConfiguration to install_cpython --- cibuildwheel/windows.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a8e82b65e..f2ddc0127 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -110,7 +110,11 @@ def _ensure_nuget() -> Path: return nuget -def install_cpython(version: str, arch: str, free_threaded: bool) -> Path: +def install_cpython(configuration: PythonConfiguration, arch: str | None = None) -> Path: + version = configuration.version + free_threaded = "t-" in configuration.identifier + if arch is None: + arch = configuration.arch base_output_dir = CIBW_CACHE_PATH / "nuget-cpython" nuget_args = get_nuget_args(version, arch, free_threaded, base_output_dir) installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools" @@ -229,21 +233,14 @@ def setup_python( log.step(f"Installing Python {implementation_id}...") if implementation_id.startswith("cp"): native_arch = platform_module.machine() - free_threaded = implementation_id.endswith("t") + base_python = install_cpython(python_configuration) if python_configuration.arch == "ARM64" != native_arch: # To cross-compile for ARM64, we need a native CPython to run the # build, and a copy of the ARM64 import libraries ('.\libs\*.lib') # for any extension modules. - python_libs_base = install_cpython( - python_configuration.version, python_configuration.arch, free_threaded - ) - python_libs_base = python_libs_base.parent / "libs" + python_libs_base = base_python.parent / "libs" log.step(f"Installing native Python {native_arch} for cross-compilation...") - base_python = install_cpython(python_configuration.version, native_arch, free_threaded) - else: - base_python = install_cpython( - python_configuration.version, python_configuration.arch, free_threaded - ) + base_python = install_cpython(python_configuration, arch=native_arch) elif implementation_id.startswith("pp"): assert python_configuration.url is not None base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url) From 037b274d9c44c95d942e7d70f34b8e75b11b2647 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 19 May 2024 16:07:02 +0200 Subject: [PATCH 4/4] doc: add documentation for the `CIBW_FREE_THREADED_SUPPORT` option --- docs/options.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/options.md b/docs/options.md index 75ac6b9be..7872ec5ef 100644 --- a/docs/options.md +++ b/docs/options.md @@ -408,6 +408,53 @@ See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for } +### `CIBW_FREE_THREADED_SUPPORT` {: #free-threaded-support} + +> Choose whether free-threaded variants should be built + +[PEP 703](https://www.python.org/dev/peps/pep-0703) introduced variants of CPython that can be built without the Global Interpreter Lock (GIL). +Those variants are also known as free-threaded / no-gil. + +Building for free-threaded variants is disabled by default. + +Building can be enabled by setting this option to `true`. The free-threaded compatible wheels will be built in addition to the standard wheels. + +This option doesn't support overrides. +If you need to enable/disable it per platform or python version, set this option to `true` and use [`CIBW_BUILD`](#build-skip)/[`CIBW_SKIP`](#build-skip) options to filter the builds. + +The build identifiers for those variants have a `t` suffix in their `python_tag` (e.g. `cp313t-manylinux_x86_64`) + +!!! note + This feature is experimental: [What’s New In Python 3.13](https://docs.python.org/3.13/whatsnew/3.13.html#free-threaded-cpython) + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Enable free-threaded support + CIBW_FREE_THREADED_SUPPORT: 1 + + # Skip building free-threaded compatible wheels on Windows + CIBW_FREE_THREADED_SUPPORT: 1 + CIBW_SKIP: *t-win* + ``` + + It is generally recommended to use `free-threaded-support` in a config file as you can statically declare that you + support free-threaded builds. + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel] + # Enable free-threaded support + free-threaded-support = true + + # Skip building free-threaded compatible wheels on Windows + free-threaded-support = true + skip = "*t-win*" + ``` + ### `CIBW_ARCHS` {: #archs} > Change the architectures built on your machine by default.