From a604d11643b37ff5dd42a155dafd42e16fbb8b9e Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 30 Oct 2022 20:56:17 +0100 Subject: [PATCH 01/12] ENH: determine wheel tags by Python interpreter introspection The extension modules filename suffixes do not contain enough information to correctly determine the wheel tags. Instead introspect the Python interpreter to derive the wheel tags. This is the same approach used by other PEP517 backends, most notably wheel. The wheel contents only to determine whether the wheel contains python ABI dependent modules or other platform dependent code. The packaging module is the reference wheel tags derivation implementation and it is used (or vendored) by most python packages dealing with wheels. However, the API provided by packaging is cumbersome to use for our purposes and, with the goal of merging this code into Meson in the future, it is good to avoid an additional dependency. Therefore, the tags derivation code is reimplemented. Tests are added to verify that the tags produced by meson-python agree with the ones produced by packaging to ensure that the two implementations will not diverge. Fixes #142, fixes #189, fixes #190. --- mesonpy/__init__.py | 223 +++++++++--------------------------- mesonpy/_compat.py | 7 +- mesonpy/_tags.py | 273 ++++++++++++++++++++++++-------------------- tests/conftest.py | 11 ++ tests/test_tags.py | 157 ++++++++++++++----------- tests/test_wheel.py | 56 +++------ 6 files changed, 324 insertions(+), 403 deletions(-) diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 6a271d781..1e5d1591c 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -13,6 +13,7 @@ import collections import contextlib import functools +import importlib.machinery import io import itertools import json @@ -46,7 +47,7 @@ import mesonpy._tags import mesonpy._util -from mesonpy._compat import Collection, Iterator, Mapping, Path +from mesonpy._compat import Iterator, Path if typing.TYPE_CHECKING: # pragma: no cover @@ -102,9 +103,9 @@ def _init_colors() -> Dict[str, str]: _STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS -_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.so$') -_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.pyd$') -_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P[0-9]+)$') +_EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES) +_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P[^.]+)\.)?(?:so|pyd)$') +assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES) def _showwarning( @@ -179,6 +180,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]: def _has_internal_libs(self) -> bool: return bool(self._wheel_files['mesonpy-libs']) + @property + def _has_extension_modules(self) -> bool: + # Assume that all code installed in {platlib} is Python ABI dependent. + return bool(self._wheel_files['platlib']) + @property def basename(self) -> str: """Normalized wheel name and version (eg. meson_python-1.0.0).""" @@ -187,14 +193,25 @@ def basename(self) -> str: version=self._project.version, ) + @property + def tag(self) -> mesonpy._tags.Tag: + """Wheel tags.""" + if self.is_pure: + return mesonpy._tags.Tag('py3', 'none', 'any') + if not self._has_extension_modules: + # The wheel has platform dependent code (is not pure) but + # does not contain any extension module (does not + # distribute any file in {platlib}) thus use generic + # implementation and ABI tags. + return mesonpy._tags.Tag('py3', 'none', None) + return mesonpy._tags.Tag(None, self._stable_abi, None) + @property def name(self) -> str: - """Wheel name, this includes the basename and tags.""" - return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format( + """Wheel name, this includes the basename and tag.""" + return '{basename}-{tag}'.format( basename=self.basename, - python_tag=self.python_tag, - abi_tag=self.abi_tag, - platform_tag=self.platform_tag, + tag=self.tag, ) @property @@ -226,10 +243,10 @@ def wheel(self) -> bytes: # noqa: F811 Wheel-Version: 1.0 Generator: meson Root-Is-Purelib: {is_purelib} - Tag: {tags} + Tag: {tag} ''').strip().format( is_purelib='true' if self.is_pure else 'false', - tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}', + tag=self.tag, ).encode() @property @@ -267,166 +284,40 @@ def _debian_python(self) -> bool: except ModuleNotFoundError: return False - @property - def python_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag and selected_tag.python: - return selected_tag.python - return 'py3' - - @property - def abi_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag: - return selected_tag.abi - return 'none' - @cached_property - def platform_tag(self) -> str: - if self.is_pure: - return 'any' - # XXX: Choose the sysconfig platform here and let something like auditwheel - # fix it later if there are system dependencies (eg. replace it with a manylinux tag) - platform_ = sysconfig.get_platform() - parts = platform_.split('-') - if parts[0] == 'macosx': - target = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if target: - print( - '{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the ' - 'platform tag to {target}{reset}'.format(target=target, **_STYLES) - ) - parts[1] = target - else: - # If no target macOS version is specified fallback to - # platform.mac_ver() instead of sysconfig.get_platform() as the - # latter specifies the target macOS version Python was built - # against. - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - # Only pick up the major version, which changed from 10.X - # to X.0 from macOS 11 onwards. See - # https://github.com/mesonbuild/meson-python/issues/160 - parts[1] = parts[1].split('.')[0] - - if parts[1] in ('11', '12'): - # Workaround for bug where pypa/packaging does not consider macOS - # tags without minor versions valid. Some Python flavors (Homebrew - # for example) on macOS started to do this in version 11, and - # pypa/packaging should handle things correctly from version 13 and - # forward, so we will add a 0 minor version to MacOS 11 and 12. - # https://github.com/mesonbuild/meson-python/issues/91 - # https://github.com/pypa/packaging/issues/578 - parts[1] += '.0' - - platform_ = '-'.join(parts) - elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff: - # 32-bit Python running on an x86_64 host - # https://github.com/mesonbuild/meson-python/issues/123 - parts[1] = 'i686' - platform_ = '-'.join(parts) - return platform_.replace('-', '_').replace('.', '_') - - def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Windows tag from the Python extension file name.""" - match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename) - if not match: - return None - tag = match.group('tag') + def _stable_abi(self) -> Optional[str]: + """Determine stabe ABI compatibility. - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Posix tag from the Python extension file name.""" - # sysconfig is not guaranted to export SHLIB_SUFFIX but let's be - # preventive and check its value to make sure it matches our expectations - try: - extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so') - if extension != '.so': - raise NotImplementedError( - f"We don't currently support the {extension} extension. " - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include information about your operating system.' - ) - except KeyError: - warnings.warn( - 'sysconfig does not export SHLIB_SUFFIX, so we are unable to ' - 'perform the sanity check regarding the extension suffix. ' - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include the output of `python -m sysconfig`.' - ) - match = _LINUX_NATIVE_MODULE_REGEX.match(filename) - if not match: # this file does not appear to be a native module - return None - tag = match.group('tag') + Examine all files installed in {platlib} that look like + extension modules (extension .pyd on Windows and .so on other + platforms) and, if they all share the same PEP 3149 filename + stable ABI tag, return it. - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the ABI tag from the Python extension file name.""" - if os.name == 'nt': - return self._calculate_file_abi_tag_heuristic_windows(filename) - # everything else *should* follow the POSIX way, at least to my knowledge - return self._calculate_file_abi_tag_heuristic_posix(filename) - - def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str: - if len(files) > max_count: - files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))'] - return ''.join(f'{prefix}- {file}\n' for file in files) - - def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]: - """Map files into ABI tags.""" - files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list) - - for _, file in self._wheel_files['platlib']: - # if in platlib, calculate the ABI tag - tag = self._calculate_file_abi_tag_heuristic(file) - if tag: - files_by_tag[tag].append(file) - - return files_by_tag - - def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]: # noqa: C901 - """Given a list of ABI tags, selects the most specific one. - - Raises an error if there are incompatible tags. + All files that look like extension modules are verified to + have a file name compatibel with what is expected by the + Python interpreter. An exception is raised otherwise. + + Other files are ignored. """ - # Possibilities: - # - interpreter specific (cpython/pypy/etc, version) - # - stable abi (abiX) - tags = self._files_by_tag() - selected_tag = None - for tag, files in tags.items(): - # no selected tag yet, let's assign this one - if not selected_tag: - selected_tag = tag - # interpreter tag - elif isinstance(tag, mesonpy._tags.InterpreterTag): - if tag != selected_tag: - if isinstance(selected_tag, mesonpy._tags.InterpreterTag): - raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - selected_tag = tag - # stable ABI - elif isinstance(tag, mesonpy._tags.StableABITag): - if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag: + abis = [] + + for path, src in self._wheel_files['platlib']: + if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so': + match = re.match(r'^[^.]+(.*)$', path.name) + assert match is not None + suffix = match.group(1) + if suffix not in _EXTENSION_SUFFIXES: raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - return selected_tag + f'Extension module {str(path)!r} not compatible with Python interpreter. ' + f'Filename suffix {suffix!r} not in {set(_EXTENSION_SUFFIXES)}.') + match = _EXTENSION_SUFFIX_REGEX.match(suffix) + assert match is not None + abis.append(match.group('abi')) + + stable = [x for x in abis if x and re.match(r'abi\d+', x)] + if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]): + return stable[0] + return None def _is_native(self, file: Union[str, pathlib.Path]) -> bool: """Check if file is a native file.""" diff --git a/mesonpy/_compat.py b/mesonpy/_compat.py index 9fab9054f..dc74de40b 100644 --- a/mesonpy/_compat.py +++ b/mesonpy/_compat.py @@ -10,11 +10,9 @@ if sys.version_info >= (3, 9): - from collections.abc import ( - Collection, Iterable, Iterator, Mapping, Sequence - ) + from collections.abc import Collection, Iterable, Iterator, Sequence else: - from typing import Collection, Iterable, Iterator, Mapping, Sequence + from typing import Collection, Iterable, Iterator, Sequence if sys.version_info >= (3, 8): @@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool: 'Iterable', 'Iterator', 'Literal', - 'Mapping', 'Path', 'Sequence', ] diff --git a/mesonpy/_tags.py b/mesonpy/_tags.py index af83892e0..81725e0d6 100644 --- a/mesonpy/_tags.py +++ b/mesonpy/_tags.py @@ -1,130 +1,151 @@ # SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Quansight, LLC -# SPDX-FileCopyrightText: 2021 Filipe Laíns -import abc -import re - -from typing import Any, Optional - -from mesonpy._compat import Literal, Sequence - - -class Tag(abc.ABC): - @abc.abstractmethod - def __init__(self, value: str) -> None: ... - - @abc.abstractmethod - def __str__(self) -> str: ... - - @property - @abc.abstractmethod - def python(self) -> Optional[str]: - """Python tag.""" - - @property - @abc.abstractmethod - def abi(self) -> str: - """ABI tag.""" - - -class StableABITag(Tag): - _REGEX = re.compile(r'^abi(?P[0-9]+)$') - - def __init__(self, value: str) -> None: - match = self._REGEX.match(value) - if not match: - raise ValueError(f'Invalid PEP 3149 stable ABI tag, expecting pattern `{self._REGEX.pattern}`') - self._abi_number = int(match.group('abi_number')) - - @property - def abi_number(self) -> int: - return self._abi_number - - def __str__(self) -> str: - return f'abi{self.abi_number}' - - @property - def python(self) -> Literal[None]: - return None - - @property - def abi(self) -> str: - return f'abi{self.abi_number}' - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and other.abi_number == self.abi_number - - def __hash__(self) -> int: - return hash(str(self)) - - -class InterpreterTag(Tag): - def __init__(self, value: str) -> None: - parts = value.split('-') - if len(parts) < 2: - raise ValueError( - 'Invalid PEP 3149 interpreter tag, expected at ' - f'least 2 parts but got {len(parts)}' - ) - - # On Windows, file extensions look like `.cp311-win_amd64.pyd`, so the - # implementation part (`cpython-`) is different from Linux. Handle that here: - if parts[0].startswith('cp3'): - parts.insert(0, 'cpython') - parts[1] = parts[1][2:] # strip 'cp' - - self._implementation = parts[0] - self._interpreter_version = parts[1] - self._additional_information = parts[2:] - - if self.implementation != 'cpython' and not self.implementation.startswith('pypy'): - raise NotImplementedError(f'Unknown Python implementation: {self.implementation}.') - - @property - def implementation(self) -> str: - return self._implementation - - @property - def interpreter_version(self) -> str: - return self._interpreter_version - - @property - def additional_information(self) -> Sequence[str]: - return tuple(self._additional_information) +import os +import platform +import sys +import sysconfig + +from typing import Optional, Union + + +# https://peps.python.org/pep-0425/#python-tag +INTERPRETERS = { + 'python': 'py', + 'cpython': 'cp', + 'pypy': 'pp', + 'ironpython': 'ip', + 'jython': 'jy', +} + + +_32_BIT_INTERPRETER = sys.maxsize <= 2**32 + + +def get_interpreter_tag() -> str: + name = sys.implementation.name + name = INTERPRETERS.get(name, name) + version = sys.version_info + return f'{name}{version[0]}{version[1]}' + + +def _get_config_var(name: str, default: Union[str, int, None] = None) -> Union[str, int, None]: + value = sysconfig.get_config_var(name) + if value is None: + return default + return value + + +def _get_cpython_abi() -> str: + version = sys.version_info + debug = pymalloc = '' + if _get_config_var('Py_DEBUG', hasattr(sys, 'gettotalrefcount')): + debug = 'd' + if version < (3, 8) and _get_config_var('WITH_PYMALLOC', True): + pymalloc = 'm' + return f'cp{version[0]}{version[1]}{debug}{pymalloc}' + + +def get_abi_tag() -> str: + # The best solution to obtain the Python ABI is to parse the + # $SOABI or $EXT_SUFFIX sysconfig variables as defined in PEP-314. + + # PyPy reports a $SOABI that does not agree with $EXT_SUFFIX. + # Using $EXT_SUFFIX will not break when PyPy will fix this. + # See https://foss.heptapod.net/pypy/pypy/-/issues/3816 and + # https://github.com/pypa/packaging/pull/607. + try: + empty, abi, ext = str(sysconfig.get_config_var('EXT_SUFFIX')).split('.') + except ValueError: + # CPython <= 3.8.7 on Windows does not implement PEP3149 and + # uses '.pyd' as $EXT_SUFFIX, which does not allow to extract + # the interpreter ABI. Check that the fallback is not hit for + # any other Python implementation. + if sys.implementation.name != 'cpython': + raise NotImplementedError + return _get_cpython_abi() + + # The packaging module initially based his understanding of the + # $SOABI variable on the inconsistent value reported by PyPy, and + # did not strip architecture information from it. Therefore the + # ABI tag for later Python implementations (all the ones not + # explicitly handled below) contains architecture information too. + # Unfortunately, fixing this now would break compatibility. + + if abi.startswith('cpython'): + abi = 'cp' + abi.split('-')[1] + elif abi.startswith('cp'): + abi = abi.split('-')[0] + elif abi.startswith('pypy'): + abi = '_'.join(abi.split('-')[:2]) + + return abi.replace('.', '_').replace('-', '_') + + +def _get_macosx_platform_tag() -> str: + ver, x, arch = platform.mac_ver() + + # Override the macOS version if one is provided via the + # MACOS_DEPLOYMENT_TARGET environment variable. + try: + version = tuple(map(int, os.environ.get('MACOS_DEPLOYMENT_TARGET', '').split('.')))[:2] + except ValueError: + version = tuple(map(int, ver.split('.')))[:2] + + # Python built with older macOS SDK on macOS 11, reports an + # unexising macOS 10.16 version instead of the real version. + # + # The packaging module introduced a workaround + # https://github.com/pypa/packaging/commit/67c4a2820c549070bbfc4bfbf5e2a250075048da + # + # This results in packaging versions up to 21.3 generating + # platform tags like "macosx_10_16_x86_64" and later versions + # generating "macosx_11_0_x86_64". Using latter would be more + # correct but prevents the resulting wheel from being installed on + # systems using packaging 21.3 or earlier (pip 22.3 or earlier). + # + # Fortunately packaging versions carrying the workaround still + # accepts "macosx_11_0_x86_64" as a compatible platform tag. We + # can therefore ignore the issue and generate the slightly + # incorrect tag. + + major, minor = version + + if major >= 11: + # For macOS reelases up to 10.15, the major version number is + # actually part of the OS name and the minor version is the + # actual OS release. Starting with macOS 11, the major + # version number is the OS release and the minor version is + # the patch level. Reset the patch level to zero. + minor = 0 + + if _32_BIT_INTERPRETER: + # 32-bit Python running on a 64-bit kernel. + if arch == 'ppc64': + arch = 'ppc' + if arch == 'x86_64': + arch = 'i386' + + return f'macosx_{major}_{minor}_{arch}' + + +def get_platform_tag() -> str: + platform = sysconfig.get_platform() + if platform.startswith('macosx'): + return _get_macosx_platform_tag() + if _32_BIT_INTERPRETER: + # 32-bit Python running on a 64-bit kernel. + if platform == 'linux-x86_64': + return 'linux_i686' + if platform == 'linux-aarch64': + return 'linux_armv7l' + return platform.replace('-', '_') + + +class Tag: + def __init__(self, interpreter: Optional[str] = None, abi: Optional[str] = None, platform: Optional[str] = None): + self.interpreter = interpreter or get_interpreter_tag() + self.abi = abi or get_abi_tag() + self.platform = platform or get_platform_tag() def __str__(self) -> str: - return '-'.join(( - self.implementation, - self.interpreter_version, - *self.additional_information, - )) - - @property - def python(self) -> str: - if self.implementation == 'cpython': - # The Python tag for CPython does not seem to include the flags suffixes. - return f'cp{self.interpreter_version}'.rstrip('dmu') - elif self.implementation.startswith('pypy'): - return f'pp{self.implementation[4:]}' - # XXX: FYI older PyPy version use the following model - # pp{self.implementation[4]}{self.interpreter_version[2:]} - raise ValueError(f'Unknown implementation: {self.implementation}') - - @property - def abi(self) -> str: - if self.implementation == 'cpython': - return f'cp{self.interpreter_version}' - elif self.implementation.startswith('pypy'): - return f'{self.implementation}_{self.interpreter_version}' - raise ValueError(f'Unknown implementation: {self.implementation}') - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, self.__class__) - and other.implementation == self.implementation - and other.interpreter_version == self.interpreter_version - ) - - def __hash__(self) -> int: - return hash(str(self)) + return f'{self.interpreter}-{self.abi}-{self.platform}' diff --git a/tests/conftest.py b/tests/conftest.py index edac18edb..f101fef3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import os import os.path import pathlib +import re import shutil import tempfile @@ -15,6 +16,16 @@ import mesonpy +def adjust_packaging_platform_tag(platform: str) -> str: + # The packaging module generates overly specific platforms tags on + # Linux. The platforms tags on Linux evolved over time. + # meson-python uses more relaxed platform tags to maintain + # compatibility with old wheel installation tools. The relaxed + # platform tags match the ones generated by the wheel package. + # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ + return re.sub(r'^(many|musl)linux(1|2010|2014|_\d+_\d+)_(.*)$', r'linux_\3', platform) + + package_dir = pathlib.Path(__file__).parent / 'packages' diff --git a/tests/test_tags.py b/tests/test_tags.py index 7b7fdd57a..07ba555ec 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -1,75 +1,100 @@ # SPDX-License-Identifier: MIT +import os +import pathlib +import platform import re -import sys +import sysconfig +from collections import defaultdict + +import packaging.tags import pytest +import mesonpy import mesonpy._tags +from .conftest import adjust_packaging_platform_tag + + +# Test against the wheel tag generated by packaging module. +tag = next(packaging.tags.sys_tags()) +ABI = tag.abi +INTERPRETER = tag.interpreter +PLATFORM = adjust_packaging_platform_tag(tag.platform) + +SUFFIX = sysconfig.get_config_var('EXT_SUFFIX') +ABI3SUFFIX = next((x for x in mesonpy._EXTENSION_SUFFIXES if '.abi3.' in x), None) + + +def test_wheel_tag(): + str(mesonpy._tags.Tag()) == f'{INTERPRETER}-{ABI}-{PLATFORM}' + str(mesonpy._tags.Tag(abi='abi3')) == f'{INTERPRETER}-abi3-{PLATFORM}' + + +@pytest.mark.skipif(platform.system() != 'Darwin', reason='macOS specific test') +def test_macos_platform_tag(monkeypatch): + for minor in range(9, 16): + monkeypatch.setenv('MACOS_DEPLOYMENT_TARGET', f'10.{minor}') + assert next(packaging.tags.mac_platforms((10, minor))) == mesonpy._tags.get_platform_tag() + for major in range(11, 20): + for minor in range(3): + monkeypatch.setenv('MACOS_DEPLOYMENT_TARGET', f'{major}.{minor}') + assert next(packaging.tags.mac_platforms((major, minor))) == mesonpy._tags.get_platform_tag() + + +def wheel_builder_test_factory(monkeypatch, content): + files = defaultdict(list) + files.update({key: [(pathlib.Path(x), os.path.join('build', x)) for x in value] for key, value in content.items()}) + monkeypatch.setattr(mesonpy._WheelBuilder, '_wheel_files', files) + return mesonpy._WheelBuilder(None, None, pathlib.Path(), pathlib.Path(), pathlib.Path(), pathlib.Path(), pathlib.Path()) + + +def test_tag_empty_wheel(monkeypatch): + builder = wheel_builder_test_factory(monkeypatch, {}) + assert str(builder.tag) == 'py3-none-any' + + +def test_tag_purelib_wheel(monkeypatch): + builder = wheel_builder_test_factory(monkeypatch, { + 'purelib': ['pure.py'], + }) + assert str(builder.tag) == 'py3-none-any' + + +def test_tag_platlib_wheel(monkeypatch): + builder = wheel_builder_test_factory(monkeypatch, { + 'platlib': [f'extension{SUFFIX}'], + }) + assert str(builder.tag) == f'{INTERPRETER}-{ABI}-{PLATFORM}' + + +@pytest.mark.skipif(not ABI3SUFFIX, reason='Stable ABI not supported by Python interpreter') +def test_tag_stable_abi(monkeypatch): + builder = wheel_builder_test_factory(monkeypatch, { + 'platlib': [f'extension{ABI3SUFFIX}'], + }) + assert str(builder.tag) == f'{INTERPRETER}-abi3-{PLATFORM}' + + +@pytest.mark.skipif(not ABI3SUFFIX, reason='Stable ABI not supported by Python interpreter') +def test_tag_mixed_abi(monkeypatch): + builder = wheel_builder_test_factory(monkeypatch, { + 'platlib': [f'extension{ABI3SUFFIX}', f'another{SUFFIX}'], + }) + assert str(builder.tag) == f'{INTERPRETER}-{ABI}-{PLATFORM}' + + +@pytest.mark.skipif(platform.system() != 'Darwin', reason='macOS specific test') +def test_tag_macos_build_target(monkeypatch): + monkeypatch.setenv('MACOS_BUILD_TARGET', '12.0') + builder = wheel_builder_test_factory(monkeypatch, { + 'platlib': [f'extension{SUFFIX}'], + }) + assert builder.tag.platform == re.sub(r'\d+\.\d+', '12.0', PLATFORM) -INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' - - -@pytest.mark.parametrize( - ('value', 'number', 'abi', 'python'), - [ - ('abi3', 3, 'abi3', None), - ('abi4', 4, 'abi4', None), - ] -) -def test_stable_abi_tag(value, number, abi, python): - tag = mesonpy._tags.StableABITag(value) - assert str(tag) == value - assert tag.abi_number == number - assert tag.abi == abi - assert tag.python == python - assert tag == mesonpy._tags.StableABITag(value) - - -def test_stable_abi_tag_invalid(): - with pytest.raises(ValueError, match=re.escape( - r'Invalid PEP 3149 stable ABI tag, expecting pattern `^abi(?P[0-9]+)$`' - )): - mesonpy._tags.StableABITag('invalid') - - -@pytest.mark.parametrize( - ('value', 'implementation', 'version', 'additional', 'abi', 'python'), - [ - ('cpython-37-x86_64-linux-gnu', 'cpython', '37', ('x86_64', 'linux', 'gnu'), 'cp37', 'cp37'), - ('cpython-310-x86_64-linux-gnu', 'cpython', '310', ('x86_64', 'linux', 'gnu'), 'cp310', 'cp310'), - ('cpython-310', 'cpython', '310', (), 'cp310', 'cp310'), - ('cp311-win_amd64', 'cpython', '311', ('win_amd64', ), 'cp311', 'cp311'), - ('cpython-311-win_amd64', 'cpython', '311', ('win_amd64', ), 'cp311', 'cp311'), - ('cpython-310-special', 'cpython', '310', ('special',), 'cp310', 'cp310'), - ('cpython-310-x86_64-linux-gnu', 'cpython', '310', ('x86_64', 'linux', 'gnu'), 'cp310', 'cp310'), - ('pypy39-pp73-x86_64-linux-gnu', 'pypy39', 'pp73', ('x86_64', 'linux', 'gnu'), 'pypy39_pp73', 'pp39'), - ('pypy39-pp73-win_amd64', 'pypy39', 'pp73', ('win_amd64', ), 'pypy39_pp73', 'pp39'), - ('pypy38-pp73-darwin', 'pypy38', 'pp73', ('darwin', ), 'pypy38_pp73', 'pp38'), - ] -) -def test_interpreter_tag(value, implementation, version, additional, abi, python): - tag = mesonpy._tags.InterpreterTag(value) - if not value.startswith('cp311'): - # Avoid testing the workaround for the invalid Windows tag - assert str(tag) == value - - assert tag.implementation == implementation - assert tag.interpreter_version == version - assert tag.additional_information == additional - assert tag.abi == abi - assert tag.python == python - assert tag == mesonpy._tags.InterpreterTag(value) - - -@pytest.mark.parametrize( - ('value', 'msg'), - [ - ('', 'Invalid PEP 3149 interpreter tag, expected at least 2 parts but got 1'), - ('invalid', 'Invalid PEP 3149 interpreter tag, expected at least 2 parts but got 1'), - ] -) -def test_interpreter_tag_invalid(value, msg): - with pytest.raises(ValueError, match=msg): - mesonpy._tags.InterpreterTag(value) + monkeypatch.setenv('MACOS_BUILD_TARGET', '10.9') + builder = wheel_builder_test_factory(monkeypatch, { + 'platlib': [f'extension{SUFFIX}'], + }) + assert builder.tag.platform == re.sub(r'\d+\.\d+', '10.9', PLATFORM) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index e0c7c667f..dcbec60e7 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -8,44 +8,23 @@ import sysconfig import textwrap +import packaging.tags import pytest import wheel.wheelfile -import mesonpy._elf +import mesonpy + +from .conftest import adjust_packaging_platform_tag EXT_SUFFIX = sysconfig.get_config_var('EXT_SUFFIX') INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' - -if platform.python_implementation() == 'CPython': - INTERPRETER_TAG = f'cp{INTERPRETER_VERSION}' - PYTHON_TAG = INTERPRETER_TAG - # Py_UNICODE_SIZE has been a runtime option since Python 3.3, - # so the u suffix no longer exists - if sysconfig.get_config_var('Py_DEBUG'): - INTERPRETER_TAG += 'd' - # https://github.com/pypa/packaging/blob/5984e3b25f4fdee64aad20e98668c402f7ed5041/packaging/tags.py#L147-L150 - if sys.version_info < (3, 8): - pymalloc = sysconfig.get_config_var('WITH_PYMALLOC') - if pymalloc or pymalloc is None: # none is the default value, which is enable - INTERPRETER_TAG += 'm' -elif platform.python_implementation() == 'PyPy': - INTERPRETER_TAG = sysconfig.get_config_var('SOABI').replace('-', '_') - PYTHON_TAG = f'pp{INTERPRETER_VERSION}' -else: - raise NotImplementedError(f'Unknown implementation: {platform.python_implementation()}') - -platform_ = sysconfig.get_platform() -if platform.system() == 'Darwin': - parts = platform_.split('-') - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - parts[1] = parts[1].split('.')[0] - if parts[1] in ('11', '12'): - parts[1] += '.0' - platform_ = '-'.join(parts) -PLATFORM_TAG = platform_.replace('-', '_').replace('.', '_') +# Test against the wheel tag generated by packaging module. +tag = next(packaging.tags.sys_tags()) +ABI = tag.abi +INTERPRETER = tag.interpreter +PLATFORM = adjust_packaging_platform_tag(tag.platform) if platform.system() == 'Linux': SHARED_LIB_EXT = 'so' @@ -72,7 +51,6 @@ def wheel_filename(artifact): win_py37 = os.name == 'nt' and sys.version_info < (3, 8) -@pytest.mark.skipif(win_py37, reason='An issue with missing file extension') def test_scipy_like(wheel_scipy_like): # This test is meant to exercise features commonly needed by a regular # Python package for scientific computing or data science: @@ -104,9 +82,9 @@ def test_scipy_like(wheel_scipy_like): assert wheel_contents(artifact) == expecting name = artifact.parsed_filename - assert name.group('pyver') == PYTHON_TAG - assert name.group('abi') == INTERPRETER_TAG - assert name.group('plat') == PLATFORM_TAG + assert name.group('pyver') == INTERPRETER + assert name.group('abi') == ABI + assert name.group('plat') == PLATFORM # Extra checks to doubly-ensure that there are no issues with erroneously # considering a package with an extension module as pure @@ -130,7 +108,6 @@ def test_contents(package_library, wheel_library): assert re.match(regex, name), f'`{name}` does not match `{regex}`' -@pytest.mark.skipif(win_py37, reason='An issue with missing file extension') def test_purelib_and_platlib(wheel_purelib_and_platlib): artifact = wheel.wheelfile.WheelFile(wheel_purelib_and_platlib) @@ -206,19 +183,18 @@ def test_executable_bit(wheel_executable_bit): assert not executable_bit, f'{info.filename} should not have the executable bit set!' -@pytest.mark.skipif(win_py37, reason='An issue with missing file extension') def test_detect_wheel_tag_module(wheel_purelib_and_platlib): name = wheel.wheelfile.WheelFile(wheel_purelib_and_platlib).parsed_filename - assert name.group('pyver') == PYTHON_TAG - assert name.group('abi') == INTERPRETER_TAG - assert name.group('plat') == PLATFORM_TAG + assert name.group('pyver') == INTERPRETER + assert name.group('abi') == ABI + assert name.group('plat') == PLATFORM def test_detect_wheel_tag_script(wheel_executable): name = wheel.wheelfile.WheelFile(wheel_executable).parsed_filename assert name.group('pyver') == 'py3' assert name.group('abi') == 'none' - assert name.group('plat') == PLATFORM_TAG + assert name.group('plat') == PLATFORM @pytest.mark.skipif(platform.system() != 'Linux', reason='Unsupported on this platform for now') From dcc1b7dfe31a5f88faa5af99d5f4f53b6b191d79 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sat, 12 Nov 2022 21:16:54 +0100 Subject: [PATCH 02/12] BUG: fix extension filename suffix for Cygwin Python --- mesonpy/__init__.py | 12 +++++++----- mesonpy/_tags.py | 4 +++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 1e5d1591c..d5cab7108 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -104,7 +104,7 @@ def _init_colors() -> Dict[str, str]: _EXTENSION_SUFFIXES = frozenset(importlib.machinery.EXTENSION_SUFFIXES) -_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P[^.]+)\.)?(?:so|pyd)$') +_EXTENSION_SUFFIX_REGEX = re.compile(r'^\.(?:(?P[^.]+)\.)?(?:so|pyd|dll)$') assert all(re.match(_EXTENSION_SUFFIX_REGEX, x) for x in _EXTENSION_SUFFIXES) @@ -289,20 +289,22 @@ def _stable_abi(self) -> Optional[str]: """Determine stabe ABI compatibility. Examine all files installed in {platlib} that look like - extension modules (extension .pyd on Windows and .so on other - platforms) and, if they all share the same PEP 3149 filename - stable ABI tag, return it. + extension modules (extension .pyd on Windows, .dll on Cygwin, + and .so on other platforms) and, if they all share the same + PEP 3149 filename stable ABI tag, return it. All files that look like extension modules are verified to have a file name compatibel with what is expected by the Python interpreter. An exception is raised otherwise. Other files are ignored. + """ + soext = sorted(_EXTENSION_SUFFIXES, key=len)[0] abis = [] for path, src in self._wheel_files['platlib']: - if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so': + if path.suffix == soext: match = re.match(r'^[^.]+(.*)$', path.name) assert match is not None suffix = match.group(1) diff --git a/mesonpy/_tags.py b/mesonpy/_tags.py index 81725e0d6..355a81152 100644 --- a/mesonpy/_tags.py +++ b/mesonpy/_tags.py @@ -77,6 +77,8 @@ def get_abi_tag() -> str: abi = abi.split('-')[0] elif abi.startswith('pypy'): abi = '_'.join(abi.split('-')[:2]) + elif abi.startswith('graalpy'): + abi = '_'.join(abi.split('-')[:3]) return abi.replace('.', '_').replace('-', '_') @@ -138,7 +140,7 @@ def get_platform_tag() -> str: return 'linux_i686' if platform == 'linux-aarch64': return 'linux_armv7l' - return platform.replace('-', '_') + return platform.replace('-', '_').replace('.', '_') class Tag: From b7e693d0b0a3892a727542be16b5fc5c653aa11f Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Wed, 2 Nov 2022 15:12:32 +0100 Subject: [PATCH 03/12] TST: remove redundant and flaky test Looking for the string 'any' in the wheel filenames triggers when the platform tag contains the string 'manylinux'. The ABI and platform tags are anyhow verified a few lines above. --- tests/test_wheel.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index dcbec60e7..3e9203b09 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -86,11 +86,6 @@ def test_scipy_like(wheel_scipy_like): assert name.group('abi') == ABI assert name.group('plat') == PLATFORM - # Extra checks to doubly-ensure that there are no issues with erroneously - # considering a package with an extension module as pure - assert 'none' not in wheel_filename(artifact) - assert 'any' not in wheel_filename(artifact) - @pytest.mark.skipif(platform.system() != 'Linux', reason='Needs library vendoring, only implemented in POSIX') def test_contents(package_library, wheel_library): From 57c9e7d214cb20543e0b548612339dd9447af241 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 13 Nov 2022 12:04:19 +0100 Subject: [PATCH 04/12] TST: simplify checks for executable mode Also skip the tests on Cygwin. --- tests/test_sdist.py | 29 ++++++----------------------- tests/test_wheel.py | 10 +++------- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/tests/test_sdist.py b/tests/test_sdist.py index aea6e2ab7..fdda7218d 100644 --- a/tests/test_sdist.py +++ b/tests/test_sdist.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: MIT import os +import stat +import sys import tarfile import textwrap @@ -70,39 +72,20 @@ def bar(): assert read_data == new_data.encode() -@pytest.mark.skipif(os.name == 'nt', reason='Executable bit does not exist on Windows') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='Platform does not support executable bit') def test_executable_bit(sdist_executable_bit): sdist = tarfile.open(sdist_executable_bit, 'r:gz') expected = { - 'executable_bit-1.0.0/PKG-INFO': None, + 'executable_bit-1.0.0/PKG-INFO': False, 'executable_bit-1.0.0/example-script.py': True, 'executable_bit-1.0.0/example.c': False, 'executable_bit-1.0.0/executable_module.py': True, 'executable_bit-1.0.0/meson.build': False, 'executable_bit-1.0.0/pyproject.toml': False, } - assert set(tar.name for tar in sdist.getmembers()) == set(expected.keys()) - - def has_execbit(mode): - # Note: File perms are in octal, but Python returns it in int - # We check multiple modes, because Docker may set group permissions to - # match owner permissions - modes_execbit = 0o755, 0o775 - modes_nonexecbit = 0o644, 0o664 - if mode in modes_execbit: - return True - elif mode in modes_nonexecbit: - return False - else: - raise RuntimeError(f'Unknown file permissions mode: {mode}') - - for name, mode in set((tar.name, tar.mode) for tar in sdist.getmembers()): - if 'PKG-INFO' in name: - # We match the executable bit on everything but PKG-INFO (we create - # this ourselves) - continue - assert has_execbit(mode) == expected[name], f'Wrong mode for {name}: {mode}' + for member in sdist.getmembers(): + assert bool(member.mode & stat.S_IXUSR) == expected[member.name] def test_generated_files(sdist_generated_files): diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 3e9203b09..eb9571dd5 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -3,6 +3,7 @@ import os import platform import re +import stat import subprocess import sys import sysconfig @@ -156,7 +157,7 @@ def test_contents_license_file(wheel_license_file): assert artifact.read('license_file-1.0.0.dist-info/LICENSE.custom').rstrip() == b'Hello!' -@pytest.mark.skipif(os.name == 'nt', reason='Executable bit does not exist on Windows') +@pytest.mark.skipif(sys.platform in {'win32', 'cygwin'}, reason='Platform does not support executable bit') def test_executable_bit(wheel_executable_bit): artifact = wheel.wheelfile.WheelFile(wheel_executable_bit) @@ -168,14 +169,9 @@ def test_executable_bit(wheel_executable_bit): 'executable_bit-1.0.0.data/scripts/example-script', 'executable_bit-1.0.0.data/data/bin/example-script', } - for info in artifact.infolist(): mode = (info.external_attr >> 16) & 0o777 - executable_bit = bool(mode & 0b001_000_000) # owner execute - if info.filename in executable_files: - assert executable_bit, f'{info.filename} should have the executable bit set!' - else: - assert not executable_bit, f'{info.filename} should not have the executable bit set!' + assert bool(mode & stat.S_IXUSR) == (info.filename in executable_files) def test_detect_wheel_tag_module(wheel_purelib_and_platlib): From acf26e404c357868b3a1c654b309dde66138f014 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sat, 12 Nov 2022 23:25:07 +0100 Subject: [PATCH 05/12] TST: simplify Remove (wrong!) cross platform setup for test run only on Linux. --- tests/test_wheel.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index eb9571dd5..56b6ea9f8 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -27,15 +27,6 @@ INTERPRETER = tag.interpreter PLATFORM = adjust_packaging_platform_tag(tag.platform) -if platform.system() == 'Linux': - SHARED_LIB_EXT = 'so' -elif platform.system() == 'Darwin': - SHARED_LIB_EXT = 'dylib' -elif platform.system() == 'Windows': - SHARED_LIB_EXT = 'pyd' -else: - raise NotImplementedError(f'Unknown system: {platform.system()}') - def wheel_contents(artifact): # Sometimes directories have entries, sometimes not, so we filter them out. @@ -93,15 +84,15 @@ def test_contents(package_library, wheel_library): artifact = wheel.wheelfile.WheelFile(wheel_library) for name, regex in zip(sorted(wheel_contents(artifact)), [ - re.escape(f'.library.mesonpy.libs/libexample.{SHARED_LIB_EXT}'), + re.escape('.library.mesonpy.libs/libexample.so'), re.escape('library-1.0.0.data/headers/examplelib.h'), re.escape('library-1.0.0.data/scripts/example'), re.escape('library-1.0.0.dist-info/METADATA'), re.escape('library-1.0.0.dist-info/RECORD'), re.escape('library-1.0.0.dist-info/WHEEL'), - rf'library\.libs/libexample.*\.{SHARED_LIB_EXT}', + re.escape('library.libs/libexample.so'), ]): - assert re.match(regex, name), f'`{name}` does not match `{regex}`' + assert re.match(regex, name), f'{name!r} does not match {regex!r}' def test_purelib_and_platlib(wheel_purelib_and_platlib): From 578c2c4d30114b0f9bdd23477717c6e6a1426fae Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 13 Nov 2022 00:55:24 +0100 Subject: [PATCH 06/12] TST: drop dependency on GitPython GitPython does not bring any particularly useful abstraction, is one more dependency, and makes debugging failures in the test setup harder. --- pyproject.toml | 1 - tests/conftest.py | 29 +++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8e2faa38b..698e3fbb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ test = [ 'pytest', 'pytest-cov', 'pytest-mock', - 'GitPython', 'auditwheel', 'Cython', 'pyproject-metadata>=0.6.1', diff --git a/tests/conftest.py b/tests/conftest.py index f101fef3d..be36e33bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,11 +6,11 @@ import pathlib import re import shutil +import subprocess import tempfile from venv import EnvBuilder -import git import pytest import mesonpy @@ -42,24 +42,21 @@ def cd_package(package): @contextlib.contextmanager def in_git_repo_context(path=os.path.curdir): - path = pathlib.Path(path) - assert path.absolute().relative_to(package_dir) - shutil.rmtree(path / '.git', ignore_errors=True) + # Resist the tentation of using pathlib.Path here: it is not + # supporded by subprocess in Python 3.7. + path = os.path.abspath(path) + shutil.rmtree(os.path.join(path, '.git'), ignore_errors=True) try: - handler = git.Git(path) - handler.init() - handler.config('commit.gpgsign', 'false') - handler.config('user.name', 'Example') - handler.config('user.email', 'example@example.com') - handler.add('*') - handler.commit('--allow-empty-message', '-m', '') - handler.tag('-a', '-m', '', '1.0.0') + subprocess.check_call(['git', 'init', '-b', 'main', path]) + subprocess.check_call(['git', 'config', 'user.email', 'author@example.com'], cwd=path) + subprocess.check_call(['git', 'config', 'user.name', 'A U Thor'], cwd=path) + subprocess.check_call(['git', 'add', '*'], cwd=path) + subprocess.check_call(['git', 'commit', '-q', '-m', 'Test'], cwd=path) yield finally: - try: - shutil.rmtree(path / '.git') - except PermissionError: - pass + # PermissionError raised on Windows. + with contextlib.suppress(PermissionError): + shutil.rmtree(os.path.join(path, '.git')) @pytest.fixture(scope='session') From b58a27392b0dcb5713e27eb49cce32e2fb520ca4 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 13 Nov 2022 16:22:43 +0100 Subject: [PATCH 07/12] TST: account for Cygwin also having .dll.a files --- tests/test_wheel.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 56b6ea9f8..3717a0643 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -19,6 +19,7 @@ EXT_SUFFIX = sysconfig.get_config_var('EXT_SUFFIX') +EXT_IMP_SUFFIX = re.sub(r'.pyd$', '.dll', EXT_SUFFIX) + '.a' INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' # Test against the wheel tag generated by packaging module. @@ -63,15 +64,15 @@ def test_scipy_like(wheel_scipy_like): 'mypkg/submod/__init__.py', 'mypkg/submod/unknown_filetype.npq', } - if os.name == 'nt': - # Currently Meson is installing `.dll.a` (import libraries) next to - # `.pyd` extension modules. Those are very small, so it's not a major - # issue - just sloppy. For now, ensure we don't fail on those - actual_files = wheel_contents(artifact) - for item in expecting: - assert item in actual_files - else: - assert wheel_contents(artifact) == expecting + if sys.platform in {'win32', 'cygwin'}: + # Currently Meson is installing .dll.a (import libraries) next + # to .pyd extension modules. Those are very small, so it's not + # a major issue - just sloppy. Ensure we don't fail on those. + expecting.update({ + f'mypkg/extmod{EXT_IMP_SUFFIX}', + f'mypkg/cy_extmod{EXT_IMP_SUFFIX}', + }) + assert wheel_contents(artifact) == expecting name = artifact.parsed_filename assert name.group('pyver') == INTERPRETER @@ -105,8 +106,13 @@ def test_purelib_and_platlib(wheel_purelib_and_platlib): 'purelib_and_platlib-1.0.0.dist-info/RECORD', 'purelib_and_platlib-1.0.0.dist-info/WHEEL', } - if platform.system() == 'Windows': - expecting.add('plat{}'.format(EXT_SUFFIX.replace('pyd', 'dll.a'))) + if sys.platform in {'win32', 'cygwin'}: + # Currently Meson is installing .dll.a (import libraries) next + # to .pyd extension modules. Those are very small, so it's not + # a major issue - just sloppy. Ensure we don't fail on those. + expecting.update({ + f'plat{EXT_IMP_SUFFIX}' + }) assert wheel_contents(artifact) == expecting From 6ce635ad504980719913c408bd990aabe7bb5b98 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sat, 12 Nov 2022 14:48:39 +0100 Subject: [PATCH 08/12] CI: add Pyston on Ubuntu 20.04 configuration --- .github/workflows/tests.yml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68f3c81a3..72fa3ac82 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,3 +60,41 @@ jobs: flags: tests env_vars: PYTHON name: ${{ matrix.python }} + + pyston: + runs-on: ubuntu-20.04 + env: + FORCE_COLOR: true + strategy: + fail-fast: false + matrix: + python: + - '3.8' + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install pyston + run: | + wget https://github.com/pyston/pyston/releases/download/pyston_2.3.5/pyston_2.3.5_20.04_amd64.deb + sudo apt install $(pwd)/pyston_2.3.5_20.04_amd64.deb + + - name: Install + run: pyston -m pip --disable-pip-version-check install .[test] + + - name: Run tests + run: >- + pyston -m pytest --showlocals -vv --cov + --cov-config setup.cfg + --cov-report=xml:coverage-pyston.xml + + - name: Send coverage report + uses: codecov/codecov-action@v1 + if: ${{ always() }} + env: + PYTHON: pyston + with: + flags: tests + env_vars: PYTHON + name: pyston From d582d46f4f2f7e8351d67cbb2e2f146525138ff4 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sat, 12 Nov 2022 15:18:12 +0100 Subject: [PATCH 09/12] CI: add Homebrew Python on macOS configuration --- .github/workflows/tests.yml | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 72fa3ac82..0ed8059aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,3 +98,45 @@ jobs: flags: tests env_vars: PYTHON name: pyston + + homebrew: + runs-on: macos-latest + env: + FORCE_COLOR: true + strategy: + fail-fast: false + matrix: + python: + - '3.7' + - '3.8' + - '3.9' + - '3.10' + - '3.11' + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Homebrew Python + run: | + brew install --overwrite python@${{ matrix.python }} + echo /usr/local/opt/python@${{ matrix.python }}/libexec/bin/ >> $GITHUB_PATH + + - name: Install + run: python -m pip --disable-pip-version-check install .[test] + + - name: Run tests + run: >- + python -m pytest --showlocals -vv --cov + --cov-config setup.cfg + --cov-report=xml:coverage-homebrew-${{ matrix.python }}.xml + + - name: Send coverage report + uses: codecov/codecov-action@v1 + if: ${{ always() }} + env: + PYTHON: homebrew-${{ matrix.python }} + with: + flags: tests + env_vars: PYTHON + name: homebrew-${{ matrix.python }} From d893887a5f169562c4d20235a61c15c5032d3882 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sat, 12 Nov 2022 18:34:38 +0100 Subject: [PATCH 10/12] CI: actually run with Cygwin Python Cygwin was installed but the regular Python interpreter was actually used to run the test. Fix this. Also merge Cygwin tests setup definition with the one for other environments. The Python distributed by Cygwin is broken and requires some peculiar maneuvers to make it suitable to create virtual environments. Address that in the CI configuration. --- .github/workflows/tests-cygwin.yml | 54 ------------------- .github/workflows/tests.yml | 83 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/tests-cygwin.yml diff --git a/.github/workflows/tests-cygwin.yml b/.github/workflows/tests-cygwin.yml deleted file mode 100644 index 17e33bcfa..000000000 --- a/.github/workflows/tests-cygwin.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: tests-cygwin - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - pytest: - runs-on: windows-latest - env: - FORCE_COLOR: true - strategy: - fail-fast: false - matrix: - python: - - '3.10' - - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up target Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - - name: Setup Cygwin - uses: cygwin/cygwin-install-action@v2 - - - name: Install nox - run: | - python -m pip install nox - nox --version - - - name: Run tests - run: nox -s test-${{ matrix.python }} - - - name: Send coverage report - uses: codecov/codecov-action@v1 - if: ${{ always() }} - env: - PYTHON: cygwin-${{ matrix.python }} - with: - flags: tests - env_vars: PYTHON - name: cygwin-${{ matrix.python }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0ed8059aa..33eb95ab1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,6 +61,89 @@ jobs: env_vars: PYTHON name: ${{ matrix.python }} + cygwin: + runs-on: windows-latest + env: + FORCE_COLOR: true + strategy: + fail-fast: false + matrix: + python: + - '3.9' + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Cygwin + uses: cygwin/cygwin-install-action@v2 + with: + packages: >- + python39 + python39-devel + python39-pip + python39-setuptools + cmake + gcc-core + gcc-g++ + git + make + ninja + + - name: Fix git dubious ownership + # This addresses the "fatal: detected dubious ownership in + # repository" and "fatal: not in a git directory" errors + # encountered when trying to run Cygwin git in a directory not + # owned by the current user. This happens when the tests run + # Cygwin git in a directory outside the Cygwin filesystem. + run: git config --global --add safe.directory '*' + shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0} + + - name: Get pip cache path + id: pip-cache-path + run: echo "path=$(cygpath -w $(python -m pip cache dir))" >> $GITHUB_OUTPUT + shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0} + + - name: Restore cache + # Cygwin Python cannot use binary wheels from PyPI. Building + # some dependencies takes considerable time. Caching the built + # wheels speeds up the CI job quite a bit. + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache-path.outputs.path }} + key: cygwin-pip-${{ github.sha }} + restore-keys: cygwin-pip- + + - name: Install + # Cygwin patches Python's ensurepip module to look for the + # wheels needed to initialize a new virtual environment in + # /usr/share/python-wheels/ but nothing in Cygwin actually + # puts the setuptools and pip wheels there. Fix this. + run: | + mkdir /usr/share/python-wheels/ + pushd /usr/share/python-wheels/ + python -m pip --disable-pip-version-check download setuptools pip + popd + python -m pip --disable-pip-version-check install .[test] + shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0} + + - name: Run tests + run: >- + python -m pytest --showlocals -vv --cov + --cov-config setup.cfg + --cov-report=xml:coverage-${{ matrix.python }}.xml + shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0} + + - name: Send coverage report + uses: codecov/codecov-action@v1 + if: ${{ always() }} + env: + PYTHON: cygwin-${{ matrix.python }} + with: + flags: tests + env_vars: PYTHON + name: cygwin-${{ matrix.python }} + pyston: runs-on: ubuntu-20.04 env: From a03c9e7acc4610b861d496000e66f10e8b369b3c Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 13 Nov 2022 17:29:19 +0100 Subject: [PATCH 11/12] CI: update GitHub Actions and sundry tweaks --- .github/workflows/checks.yml | 12 ++++++------ .github/workflows/ci-sage.yml | 2 +- .github/workflows/tests.yml | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a6ed8f2a5..9135981a1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,15 +17,15 @@ jobs: env: FORCE_COLOR: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.7 - - name: Install nox - run: python -m pip install nox + - name: Install mypy + run: python -m pip --disable-pip-version-check install mypy==0.981 - - name: Run check for type - run: nox -s mypy + - name: Run mypy + run: mypy -p mesonpy diff --git a/.github/workflows/ci-sage.yml b/.github/workflows/ci-sage.yml index 7e1a438b6..53651b931 100644 --- a/.github/workflows/ci-sage.yml +++ b/.github/workflows/ci-sage.yml @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out ${{ env.SPKG }} - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: build/pkgs/${{ env.SPKG }}/src - name: Install prerequisites diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 33eb95ab1..d421568b2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ concurrency: cancel-in-progress: true jobs: - pytest: + test: runs-on: ${{ matrix.os }}-latest env: FORCE_COLOR: true @@ -35,10 +35,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Set up target Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} @@ -73,7 +73,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Cygwin uses: cygwin/cygwin-install-action@v2 @@ -156,7 +156,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install pyston run: | @@ -198,7 +198,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Homebrew Python run: | From 65f1b1c669598a01667031e12785c55ef2ffff50 Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Wed, 16 Nov 2022 01:47:32 +0100 Subject: [PATCH 12/12] CI: patch pip to fix issue with Homebrew Python See https://github.com/pypa/pip/pull/11598. --- .github/workflows/tests.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d421568b2..b9a0b53eb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -205,6 +205,18 @@ jobs: brew install --overwrite python@${{ matrix.python }} echo /usr/local/opt/python@${{ matrix.python }}/libexec/bin/ >> $GITHUB_PATH + - name: Patch pip + # Patch https://github.com/pypa/pip/issues/11539 + run: | + cat >>/usr/local/lib/python${{ matrix.python }}/site-packages/pip/_internal/locations/_sysconfig.py < typing.Tuple[str, str]: + if "venv" in sysconfig.get_scheme_names(): + paths = sysconfig.get_paths(vars={"base": prefix, "platbase": prefix}, scheme="venv") + else: + paths = sysconfig.get_paths(vars={"base": prefix, "platbase": prefix}) + return (paths["purelib"], paths["platlib"]) + EOF + - name: Install run: python -m pip --disable-pip-version-check install .[test]