diff --git a/src/auditwheel/policy/__init__.py b/src/auditwheel/policy/__init__.py index d66c00f3..93cd7399 100644 --- a/src/auditwheel/policy/__init__.py +++ b/src/auditwheel/policy/__init__.py @@ -3,17 +3,20 @@ import json import logging import platform as _platform_module +import re import sys from collections import defaultdict from os.path import abspath, dirname, join from pathlib import Path +from typing import Any, Generator + +from auditwheel.elfutils import filter_undefined_symbols, is_subdir from ..libc import Libc, get_libc from ..musllinux import find_musl_libc, get_musl_version -from .external_references import lddtree_external_references -from .versioned_symbols import versioned_symbols_policy _HERE = Path(__file__).parent +LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$") logger = logging.getLogger(__name__) @@ -97,6 +100,112 @@ def get_priority_by_name(self, name: str) -> int | None: policy = self.get_policy_by_name(name) return None if policy is None else policy["priority"] + def versioned_symbols_policy(self, versioned_symbols: dict[str, set[str]]) -> int: + def policy_is_satisfied( + policy_name: str, policy_sym_vers: dict[str, set[str]] + ) -> bool: + policy_satisfied = True + for name in set(required_vers) & set(policy_sym_vers): + if not required_vers[name].issubset(policy_sym_vers[name]): + for symbol in required_vers[name] - policy_sym_vers[name]: + logger.debug( + "Package requires %s, incompatible with " + "policy %s which requires %s", + symbol, + policy_name, + policy_sym_vers[name], + ) + policy_satisfied = False + return policy_satisfied + + required_vers: dict[str, set[str]] = {} + for symbols in versioned_symbols.values(): + for symbol in symbols: + sym_name, _, _ = symbol.partition("_") + required_vers.setdefault(sym_name, set()).add(symbol) + matching_policies: list[int] = [] + for p in self.policies: + policy_sym_vers = { + sym_name: {sym_name + "_" + version for version in versions} + for sym_name, versions in p["symbol_versions"].items() + } + if policy_is_satisfied(p["name"], policy_sym_vers): + matching_policies.append(p["priority"]) + + if len(matching_policies) == 0: + # the base policy (generic linux) should always match + raise RuntimeError("Internal error") + + return max(matching_policies) + + def lddtree_external_references(self, lddtree: dict, wheel_path: str) -> dict: + # XXX: Document the lddtree structure, or put it in something + # more stable than a big nested dict + def filter_libs( + libs: set[str], whitelist: set[str] + ) -> Generator[str, None, None]: + for lib in libs: + if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]: + # always exclude ELF dynamic linker/loader + # 'ld64.so.2' on s390x + # 'ld64.so.1' on ppc64le + # 'ld-linux*' on other platforms + continue + if LIBPYTHON_RE.match(lib): + # always exclude libpythonXY + continue + if lib in whitelist: + # exclude any libs in the whitelist + continue + yield lib + + def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: + # get all the required external libraries + libs = libs.copy() + reqs = set() + while libs: + lib = libs.pop() + reqs.add(lib) + for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist): + if dep not in reqs: + libs.add(dep) + return reqs + + ret: dict[str, dict[str, Any]] = {} + for p in self.policies: + needed_external_libs: set[str] = set() + blacklist = {} + + if not (p["name"] == "linux" and p["priority"] == 0): + # special-case the generic linux platform here, because it + # doesn't have a whitelist. or, you could say its + # whitelist is the complete set of all libraries. so nothing + # is considered "external" that needs to be copied in. + whitelist = set(p["lib_whitelist"]) + blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"]) + blacklist = {k: p["blacklist"][k] for k in blacklist_libs} + blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist) + needed_external_libs = get_req_external( + set(filter_libs(lddtree["needed"], whitelist)), whitelist + ) + + pol_ext_deps = {} + for lib in needed_external_libs: + if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path): + # we didn't filter libs that resolved via RPATH out + # earlier because we wanted to make sure to pick up + # our elf's indirect dependencies. But now we want to + # filter these ones out, since they're not "external". + logger.debug("RPATH FTW: %s", lib) + continue + pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"] + ret[p["name"]] = { + "libs": pol_ext_deps, + "priority": p["priority"], + "blacklist": blacklist, + } + return ret + def get_arch_name() -> str: machine = _platform_module.machine() @@ -204,7 +313,5 @@ def _load_policy_schema(): __all__ = [ - "lddtree_external_references", - "versioned_symbols_policy", "WheelPolicies", ] diff --git a/src/auditwheel/policy/external_references.py b/src/auditwheel/policy/external_references.py deleted file mode 100644 index 2f833c2a..00000000 --- a/src/auditwheel/policy/external_references.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import logging -import re -from typing import Any, Generator - -from ..elfutils import filter_undefined_symbols, is_subdir -from . import WheelPolicies - -log = logging.getLogger(__name__) -LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$") - - -def lddtree_external_references(wheel_policies: list, lddtree: dict, wheel_path: str) -> dict: - # XXX: Document the lddtree structure, or put it in something - # more stable than a big nested dict - def filter_libs(libs: set[str], whitelist: set[str]) -> Generator[str, None, None]: - for lib in libs: - if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]: - # always exclude ELF dynamic linker/loader - # 'ld64.so.2' on s390x - # 'ld64.so.1' on ppc64le - # 'ld-linux*' on other platforms - continue - if LIBPYTHON_RE.match(lib): - # always exclude libpythonXY - continue - if lib in whitelist: - # exclude any libs in the whitelist - continue - yield lib - - def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]: - # get all the required external libraries - libs = libs.copy() - reqs = set() - while libs: - lib = libs.pop() - reqs.add(lib) - for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist): - if dep not in reqs: - libs.add(dep) - return reqs - - ret: dict[str, dict[str, Any]] = {} - for p in wheel_policies: - needed_external_libs: set[str] = set() - blacklist = {} - - if not (p["name"] == "linux" and p["priority"] == 0): - # special-case the generic linux platform here, because it - # doesn't have a whitelist. or, you could say its - # whitelist is the complete set of all libraries. so nothing - # is considered "external" that needs to be copied in. - whitelist = set(p["lib_whitelist"]) - blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"]) - blacklist = {k: p["blacklist"][k] for k in blacklist_libs} - blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist) - needed_external_libs = get_req_external( - set(filter_libs(lddtree["needed"], whitelist)), whitelist - ) - - pol_ext_deps = {} - for lib in needed_external_libs: - if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path): - # we didn't filter libs that resolved via RPATH out - # earlier because we wanted to make sure to pick up - # our elf's indirect dependencies. But now we want to - # filter these ones out, since they're not "external". - log.debug("RPATH FTW: %s", lib) - continue - pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"] - ret[p["name"]] = { - "libs": pol_ext_deps, - "priority": p["priority"], - "blacklist": blacklist, - } - return ret diff --git a/src/auditwheel/policy/versioned_symbols.py b/src/auditwheel/policy/versioned_symbols.py deleted file mode 100644 index ab90288f..00000000 --- a/src/auditwheel/policy/versioned_symbols.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import logging - -from . import WheelPolicies - -log = logging.getLogger(__name__) - - -def versioned_symbols_policy(wheel_policy: WheelPolicies, versioned_symbols: dict[str, set[str]]) -> int: - def policy_is_satisfied( - policy_name: str, policy_sym_vers: dict[str, set[str]] - ) -> bool: - policy_satisfied = True - for name in set(required_vers) & set(policy_sym_vers): - if not required_vers[name].issubset(policy_sym_vers[name]): - for symbol in required_vers[name] - policy_sym_vers[name]: - log.debug( - "Package requires %s, incompatible with " - "policy %s which requires %s", - symbol, - policy_name, - policy_sym_vers[name], - ) - policy_satisfied = False - return policy_satisfied - - required_vers: dict[str, set[str]] = {} - for symbols in versioned_symbols.values(): - for symbol in symbols: - sym_name, _, _ = symbol.partition("_") - required_vers.setdefault(sym_name, set()).add(symbol) - matching_policies: list[int] = [] - for p in wheel_policy.policies: - policy_sym_vers = { - sym_name: {sym_name + "_" + version for version in versions} - for sym_name, versions in p["symbol_versions"].items() - } - if policy_is_satisfied(p["name"], policy_sym_vers): - matching_policies.append(p["priority"]) - - if len(matching_policies) == 0: - # the base policy (generic linux) should always match - raise RuntimeError("Internal error") - - return max(matching_policies) diff --git a/src/auditwheel/wheel_abi.py b/src/auditwheel/wheel_abi.py index 988e78b4..e4a9446b 100644 --- a/src/auditwheel/wheel_abi.py +++ b/src/auditwheel/wheel_abi.py @@ -19,11 +19,7 @@ ) from .genericpkgctx import InGenericPkgCtx from .lddtree import lddtree -from .policy import ( - WheelPolicies, - lddtree_external_references, - versioned_symbols_policy, -) +from .policy import WheelPolicies log = logging.getLogger(__name__) WheelAbIInfo = namedtuple( @@ -101,8 +97,8 @@ def get_wheel_elfdata(wheel_policy: WheelPolicies, wheel_fn: str): uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf) ) - full_external_refs[fn] = lddtree_external_references( - wheel_policy.policies, elftree, ctx.path + full_external_refs[fn] = wheel_policy.lddtree_external_references( + elftree, ctx.path ) else: # If the ELF is not a Python extension, it might be @@ -144,8 +140,8 @@ def get_wheel_elfdata(wheel_policy: WheelPolicies, wheel_fn: str): # Even if a non-pyextension ELF file is not needed, we # should include it as an external reference, because # it might require additional external libraries. - full_external_refs[fn] = lddtree_external_references( - wheel_policy.policies, nonpy_elftree[fn], ctx.path + full_external_refs[fn] = wheel_policy.lddtree_external_references( + nonpy_elftree[fn], ctx.path ) log.debug("full_elftree:\n%s", json.dumps(full_elftree, indent=4)) @@ -201,7 +197,9 @@ def get_versioned_symbols(libs): return result -def get_symbol_policies(wheel_policy, versioned_symbols, external_versioned_symbols, external_refs): +def get_symbol_policies( + wheel_policy, versioned_symbols, external_versioned_symbols, external_refs +): """Get symbol policies Since white-list is different per policy, this function inspects versioned_symbol per policy when including external refs @@ -223,7 +221,9 @@ def get_symbol_policies(wheel_policy, versioned_symbols, external_versioned_symb ext_symbols = external_versioned_symbols[soname] for k in iter(ext_symbols): policy_symbols[k].update(ext_symbols[k]) - result.append((versioned_symbols_policy(wheel_policy, policy_symbols), policy_symbols)) + result.append( + (wheel_policy.versioned_symbols_policy(policy_symbols), policy_symbols) + ) return result @@ -252,7 +252,7 @@ def analyze_wheel_abi(wheel_policy: WheelPolicies, wheel_fn: str) -> WheelAbIInf symbol_policies = get_symbol_policies( wheel_policy, versioned_symbols, external_versioned_symbols, external_refs ) - symbol_policy = versioned_symbols_policy(wheel_policy, versioned_symbols) + symbol_policy = wheel_policy.versioned_symbols_policy(versioned_symbols) # let's keep the highest priority policy and # corresponding versioned_symbols diff --git a/tests/integration/test_policy_files.py b/tests/integration/test_policy_files.py index 2aafa09a..6d83757f 100644 --- a/tests/integration/test_policy_files.py +++ b/tests/integration/test_policy_files.py @@ -2,11 +2,7 @@ from jsonschema import validate -from auditwheel.policy import ( - WheelPolicies, - _load_policy_schema, - versioned_symbols_policy, -) +from auditwheel.policy import WheelPolicies, _load_policy_schema def test_policy(): @@ -18,15 +14,13 @@ def test_policy(): def test_policy_checks_glibc(): wheel_policy = WheelPolicies() - policy = versioned_symbols_policy(wheel_policy, {"some_library.so": {"GLIBC_2.17"}}) + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}}) assert policy > wheel_policy.priority_lowest - policy = versioned_symbols_policy(wheel_policy, {"some_library.so": {"GLIBC_999"}}) + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_999"}}) assert policy == wheel_policy.priority_lowest - policy = versioned_symbols_policy( - wheel_policy, {"some_library.so": {"OPENSSL_1_1_0"}} + policy = wheel_policy.versioned_symbols_policy( + {"some_library.so": {"OPENSSL_1_1_0"}} ) assert policy == wheel_policy.priority_highest - policy = versioned_symbols_policy( - wheel_policy, {"some_library.so": {"IAMALIBRARY"}} - ) + policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}}) assert policy == wheel_policy.priority_highest diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index 9552feff..181c6b29 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -9,7 +9,6 @@ _validate_pep600_compliance, get_arch_name, get_replace_platforms, - lddtree_external_references, ) @@ -227,8 +226,8 @@ def test_filter_libs(self): "libs": {lib: {"needed": [], "realpath": "/path/to/lib"} for lib in libs}, } wheel_policy = WheelPolicies() - full_external_refs = lddtree_external_references( - wheel_policy.policies, lddtree, "/path/to/wheel" + full_external_refs = wheel_policy.lddtree_external_references( + lddtree, "/path/to/wheel" ) # Assert that each policy only has the unfiltered libs.