Fix cirular import in policy submodule

The `external_versioned_symbols` and `versioned_symbols_policy`
functions used to live in two separate files, importing code from the
`__init__.py` file in the same module. As this file itself imports both
of these symbols, this creates a cirtular import. We can easily break
this by moving the two functions into the same `__init__.py` file.

src/auditwheel/policy/__init__.py

@@ -3,17 +3,20 @@
 import json
 import logging
 import platform as _platform_module
+import re
 import sys
 from collections import defaultdict
 from os.path import abspath, dirname, join
 from pathlib import Path
+from typing import Any, Generator
+
+from auditwheel.elfutils import filter_undefined_symbols, is_subdir
 
 from ..libc import Libc, get_libc
 from ..musllinux import find_musl_libc, get_musl_version
-from .external_references import lddtree_external_references
-from .versioned_symbols import versioned_symbols_policy
 
 _HERE = Path(__file__).parent
+LIBPYTHON_RE = re.compile(r"^libpython\d+\.\d+m?.so(.\d)*$")
 
 logger = logging.getLogger(__name__)
 
@@ -97,6 +100,112 @@ def get_priority_by_name(self, name: str) -> int | None:
         policy = self.get_policy_by_name(name)
         return None if policy is None else policy["priority"]
 
+    def versioned_symbols_policy(self, versioned_symbols: dict[str, set[str]]) -> int:
+        def policy_is_satisfied(
+            policy_name: str, policy_sym_vers: dict[str, set[str]]
+        ) -> bool:
+            policy_satisfied = True
+            for name in set(required_vers) & set(policy_sym_vers):
+                if not required_vers[name].issubset(policy_sym_vers[name]):
+                    for symbol in required_vers[name] - policy_sym_vers[name]:
+                        logger.debug(
+                            "Package requires %s, incompatible with "
+                            "policy %s which requires %s",
+                            symbol,
+                            policy_name,
+                            policy_sym_vers[name],
+                        )
+                    policy_satisfied = False
+            return policy_satisfied
+
+        required_vers: dict[str, set[str]] = {}
+        for symbols in versioned_symbols.values():
+            for symbol in symbols:
+                sym_name, _, _ = symbol.partition("_")
+                required_vers.setdefault(sym_name, set()).add(symbol)
+        matching_policies: list[int] = []
+        for p in self.policies:
+            policy_sym_vers = {
+                sym_name: {sym_name + "_" + version for version in versions}
+                for sym_name, versions in p["symbol_versions"].items()
+            }
+            if policy_is_satisfied(p["name"], policy_sym_vers):
+                matching_policies.append(p["priority"])
+
+        if len(matching_policies) == 0:
+            # the base policy (generic linux) should always match
+            raise RuntimeError("Internal error")
+
+        return max(matching_policies)
+
+    def lddtree_external_references(self, lddtree: dict, wheel_path: str) -> dict:
+        # XXX: Document the lddtree structure, or put it in something
+        # more stable than a big nested dict
+        def filter_libs(
+            libs: set[str], whitelist: set[str]
+        ) -> Generator[str, None, None]:
+            for lib in libs:
+                if "ld-linux" in lib or lib in ["ld64.so.2", "ld64.so.1"]:
+                    # always exclude ELF dynamic linker/loader
+                    # 'ld64.so.2' on s390x
+                    # 'ld64.so.1' on ppc64le
+                    # 'ld-linux*' on other platforms
+                    continue
+                if LIBPYTHON_RE.match(lib):
+                    # always exclude libpythonXY
+                    continue
+                if lib in whitelist:
+                    # exclude any libs in the whitelist
+                    continue
+                yield lib
+
+        def get_req_external(libs: set[str], whitelist: set[str]) -> set[str]:
+            # get all the required external libraries
+            libs = libs.copy()
+            reqs = set()
+            while libs:
+                lib = libs.pop()
+                reqs.add(lib)
+                for dep in filter_libs(lddtree["libs"][lib]["needed"], whitelist):
+                    if dep not in reqs:
+                        libs.add(dep)
+            return reqs
+
+        ret: dict[str, dict[str, Any]] = {}
+        for p in self.policies:
+            needed_external_libs: set[str] = set()
+            blacklist = {}
+
+            if not (p["name"] == "linux" and p["priority"] == 0):
+                # special-case the generic linux platform here, because it
+                # doesn't have a whitelist. or, you could say its
+                # whitelist is the complete set of all libraries. so nothing
+                # is considered "external" that needs to be copied in.
+                whitelist = set(p["lib_whitelist"])
+                blacklist_libs = set(p["blacklist"].keys()) & set(lddtree["needed"])
+                blacklist = {k: p["blacklist"][k] for k in blacklist_libs}
+                blacklist = filter_undefined_symbols(lddtree["realpath"], blacklist)
+                needed_external_libs = get_req_external(
+                    set(filter_libs(lddtree["needed"], whitelist)), whitelist
+                )
+
+            pol_ext_deps = {}
+            for lib in needed_external_libs:
+                if is_subdir(lddtree["libs"][lib]["realpath"], wheel_path):
+                    # we didn't filter libs that resolved via RPATH out
+                    # earlier because we wanted to make sure to pick up
+                    # our elf's indirect dependencies. But now we want to
+                    # filter these ones out, since they're not "external".
+                    logger.debug("RPATH FTW: %s", lib)
+                    continue
+                pol_ext_deps[lib] = lddtree["libs"][lib]["realpath"]
+            ret[p["name"]] = {
+                "libs": pol_ext_deps,
+                "priority": p["priority"],
+                "blacklist": blacklist,
+            }
+        return ret
+
 
 def get_arch_name() -> str:
     machine = _platform_module.machine()
@@ -204,7 +313,5 @@ def _load_policy_schema():
 
 
 __all__ = [
-    "lddtree_external_references",
-    "versioned_symbols_policy",
     "WheelPolicies",
 ]

src/auditwheel/wheel_abi.py

@@ -19,11 +19,7 @@
 )
 from .genericpkgctx import InGenericPkgCtx
 from .lddtree import lddtree
-from .policy import (
-    WheelPolicies,
-    lddtree_external_references,
-    versioned_symbols_policy,
-)
+from .policy import WheelPolicies
 
 log = logging.getLogger(__name__)
 WheelAbIInfo = namedtuple(
@@ -101,8 +97,8 @@ def get_wheel_elfdata(wheel_policy: WheelPolicies, wheel_fn: str):
                         uses_ucs2_symbols |= any(
                             True for _ in elf_find_ucs2_symbols(elf)
                         )
-                    full_external_refs[fn] = lddtree_external_references(
-                        wheel_policy.policies, elftree, ctx.path
+                    full_external_refs[fn] = wheel_policy.lddtree_external_references(
+                        elftree, ctx.path
                     )
                 else:
                     # If the ELF is not a Python extension, it might be
@@ -144,8 +140,8 @@ def get_wheel_elfdata(wheel_policy: WheelPolicies, wheel_fn: str):
             # Even if a non-pyextension ELF file is not needed, we
             # should include it as an external reference, because
             # it might require additional external libraries.
-            full_external_refs[fn] = lddtree_external_references(
-                wheel_policy.policies, nonpy_elftree[fn], ctx.path
+            full_external_refs[fn] = wheel_policy.lddtree_external_references(
+                nonpy_elftree[fn], ctx.path
             )
 
     log.debug("full_elftree:\n%s", json.dumps(full_elftree, indent=4))
@@ -201,7 +197,9 @@ def get_versioned_symbols(libs):
     return result
 
 
-def get_symbol_policies(wheel_policy, versioned_symbols, external_versioned_symbols, external_refs):
+def get_symbol_policies(
+    wheel_policy, versioned_symbols, external_versioned_symbols, external_refs
+):
     """Get symbol policies
     Since white-list is different per policy, this function inspects
     versioned_symbol per policy when including external refs
@@ -223,7 +221,9 @@ def get_symbol_policies(wheel_policy, versioned_symbols, external_versioned_symb
             ext_symbols = external_versioned_symbols[soname]
             for k in iter(ext_symbols):
                 policy_symbols[k].update(ext_symbols[k])
-        result.append((versioned_symbols_policy(wheel_policy, policy_symbols), policy_symbols))
+        result.append(
+            (wheel_policy.versioned_symbols_policy(policy_symbols), policy_symbols)
+        )
     return result
 
 
@@ -252,7 +252,7 @@ def analyze_wheel_abi(wheel_policy: WheelPolicies, wheel_fn: str) -> WheelAbIInf
     symbol_policies = get_symbol_policies(
         wheel_policy, versioned_symbols, external_versioned_symbols, external_refs
     )
-    symbol_policy = versioned_symbols_policy(wheel_policy, versioned_symbols)
+    symbol_policy = wheel_policy.versioned_symbols_policy(versioned_symbols)
 
     # let's keep the highest priority policy and
     # corresponding versioned_symbols

tests/integration/test_policy_files.py

@@ -2,11 +2,7 @@
 
 from jsonschema import validate
 
-from auditwheel.policy import (
-    WheelPolicies,
-    _load_policy_schema,
-    versioned_symbols_policy,
-)
+from auditwheel.policy import WheelPolicies, _load_policy_schema
 
 
 def test_policy():
@@ -18,15 +14,13 @@ def test_policy():
 def test_policy_checks_glibc():
     wheel_policy = WheelPolicies()
 
-    policy = versioned_symbols_policy(wheel_policy, {"some_library.so": {"GLIBC_2.17"}})
+    policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_2.17"}})
     assert policy > wheel_policy.priority_lowest
-    policy = versioned_symbols_policy(wheel_policy, {"some_library.so": {"GLIBC_999"}})
+    policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"GLIBC_999"}})
     assert policy == wheel_policy.priority_lowest
-    policy = versioned_symbols_policy(
-        wheel_policy, {"some_library.so": {"OPENSSL_1_1_0"}}
+    policy = wheel_policy.versioned_symbols_policy(
+        {"some_library.so": {"OPENSSL_1_1_0"}}
     )
     assert policy == wheel_policy.priority_highest
-    policy = versioned_symbols_policy(
-        wheel_policy, {"some_library.so": {"IAMALIBRARY"}}
-    )
+    policy = wheel_policy.versioned_symbols_policy({"some_library.so": {"IAMALIBRARY"}})
     assert policy == wheel_policy.priority_highest

tests/unit/test_policy.py

@@ -9,7 +9,6 @@
     _validate_pep600_compliance,
     get_arch_name,
     get_replace_platforms,
-    lddtree_external_references,
 )
 
 
@@ -227,8 +226,8 @@ def test_filter_libs(self):
             "libs": {lib: {"needed": [], "realpath": "/path/to/lib"} for lib in libs},
         }
         wheel_policy = WheelPolicies()
-        full_external_refs = lddtree_external_references(
-            wheel_policy.policies, lddtree, "/path/to/wheel"
+        full_external_refs = wheel_policy.lddtree_external_references(
+            lddtree, "/path/to/wheel"
         )
 
         # Assert that each policy only has the unfiltered libs.