Merge pull request #99 from xen0n/fix-ssl-for-good

Really fix system certificate path querying & allow execution from progcache

pyproject.toml

@@ -25,7 +25,7 @@ target-version = ['py311']
 
 
 [tool.mypy]
-files = ["ruyi"]
+files = ["ruyi", "scripts"]
 show_error_codes = true
 strict = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -68,3 +68,5 @@ nuitka = "^2.0"
 
 [tool.poetry.group.release-worker.dependencies]
 requests = "^2"
+
+types-requests = "^2.31.0.20240311"

ruyi/__init__.py

@@ -1,3 +1,5 @@
+import typing
+
 _is_debug = False
 
 
@@ -31,3 +33,21 @@ def record_self_exe(argv0: str, x: str) -> None:
 
 # This is true if we're packaged
 IS_PACKAGED = False
+
+if typing.TYPE_CHECKING:
+
+    class NuitkaVersion(typing.NamedTuple):
+        major: int
+        minor: int
+        micro: int
+        releaselevel: str
+        containing_dir: str
+        standalone: bool
+        onefile: bool
+        macos_bundle_mode: bool
+        no_asserts: bool
+        no_docstrings: bool
+        no_annotations: bool
+        module: bool
+
+    __compiled__: NuitkaVersion

ruyi/__main__.py

@@ -24,8 +24,11 @@
     # we assume the one-file build if Nuitka is detected; sys.argv[0] does NOT
     # work if it's just `ruyi` so we have to check our parent process in that case
     if hasattr(ruyi, "__compiled__"):
-        self_exe = get_nuitka_self_exe()
         ruyi.IS_PACKAGED = True
+        log.D(
+            f"__file__ = {__file__}, sys.executable = {sys.executable}, __compiled__ = {ruyi.__compiled__}"
+        )
+        self_exe = get_nuitka_self_exe()
     else:
         self_exe = __file__
 

ruyi/cli/nuitka.py

@@ -2,8 +2,16 @@
 
 
 def get_nuitka_self_exe() -> str:
-    # Assume we're a Nuitka onefile build, so our parent process is the onefile
-    # bootstrap process. The onefile bootstrapper puts "our path" in the
-    # undocumented environment variable $NUITKA_ONEFILE_BINARY, which works
-    # on both Linux and Windows.
-    return os.environ["NUITKA_ONEFILE_BINARY"]
+    try:
+        # Assume we're a Nuitka onefile build, so our parent process is the onefile
+        # bootstrap process. The onefile bootstrapper puts "our path" in the
+        # undocumented environment variable $NUITKA_ONEFILE_BINARY, which works
+        # on both Linux and Windows.
+        return os.environ["NUITKA_ONEFILE_BINARY"]
+    except KeyError:
+        # It seems we are instead launched from the extracted onefile tempdir.
+        # Assume our name is "ruyi" in this case; directory is available in
+        # Nuitka metadata.
+        import ruyi
+
+        return os.path.join(ruyi.__compiled__.containing_dir, "ruyi")

ruyi/utils/ssl_patch.py

@@ -62,7 +62,11 @@ def _query_linux_system_ssl_default_cert_paths(
     soname: str | None = None,
 ) -> tuple[str, str, str, str]:
     if soname is None:
-        for soname in ("libssl.so", "libssl.so.3", "libssl.so.1.1"):
+        # check libcrypto instead of libssl, because if the system libssl is
+        # newer than the bundled one, the system libssl will depend on the
+        # bundled libcrypto that may lack newer ELF symbol version(s). The
+        # functions actually reside in libcrypto, after all.
+        for soname in ("libcrypto.so", "libcrypto.so.3", "libcrypto.so.1.1"):
             try:
                 return _query_linux_system_ssl_default_cert_paths(soname)
             except OSError as e:
@@ -71,25 +75,31 @@ def _query_linux_system_ssl_default_cert_paths(
 
         # cannot proceed without certificates info (pygit2 initialization is
         # bound to fail anyway)
-        log.F("cannot find the system libssl")
+        log.F("cannot find the system libcrypto")
         log.I("TLS certificates and library are required for Ruyi to function")
         raise SystemExit(1)
 
-    # this can work because right now Nuitka packages the libssl as "libssl.so.X"
-    # notice the presence of sover suffix
-    # so dlopen-ing "libssl.so" will get us the system library
-    libssl = ctypes.CDLL(soname)
-    libssl.X509_get_default_cert_file_env.restype = ctypes.c_void_p
-    libssl.X509_get_default_cert_file.restype = ctypes.c_void_p
-    libssl.X509_get_default_cert_dir_env.restype = ctypes.c_void_p
-    libssl.X509_get_default_cert_dir.restype = ctypes.c_void_p
-
-    return (
-        _decode_fsdefault_or_none(libssl.X509_get_default_cert_file_env()),
-        _decode_fsdefault_or_none(libssl.X509_get_default_cert_file()),
-        _decode_fsdefault_or_none(libssl.X509_get_default_cert_dir_env()),
-        _decode_fsdefault_or_none(libssl.X509_get_default_cert_dir()),
+    # dlopen-ing the bare soname will get us the system library
+    lib = ctypes.CDLL(soname)
+    lib.X509_get_default_cert_file_env.restype = ctypes.c_void_p
+    lib.X509_get_default_cert_file.restype = ctypes.c_void_p
+    lib.X509_get_default_cert_dir_env.restype = ctypes.c_void_p
+    lib.X509_get_default_cert_dir.restype = ctypes.c_void_p
+
+    result = (
+        _decode_fsdefault_or_none(lib.X509_get_default_cert_file_env()),
+        _decode_fsdefault_or_none(lib.X509_get_default_cert_file()),
+        _decode_fsdefault_or_none(lib.X509_get_default_cert_dir_env()),
+        _decode_fsdefault_or_none(lib.X509_get_default_cert_dir()),
     )
 
+    log.D(f"got defaults from system libcrypto {soname}")
+    log.D(f"X509_get_default_cert_file_env() = {result[0]}")
+    log.D(f"X509_get_default_cert_file() = {result[1]}")
+    log.D(f"X509_get_default_cert_dir_env() = {result[2]}")
+    log.D(f"X509_get_default_cert_dir() = {result[3]}")
+
+    return result
+
 
 ssl.get_default_verify_paths = get_system_ssl_default_verify_paths

scripts/build-pygit2.py

@@ -5,9 +5,10 @@
 import subprocess
 import sys
 import tomllib
+from typing import cast
 
 
-def get_pygit2_src_uri(tag: str) -> (str, str):
+def get_pygit2_src_uri(tag: str) -> tuple[str, str]:
     filename = f"{tag}.tar.gz"
     return (filename, f"https://github.com/libgit2/pygit2/archive/refs/tags/{filename}")
 
@@ -115,14 +116,14 @@ def get_pygit2_version() -> str:
         info = tomllib.load(fp)
 
     pygit2 = [pkg for pkg in info["package"] if pkg["name"] == "pygit2"][0]
-    return pygit2["version"]
+    return cast(str, pygit2["version"])
 
 
 def get_pygit2_wheel_build_env(pygit2_dir: str) -> dict[str, str]:
     with open(os.path.join(pygit2_dir, "pyproject.toml"), "rb") as fp:
         pyproject = tomllib.load(fp)
 
-    r = pyproject["tool"]["cibuildwheel"]["environment"]
+    r: dict[str, str] = pyproject["tool"]["cibuildwheel"]["environment"]
     if "LIBGIT2" in r:
         # this is unnecessary
         del r["LIBGIT2"]

scripts/dist-inner.py

@@ -7,6 +7,7 @@
 import sys
 import time
 import tomllib
+from typing import cast
 
 from pygit2.repository import Repository
 from rich.console import Console
@@ -85,7 +86,7 @@ def main() -> None:
         set_release_mirror_url_for_gha(vers["semver"])
 
 
-def ensure_dir(d: str) -> None:
+def ensure_dir(d: str | pathlib.Path) -> None:
     try:
         os.mkdir(d)
     except FileExistsError:
@@ -111,6 +112,7 @@ def delete_cached_files_older_than_days(root: str, days: int, epoch: int) -> Non
             INFO.print(f"ignoring pygit2 cache [cyan]{f}")
             continue
 
+        ts: int | None
         try:
             ts = int((f / "timestamp").read_text().strip(), 10)
         except (FileNotFoundError, ValueError):
@@ -152,7 +154,7 @@ def add_pythonpath(path: str) -> None:
 
 def make_nuitka_ext(module_name: str, out_dir: str) -> None:
     mod = __import__(module_name)
-    mod_dir = os.path.dirname(mod.__file__)
+    mod_dir = os.path.dirname(cast(str, mod.__file__))
     INFO.print(f"Building [cyan]{module_name}[/] at [cyan]{mod_dir}[/] into extension")
     call_nuitka(
         "--module",

scripts/organize-release-artifacts.py

@@ -68,6 +68,8 @@ def main(argv: list[str]) -> int:
         os.chmod(dest_name, 0o755)
         os.rmdir(name)
 
+    return 0
+
 
 if __name__ == "__main__":
     sys.exit(main(sys.argv))

scripts/release-worker/sync-releases.py

@@ -36,18 +36,18 @@
 )
 
 LOG = Console(stderr=True, highlight=False)
-IS_DEBUG = False
-PROGRAM_NAME = __file__
+is_debug = False
+program_name = __file__
 
 
 def debug(*args: object) -> None:
-    if not IS_DEBUG:
+    if not is_debug:
         return
     return LOG.log(*args)
 
 
 def usage() -> None:
-    LOG.print(USAGE.format(program_name=PROGRAM_NAME))
+    LOG.print(USAGE.format(program_name=program_name))
 
 
 def getenv_or_die(key: str) -> str:
@@ -65,7 +65,7 @@ def github_get(
     url: str,
     accept: str,
     stream: bool = False,
-    **kwargs: object,
+    **kwargs: str | int,
 ) -> requests.Response:
     return requests.get(
         url,
@@ -205,7 +205,7 @@ def __init__(self, conn_url: str, password: str | None = None) -> None:
         self.conn_url = conn_url
         self.password = password
 
-    def sync(self, rel: Release, local_dir: str) -> None:
+    def sync(self, rel: Release, local_dir: str | pathlib.Path) -> None:
         new_env: dict[bytes, bytes] | None = None
         if self.password is not None:
             new_env = os.environb.copy()
@@ -237,9 +237,9 @@ def mark_release_synced(self, rel: Release) -> None:
 
 
 if __name__ == "__main__":
-    PROGRAM_NAME = sys.argv[0]
+    program_name = sys.argv[0]
     # same as ruyi.cli.init_debug_status
     debug_env = os.environ.get("RUYI_DEBUG", "")
-    IS_DEBUG = debug_env.lower() in {"1", "true", "x", "y", "yes"}
+    is_debug = debug_env.lower() in {"1", "true", "x", "y", "yes"}
 
     sys.exit(main(sys.argv))