Use unearth as the backend to find and download packages (#1096)

.pre-commit-config.yaml

@@ -25,5 +25,4 @@ repos:
         args: [pdm]
         pass_filenames: false
         additional_dependencies:
-          - types-setuptools
-          - types-toml
+          - types-requests

docs/docs/usage/dependency.md

@@ -252,6 +252,21 @@ Include the following setting in `pyproject.toml` to enable:
 allow_prereleases = true
 ```
 
+## Set acceptable format for locking or installing
+
+If you want to control the format(binary/sdist) of the packages, you can set the env vars `PDM_NO_BINARY` and `PDM_ONLY_BINARY`.
+
+Each env var is a comma-separated list of package name. You can set it to `:all:` to apply to all packages. For example:
+
+```
+# No binary for werkzeug will be locked nor used for installation
+PDM_NO_BINARY=werkzeug pdm add flask
+# Only binaries will be locked in the lock file
+PDM_ONLY_BINARY=:all: pdm lock
+# No binaries will be used for installation
+PDM_NO_BINARY=:all: pdm install
+```
+
 ## Solve the locking failure
 
 If PDM is not able to find a resolution to satisfy the requirements, it will raise an error. For example,

news/1072.dep.md

@@ -0,0 +1 @@
+Prefer tomllib on Python 3.11

news/1096.feature.md

@@ -0,0 +1 @@
+Improved the terminal UI and logging. Disable live progress under verbose mode. The logger levels can be controlled by the `-v` option.

news/1096.refactor.md

@@ -0,0 +1 @@
+Use `unearth` to replace `pip`'s `PackageFinder` and related data models. PDM no longer relies on `pip` internals, which are unstable across updates.

pdm/__init__.py

@@ -16,14 +16,3 @@
     "Synchronizer",
     "Core",
 )
-
-
-def _fix_pkg_resources() -> None:
-    import importlib
-    import sys
-
-    sys.modules["pkg_resources"] = importlib.import_module("pip._vendor.pkg_resources")
-
-
-_fix_pkg_resources()
-del _fix_pkg_resources

pdm/_types.py

@@ -9,7 +9,7 @@ class Source(TypedDict, total=False):
     url: str
     verify_ssl: bool
     name: str
-    type: Literal["index"] | Literal["find_links"]
+    type: Literal["index", "find_links"]
 
 
 RequirementDict = Union[str, Dict[str, Union[str, bool]]]

pdm/builders/base.py

@@ -74,7 +74,7 @@ def log_subprocessor(
     env = os.environ.copy()
     if extra_environ:
         env.update(extra_environ)
-    outstream = LoggerWrapper(logger, logging.DEBUG)
+    outstream = LoggerWrapper(logger, logging.INFO)
     try:
         subprocess.check_call(
             cmd,
@@ -171,7 +171,7 @@ def __init__(self, src_dir: str | Path, environment: Environment) -> None:
         self.executable = self._env.interpreter.executable.as_posix()
         self.src_dir = src_dir
         self.isolated = environment.project.config["build_isolation"]
-        logger.debug("Preparing isolated env for PEP 517 build...")
+        logger.info("Preparing isolated env for PEP 517 build...")
         try:
             with open(os.path.join(src_dir, "pyproject.toml"), "rb") as f:
                 spec = tomllib.load(f)

pdm/cli/actions.py

@@ -790,8 +790,6 @@ def print_pep582_command(ui: termui.UI, shell: str = "AUTO") -> None:
 
 def get_latest_version(project: Project) -> str | None:
     """Get the latest version of PDM from PyPI, cache for 7 days"""
-    from pdm.utils import get_finder
-
     cache_key = hashlib.sha224(sys.executable.encode()).hexdigest()
     cache_file = project.cache("self-check") / cache_key
     if cache_file.exists():
@@ -804,10 +802,13 @@ def get_latest_version(project: Project) -> str | None:
         and current_time - state["last-check"] < 60 * 60 * 24 * 7
     ):
         return cast(str, state["latest-version"])
-    candidate = get_finder([], project.cache_dir.as_posix()).find_best_candidate("pdm")
-    if not candidate.best_candidate:
+    with project.environment.get_finder(
+        [project.default_source], ignore_compatibility=True
+    ) as finder:
+        candidate = finder.find_best_match("pdm").best
+    if not candidate:
         return None
-    latest_version = str(candidate.best_candidate.version)
+    latest_version = str(candidate.version)
     state.update({"latest-version": latest_version, "last-check": current_time})
     cache_file.write_text(json.dumps(state))
     return latest_version
@@ -818,7 +819,7 @@ def check_update(project: Project) -> None:
     import sys
     from shlex import quote
 
-    from pip._vendor.packaging.version import parse as parse_version
+    from packaging.version import parse as parse_version
 
     from pdm.cli.utils import (
         is_homebrew_installation,

pdm/cli/commands/cache.py

@@ -7,7 +7,6 @@
 from pdm.cli.commands.base import BaseCommand
 from pdm.cli.options import verbose_option
 from pdm.exceptions import PdmUsageError
-from pdm.models.pip_shims import directory_size, file_size, find_files
 from pdm.project import Project
 
 
@@ -29,6 +28,22 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
         self.parser.print_help()
 
 
+def file_size(file: Path) -> int:
+    if file.is_symlink():
+        return 0
+    return os.path.getsize(file)
+
+
+def find_files(parent: Path, pattern: str) -> Iterable[Path]:
+    for file in parent.rglob(pattern):
+        if file.is_file() or file.is_symlink():
+            yield file
+
+
+def directory_size(directory: Path) -> int:
+    return sum(map(file_size, find_files(directory, "*")))
+
+
 def format_size(size: float) -> str:
     if size > 1000 * 1000:
         return "{:.1f} MB".format(size / 1000.0 / 1000)
@@ -44,11 +59,8 @@ def remove_cache_files(project: Project, pattern: str) -> None:
     if not pattern:
         raise PdmUsageError("Please provide a pattern")
 
-    if pattern == "*":
-        files = list(find_files(project.cache_dir.as_posix(), pattern))
-    else:
-        # Only remove wheel files which specific pattern is given
-        files = list(find_files(project.cache("wheels").as_posix(), pattern))
+    wheel_cache = project.cache("wheels")
+    files = list(find_files(wheel_cache, pattern))
 
     if not files:
         raise PdmUsageError("No matching files found")
@@ -89,15 +101,16 @@ def _clear_packages(root: Path) -> int:
 
     @staticmethod
     def _clear_files(root: Path) -> int:
-        files = find_files(root.as_posix(), "*")
+        files = list(find_files(root, "*"))
         for file in files:
             os.unlink(file)
         return len(files)
 
     def handle(self, project: Project, options: argparse.Namespace) -> None:
 
+        types: Iterable[str] = ()
         if not options.type:
-            types: Iterable[str] = self.CACHE_TYPES
+            pass
         elif options.type not in self.CACHE_TYPES:
             raise PdmUsageError(
                 f"Invalid cache type {options.type}, should one of {self.CACHE_TYPES}"
@@ -109,11 +122,14 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
         with project.core.ui.open_spinner(
             f"Clearing {options.type or 'all'} caches..."
         ):
-            for type_ in types:
-                if type_ == "packages":
-                    packages += self._clear_packages(project.cache(type_))
-                else:
-                    files += self._clear_files(project.cache(type_))
+            if not options.type:
+                packages, files = 0, self._clear_files(project.cache_dir)
+            else:
+                for type_ in types:
+                    if type_ == "packages":
+                        packages += self._clear_packages(project.cache(type_))
+                    else:
+                        files += self._clear_files(project.cache(type_))
             message = []
             if packages:
                 message.append(f"{packages} package{'s' if packages > 1 else ''}")
@@ -150,8 +166,8 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None:
 
     def handle(self, project: Project, options: argparse.Namespace) -> None:
         rows = [
-            (format_size(file_size(file)), os.path.basename(file))
-            for file in find_files(project.cache("wheels").as_posix(), options.pattern)
+            (format_size(file_size(file)), file.name)
+            for file in find_files(project.cache("wheels"), options.pattern)
         ]
         project.core.ui.display_columns(rows, [">Size", "Filename"])
 
@@ -165,18 +181,18 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
         with project.core.ui.open_spinner("Calculating cache files"):
             output = [
                 f"[cyan]Cache Root[/]: {project.cache_dir}, "
-                f"Total size: {format_size(directory_size(str(project.cache_dir)))}"
+                f"Total size: {format_size(directory_size(project.cache_dir))}"
             ]
             for name, description in [
-                ("hashes", "File Hashe Cache"),
+                ("hashes", "File Hash Cache"),
                 ("http", "HTTP Cache"),
                 ("wheels", "Wheels Cache"),
                 ("metadata", "Metadata Cache"),
                 ("packages", "Package Cache"),
             ]:
                 cache_location = project.cache(name)
-                files = list(find_files(cache_location.as_posix(), "*"))
-                size = directory_size(cache_location.as_posix())
+                files = list(find_files(cache_location, "*"))
+                size = directory_size(cache_location)
                 output.append(f"  [cyan]{description}[/]: {cache_location}")
                 output.append(f"    Files: {len(files)}, Size: {format_size(size)}")
 

pdm/cli/commands/info.py

@@ -44,7 +44,7 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
             )
         else:
 
-            rows = zip(
+            for name, value in zip(
                 [
                     f"[bold cyan]{key}[/]:"
                     for key in [
@@ -60,5 +60,5 @@ def handle(self, project: Project, options: argparse.Namespace) -> None:
                     project.root.as_posix(),
                     str(project.environment.packages_path),
                 ],
-            )
-            project.core.ui.display_columns(list(rows))
+            ):
+                project.core.ui.echo(f"{name}\n  {value}")

pdm/cli/utils.py

@@ -29,7 +29,6 @@
 from pdm.exceptions import PdmUsageError, ProjectError
 from pdm.formats import FORMATS
 from pdm.formats.base import make_array, make_inline_table
-from pdm.models.pip_shims import url_to_path
 from pdm.models.requirements import (
     Requirement,
     filter_requirements_with_extras,
@@ -39,7 +38,7 @@
 from pdm.models.specifiers import get_specifier
 from pdm.models.working_set import WorkingSet
 from pdm.project import Project
-from pdm.utils import is_path_relative_to
+from pdm.utils import is_path_relative_to, url_to_path
 
 if TYPE_CHECKING:
     from resolvelib.resolvers import RequirementInformation, ResolutionImpossible

pdm/core.py

@@ -134,8 +134,6 @@ def main(
         **extra: Any,
     ) -> None:
         """The main entry function"""
-        from pdm.models.pip_shims import global_tempdir_manager
-
         options = self.parser.parse_args(args or None)
         self.ui.set_verbosity(options.verbose)
         if options.ignore_python:
@@ -154,8 +152,7 @@ def main(
             sys.exit(1)
         else:
             try:
-                with global_tempdir_manager():
-                    f(options.project, options)
+                f(options.project, options)
             except Exception:
                 etype, err, traceback = sys.exc_info()
                 should_show_tb = not isinstance(err, PdmUsageError)