diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c034c..77aed952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ All versions prior to 0.0.9 are untracked. ## [Unreleased] +### Changed + +* Added option to skip dependency resolution via `pip` with the `--disable-pip` + flag. This option can only be used with hashed requirements files or when the + `--no-deps` flag has been provided. + ([#610](https://github.com/pypa/pip-audit/pull/610)) + ## [2.5.6] ### Fixed diff --git a/README.md b/README.md index 3cdb4905..fc751b7b 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ usage: pip-audit [-h] [-V] [-l] [-r REQUIREMENT] [-f FORMAT] [-s SERVICE] [-d] [--path PATH] [-v] [--fix] [--require-hashes] [--index-url INDEX_URL] [--extra-index-url URL] [--skip-editable] [--no-deps] [-o FILE] [--ignore-vuln ID] + [--disable-pip] [project_path] audit the Python environment for dependencies with known vulnerabilities @@ -206,6 +207,9 @@ optional arguments: --ignore-vuln ID ignore a specific vulnerability by its vulnerability ID; this option can be used multiple times (default: []) + --disable-pip don't use `pip` for dependency resolution; this can + only be used with hashed requirements files or if the + `--no-deps` flag has been provided (default: False) ``` diff --git a/pip_audit/_cli.py b/pip_audit/_cli.py index b8e14e58..8b6b8e83 100644 --- a/pip_audit/_cli.py +++ b/pip_audit/_cli.py @@ -327,6 +327,13 @@ def _parser() -> argparse.ArgumentParser: # pragma: no cover "this option can be used multiple times" ), ) + parser.add_argument( + "--disable-pip", + action="store_true", + help="don't use `pip` for dependency resolution; " + "this can only be used with hashed requirements files or if the `--no-deps` flag has been " + "provided", + ) return parser @@ -380,11 +387,16 @@ def audit() -> None: # pragma: no cover parser.error("The --extra-index-url flag can only be used with --requirement (-r)") elif args.no_deps: parser.error("The --no-deps flag can only be used with --requirement (-r)") + elif args.disable_pip: + parser.error("The --disable-pip flag can only be used with --requirement (-r)") # Nudge users to consider alternate workflows. if args.require_hashes and args.no_deps: logger.warning("The --no-deps flag is redundant when used with --require-hashes") + if args.no_deps and args.disable_pip: + logger.warning("The --no-deps flag is redundant when used with --disable-pip") + if args.require_hashes and isinstance(service, OsvService): logger.warning( "The --require-hashes flag with --service osv only enforces hash presence NOT hash " @@ -414,6 +426,7 @@ def audit() -> None: # pragma: no cover req_files, require_hashes=args.require_hashes, no_deps=args.no_deps, + disable_pip=args.disable_pip, skip_editable=args.skip_editable, index_url=args.index_url, extra_index_urls=args.extra_index_urls, diff --git a/pip_audit/_dependency_source/__init__.py b/pip_audit/_dependency_source/__init__.py index 3e8eb6fc..3d19e722 100644 --- a/pip_audit/_dependency_source/__init__.py +++ b/pip_audit/_dependency_source/__init__.py @@ -2,7 +2,13 @@ Dependency source interfaces and implementations for `pip-audit`. """ -from .interface import PYPI_URL, DependencyFixError, DependencySource, DependencySourceError +from .interface import ( + PYPI_URL, + DependencyFixError, + DependencySource, + DependencySourceError, + InvalidRequirementSpecifier, +) from .pip import PipSource, PipSourceError from .pyproject import PyProjectSource from .requirement import RequirementSource @@ -12,6 +18,7 @@ "DependencyFixError", "DependencySource", "DependencySourceError", + "InvalidRequirementSpecifier", "PipSource", "PipSourceError", "PyProjectSource", diff --git a/pip_audit/_dependency_source/interface.py b/pip_audit/_dependency_source/interface.py index 8655ecc0..0c888726 100644 --- a/pip_audit/_dependency_source/interface.py +++ b/pip_audit/_dependency_source/interface.py @@ -57,3 +57,12 @@ class DependencyFixError(Exception): """ pass + + +class InvalidRequirementSpecifier(DependencySourceError): + """ + A `DependencySourceError` specialized for the case of a non-PEP 440 requirements + specifier. + """ + + pass diff --git a/pip_audit/_dependency_source/requirement.py b/pip_audit/_dependency_source/requirement.py index 33c9993f..5d1ba493 100644 --- a/pip_audit/_dependency_source/requirement.py +++ b/pip_audit/_dependency_source/requirement.py @@ -15,12 +15,18 @@ from packaging.specifiers import SpecifierSet from packaging.utils import canonicalize_name +from packaging.version import Version from pip_requirements_parser import InstallRequirement, InvalidRequirementLine, RequirementsFile -from pip_audit._dependency_source import DependencyFixError, DependencySource, DependencySourceError +from pip_audit._dependency_source import ( + DependencyFixError, + DependencySource, + DependencySourceError, + InvalidRequirementSpecifier, +) from pip_audit._fix import ResolvedFixVersion from pip_audit._service import Dependency -from pip_audit._service.interface import ResolvedDependency +from pip_audit._service.interface import ResolvedDependency, SkippedDependency from pip_audit._state import AuditState from pip_audit._virtual_env import VirtualEnv, VirtualEnvError @@ -40,6 +46,7 @@ def __init__( *, require_hashes: bool = False, no_deps: bool = False, + disable_pip: bool = False, skip_editable: bool = False, index_url: str | None = None, extra_index_urls: list[str] = [], @@ -53,10 +60,14 @@ def __init__( `require_hashes` controls the hash policy: if `True`, dependency collection will fail unless all requirements include hashes. - `no_deps` controls the dependency resolution policy: if `True`, + `disable_pip` controls the dependency resolution policy: if `True`, dependency resolution is not performed and the inputs are checked and treated as "frozen". + `no_deps` controls whether dependency resolution can be disabled even without + hashed requirements (which implies a fully resolved requirements file): if `True`, + `disable_pip` is allowed without a hashed requirements file. + `skip_editable` controls whether requirements marked as "editable" are skipped. By default, editable requirements are not skipped. @@ -69,6 +80,7 @@ def __init__( self._filenames = filenames self._require_hashes = require_hashes self._no_deps = no_deps + self._disable_pip = disable_pip self._skip_editable = skip_editable self._index_url = index_url self._extra_index_urls = extra_index_urls @@ -123,9 +135,35 @@ def collect(self) -> Iterator[Dependency]: t.unlink() def _collect_from_files(self, filenames: list[Path]) -> Iterator[Dependency]: + # Figure out whether we have a fully resolved set of dependencies. + reqs: list[InstallRequirement] = [] + require_hashes: bool = self._require_hashes + for filename in filenames: + rf = RequirementsFile.from_file(filename) + if len(rf.invalid_lines) > 0: + invalid = rf.invalid_lines[0] + raise InvalidRequirementSpecifier( + f"requirement file {filename} contains invalid specifier at " + f"line {invalid.line_number}: {invalid.error_message}" + ) + + # If one or more requirements have a hash, this implies `--require-hashes`. + require_hashes = require_hashes or any(req.hash_options for req in rf.requirements) + reqs.extend(rf.requirements) + + # If the user has supplied `--no-deps` or there are hashed requirements, we should assume + # that we have a fully resolved set of dependencies and we should waste time by invoking + # `pip`. + if self._disable_pip: + if not self._no_deps and not require_hashes: + raise RequirementSourceError( + "the --disable-pip flag can only be used with a hashed requirements files or " + "if the --no-deps flag has been provided" + ) + yield from self._collect_preresolved_deps(iter(reqs), require_hashes) + return + ve_args = [] - if self._no_deps: - ve_args.append("--no-deps") if self._require_hashes: ve_args.append("--require-hashes") for filename in filenames: @@ -246,6 +284,61 @@ def _recover_files(self, tmp_files: list[IO[str]]) -> None: logger.warning(f"encountered an exception during file recovery: {e}") continue + def _collect_preresolved_deps( + self, reqs: Iterator[InstallRequirement], require_hashes: bool + ) -> Iterator[Dependency]: + """ + Collect pre-resolved (pinned) dependencies. + """ + req_names: set[str] = set() + for req in reqs: + if not req.hash_options and require_hashes: + raise RequirementSourceError(f"requirement {req.dumps()} does not contain a hash") + if req.req is None: + # PEP 508-style URL requirements don't have a pre-declared version, even + # when hashed; the `#egg=name==version` syntax is non-standard and not supported + # by `pip` itself. + # + # In this case, we can't audit the dependency so we should signal to the + # caller that we're skipping it. + yield SkippedDependency( + name=req.requirement_line.line, + skip_reason="could not deduce package version from URL requirement", + ) + continue + if self._skip_editable and req.is_editable: + yield SkippedDependency(name=req.name, skip_reason="requirement marked as editable") + if req.marker is not None and not req.marker.evaluate(): + continue # pragma: no cover + + # This means we have a duplicate requirement for the same package + if req.name in req_names: + raise RequirementSourceError( + f"package {req.name} has duplicate requirements: {str(req)}" + ) + req_names.add(req.name) + + # NOTE: URL dependencies cannot be pinned, so skipping them + # makes sense (under the same principle of skipping dependencies + # that can't be found on PyPI). This is also consistent with + # what `pip --no-deps` does (installs the URL dependency, but + # not any subdependencies). + if req.is_url: + yield SkippedDependency( + name=req.name, + skip_reason="URL requirements cannot be pinned to a specific package version", + ) + elif not req.specifier: + raise RequirementSourceError(f"requirement {req.name} is not pinned: {str(req)}") + else: + pinned_specifier = PINNED_SPECIFIER_RE.match(str(req.specifier)) + if pinned_specifier is None: + raise RequirementSourceError( + f"requirement {req.name} is not pinned to an exact version: {str(req)}" + ) + + yield ResolvedDependency(req.name, Version(pinned_specifier.group("version"))) + class RequirementSourceError(DependencySourceError): """A requirements-parsing specific `DependencySourceError`.""" diff --git a/test/dependency_source/test_requirement.py b/test/dependency_source/test_requirement.py index a5ac69ad..8838cf88 100644 --- a/test/dependency_source/test_requirement.py +++ b/test/dependency_source/test_requirement.py @@ -17,7 +17,7 @@ requirement, ) from pip_audit._fix import ResolvedFixVersion -from pip_audit._service import ResolvedDependency +from pip_audit._service import ResolvedDependency, SkippedDependency from pip_audit._state import AuditState from pip_audit._virtual_env import VirtualEnv, VirtualEnvError @@ -499,12 +499,25 @@ def test_requirement_source_require_hashes_incorrect_hash(req_file): list(source.collect()) -@pytest.mark.online -def test_requirement_source_no_deps(req_file): - source = _init_requirement([(req_file(), "flask==2.0.1")], no_deps=True) +def test_requirement_source_disable_pip_editable_skip(req_file): + source = _init_requirement( + [(req_file(), "-e file:flask.py#egg=flask==2.0.1")], + disable_pip=True, + no_deps=True, + skip_editable=True, + ) specs = list(source.collect()) - assert specs == [ResolvedDependency("Flask", Version("2.0.1"))] + assert SkippedDependency(name="flask", skip_reason="requirement marked as editable") in specs + + +def test_requirement_source_disable_pip_duplicate_dependencies(req_file): + source = _init_requirement( + [(req_file(), "flask==1.0\nflask==1.0")], disable_pip=True, no_deps=True + ) + + with pytest.raises(DependencySourceError): + list(source.collect()) def test_requirement_source_no_double_open(monkeypatch, req_file): @@ -706,3 +719,135 @@ def test_requirement_source_fix_invalid_lines(req_file): dep=ResolvedDependency(name="flask", version=Version("0.5")), version=Version("1.0") ) ) + + +def test_requirement_source_disable_pip(req_file): + source = _init_requirement([(req_file(), "flask==2.0.1")], disable_pip=True, no_deps=True) + + specs = list(source.collect()) + assert specs == [ResolvedDependency("flask", Version("2.0.1"))] + + +def test_requirement_source_disable_pip_without_no_deps(req_file): + # In order to use `--disable-pip`, the requirements file must either be hashed or `--no-deps` + # must be provided. + # + # Since neither is true, we expect a failure. + source = _init_requirement( + [(req_file(), "flask==2.0.1")], + disable_pip=True, + ) + + with pytest.raises( + DependencySourceError, + match="the --disable-pip flag can only be used with a hashed requirements files or if the " + "--no-deps flag has been provided", + ): + list(source.collect()) + + +def test_requirement_source_disable_pip_hashes_without_no_deps(req_file): + # In this case, `--no-deps` is not provided but since the requirements file is hashed, providing + # `--disable-pip` is valid. + source = _init_requirement( + [ + ( + req_file(), + "flask==2.0.1 " + "--hash=sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9", + ) + ], + disable_pip=True, + ) + + specs = list(source.collect()) + assert specs == [ResolvedDependency("flask", Version("2.0.1"))] + + +def test_requirement_source_disable_pip_incomplete_hashes(req_file): + # In this case, `--no-deps` is not provided but since the requirements file is hashed, providing + # `--disable-pip` is valid. + source = _init_requirement( + [ + ( + req_file(), + "flask==2.0.1 " + "--hash=sha256:a6209ca15eb63fc9385f38e452704113d679511d9574d09b2cf9183ae7d20dc9\n" + "requests==1.0", + ) + ], + disable_pip=True, + ) + + with pytest.raises( + DependencySourceError, match="requirement requests==1.0 does not contain a hash" + ): + list(source.collect()) + + +def test_requirement_source_disable_pip_unpinned(req_file): + source = _init_requirement( + [(req_file(), "flask\nrequests==1.0")], disable_pip=True, no_deps=True + ) + + # When dependency resolution is disabled, all requirements must be pinned. + with pytest.raises(DependencySourceError): + list(source.collect()) + + +def test_requirement_source_disable_pip_not_exact_version(req_file): + source = _init_requirement( + [(req_file(), "flask==1.0\nrequests>=1.0")], disable_pip=True, no_deps=True + ) + + # When dependency resolution is disabled, all requirements must be pinned. + with pytest.raises(DependencySourceError): + list(source.collect()) + + +def test_requirement_source_disable_pip_unpinned_url(req_file): + source = _init_requirement( + [ + ( + req_file(), + "https://github.com/pallets/flask/archive/refs/tags/2.0.1.tar.gz#egg=flask\n", + ) + ], + disable_pip=True, + no_deps=True, + ) + + assert list(source.collect()) == [ + SkippedDependency( + name="flask", + skip_reason="URL requirements cannot be pinned to a specific package version", + ) + ] + + +def test_requirement_source_disable_pip_editable_with_egg_fragment(req_file): + source = _init_requirement( + [(req_file(), "-e file:flask.py#egg=flask==2.0.1")], disable_pip=True, no_deps=True + ) + + specs = list(source.collect()) + assert ( + SkippedDependency( + name="flask", + skip_reason="URL requirements cannot be pinned to a specific package version", + ) + in specs + ) + + +def test_requirement_source_disable_pip_editable_without_egg_fragment(req_file): + source = _init_requirement([(req_file(), "-e file:flask.py")], disable_pip=True, no_deps=True) + + specs = list(source.collect()) + assert ( + SkippedDependency( + name="-e file:flask.py", + skip_reason="could not deduce package version from URL requirement", + ) + in specs + )