diff --git a/poetry.lock b/poetry.lock index a3b7fdd7548..cca9e6bdf05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -458,14 +458,6 @@ python-versions = "*" [package.dependencies] six = ">=1.0.0,<2.0.0" -[[package]] -name = "more-itertools" -version = "8.4.0" -description = "More routines for operating on iterables, beyond itertools" -category = "dev" -optional = false -python-versions = ">=3.5" - [[package]] name = "more-itertools" version = "8.5.0" @@ -560,7 +552,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "poetry-core" -version = "1.0.0rc2" +version = "1.0.0rc3" description = "Poetry PEP 517 Build Backend" category = "main" optional = false @@ -982,7 +974,7 @@ testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] [metadata] lock-version = "1.1" python-versions = "~2.7 || ^3.5" -content-hash = "147ef9a61f96f4943c6c8f48cb6d0863b6f614612cf658dbb60e0546e4e7d349" +content-hash = "d97a19b321b206d7098b165c7285e3a3db28c3ae9b6a17349dd51b605a55223b" [metadata.files] appdirs = [ @@ -1226,8 +1218,6 @@ more-itertools = [ {file = "more-itertools-5.0.0.tar.gz", hash = "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4"}, {file = "more_itertools-5.0.0-py2-none-any.whl", hash = "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc"}, {file = "more_itertools-5.0.0-py3-none-any.whl", hash = "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"}, - {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, - {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, {file = "more-itertools-8.5.0.tar.gz", hash = "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20"}, {file = "more_itertools-8.5.0-py3-none-any.whl", hash = "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c"}, ] @@ -1280,8 +1270,8 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] poetry-core = [ - {file = "poetry-core-1.0.0rc2.tar.gz", hash = "sha256:abbe4059433e6d51aff024986b19919319e084592203f17e5354e7c0a7dee6f4"}, - {file = "poetry_core-1.0.0rc2-py2.py3-none-any.whl", hash = "sha256:f536d59ee0be81f7c689a1a30e9894cb24abeb23f4ff4e2130583fca9863272d"}, + {file = "poetry-core-1.0.0rc3.tar.gz", hash = "sha256:0e3d23a4c5acc14c5f1703b12645692bf24461c8c8e3fff32ca80a5462beccdf"}, + {file = "poetry_core-1.0.0rc3-py2.py3-none-any.whl", hash = "sha256:2c9ee2b3f7b40047bafdc2239fbb9de73637edc07a9697a3a66ac15fe6b040f5"}, ] pre-commit = [ {file = "pre_commit-2.7.1-py2.py3-none-any.whl", hash = "sha256:810aef2a2ba4f31eed1941fc270e72696a1ad5590b9751839c90807d0fff6b9a"}, diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 393f60eb6f9..2b696b3450a 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -1,10 +1,14 @@ +import itertools import json import logging import os import re +from copy import deepcopy from hashlib import sha256 +from typing import Any from typing import List +from typing import Optional from tomlkit import array from tomlkit import document @@ -177,6 +181,84 @@ def locked_repository( return packages + def get_project_dependencies( + self, project_requires, pinned_versions=False, with_nested=False + ): # type: (List[Dependency], bool, bool) -> Any + packages = self.locked_repository().packages + + # group packages entries by name, this is required because requirement might use + # different constraints + packages_by_name = {} + for pkg in packages: + if pkg.name not in packages_by_name: + packages_by_name[pkg.name] = [] + packages_by_name[pkg.name].append(pkg) + + def __get_locked_package( + _dependency, + ): # type: (Dependency) -> Optional[Package] + """ + Internal helper to identify corresponding locked package using dependency + version constraints. + """ + for _package in packages_by_name.get(_dependency.name, []): + if _dependency.constraint.allows(_package.version): + return _package + return None + + project_level_dependencies = set() + dependencies = [] + + for dependency in project_requires: + dependency = deepcopy(dependency) + if pinned_versions: + locked_package = __get_locked_package(dependency) + if locked_package: + dependency.set_constraint(locked_package.to_dependency().constraint) + project_level_dependencies.add(dependency.name) + dependencies.append(dependency) + + if not with_nested: + # return only with project level dependencies + return dependencies + + nested_dependencies = list() + + for pkg in packages: # type: Package + for requirement in pkg.requires: # type: Dependency + if requirement.name in project_level_dependencies: + # project level dependencies take precedence + continue + + if pinned_versions: + requirement.set_constraint( + __get_locked_package(requirement).to_dependency().constraint + ) + + # dependencies use extra to indicate that it was activated via parent + # package's extras + marker = requirement.marker.without_extras() + for project_requirement in project_requires: + if ( + pkg.name == project_requirement.name + and project_requirement.constraint.allows(pkg.version) + ): + requirement.marker = marker.intersect( + project_requirement.marker + ) + break + else: + # this dependency was not from a project requirement + requirement.marker = marker.intersect(pkg.marker) + + if requirement not in nested_dependencies: + nested_dependencies.append(requirement) + + return sorted( + itertools.chain(dependencies, nested_dependencies), + key=lambda x: x.name.lower(), + ) + def set_lock_data(self, root, packages): # type: (...) -> bool files = table() packages = self._lock_packages(packages) diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index e8c91e66e0b..19b48452346 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -1,13 +1,7 @@ -import os - from typing import Union from clikit.api.io import IO -from poetry.core.packages.directory_dependency import DirectoryDependency -from poetry.core.packages.file_dependency import FileDependency -from poetry.core.packages.url_dependency import URLDependency -from poetry.core.packages.vcs_dependency import VCSDependency from poetry.poetry import Poetry from poetry.utils._compat import Path from poetry.utils._compat import decode @@ -60,75 +54,54 @@ def _export_requirements_txt( ): # type: (Path, Union[IO, str], bool, bool, bool) -> None indexes = set() content = "" - packages = self._poetry.locker.locked_repository(dev).packages + repository = self._poetry.locker.locked_repository(dev) # Build a set of all packages required by our selected extras extra_package_names = set( get_extra_package_names( - packages, self._poetry.locker.lock_data.get("extras", {}), extras or () + repository.packages, + self._poetry.locker.lock_data.get("extras", {}), + extras or (), ) ) - for package in sorted(packages, key=lambda p: p.name): + dependency_lines = set() + + for dependency in self._poetry.locker.get_project_dependencies( + project_requires=self._poetry.package.requires + if not dev + else self._poetry.package.all_requires, + with_nested=True, + ): + package = repository.find_packages(dependency=dependency)[0] + # If a package is optional and we haven't opted in to it, continue if package.optional and package.name not in extra_package_names: continue - if package.source_type == "git": - dependency = VCSDependency( - package.name, - package.source_type, - package.source_url, - package.source_reference, - ) - dependency.marker = package.marker - line = "-e git+{}@{}#egg={}".format( - package.source_url, package.source_reference, package.name - ) - elif package.source_type in ["directory", "file", "url"]: - url = package.source_url - if package.source_type == "file": - dependency = FileDependency( - package.name, - Path(package.source_url), - base=self._poetry.locker.lock.path.parent, - ) - url = Path( - os.path.relpath( - url, self._poetry.locker.lock.path.parent.as_posix() - ) - ).as_posix() - elif package.source_type == "directory": - dependency = DirectoryDependency( - package.name, - Path(package.source_url), - base=self._poetry.locker.lock.path.parent, - ) - url = Path( - os.path.relpath( - url, self._poetry.locker.lock.path.parent.as_posix() - ) - ).as_posix() - else: - dependency = URLDependency(package.name, package.source_url) + line = "" - dependency.marker = package.marker + if package.develop: + line += "-e " - line = "{}".format(url) - if package.develop and package.source_type == "directory": - line = "-e " + line + requirement = dependency.to_pep_508(with_extras=False) + is_direct_reference = ( + dependency.is_vcs() + or dependency.is_url() + or dependency.is_file() + or dependency.is_directory() + ) + + if is_direct_reference: + line = requirement else: - dependency = package.to_dependency() line = "{}=={}".format(package.name, package.version) + if ";" in requirement: + markers = requirement.split(";", 1)[1].strip() + if markers: + line += "; {}".format(markers) - requirement = dependency.to_pep_508() - if ";" in requirement: - line += "; {}".format(requirement.split(";")[1].strip()) - - if ( - package.source_type not in {"git", "directory", "file", "url"} - and package.source_url - ): + if not is_direct_reference and package.source_url: indexes.add(package.source_url) if package.files and with_hashes: @@ -150,9 +123,10 @@ def _export_requirements_txt( line += " --hash={}{}".format( h, " \\\n" if i < len(hashes) - 1 else "" ) + dependency_lines.add(line) - line += "\n" - content += line + content += "\n".join(sorted(dependency_lines)) + content += "\n" if indexes: # If we have extra indexes, we add them to the beginning of the output diff --git a/pyproject.toml b/pyproject.toml index dd556a305ac..a851e086588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ [tool.poetry.dependencies] python = "~2.7 || ^3.5" -poetry-core = "^1.0.0rc2" +poetry-core = "^1.0.0rc3" cleo = "^0.8.1" clikit = "^0.6.2" crashtest = { version = "^0.3.0", python = "^3.6" } diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index ede834a6bdf..c2eee0d1f11 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -34,6 +34,18 @@ def _get_content_hash(self): return "123456789" +@pytest.fixture +def working_directory(): + return Path(__file__).parent.parent.parent + + +@pytest.fixture(autouse=True) +def mock_path_cwd(mocker, working_directory): + yield mocker.patch( + "poetry.core.utils._compat.Path.cwd", return_value=working_directory + ) + + @pytest.fixture() def locker(): return Locker() @@ -47,7 +59,19 @@ def poetry(fixture_dir, locker): return p -def test_exporter_can_export_requirements_txt_with_standard_packages(tmp_dir, poetry): +def set_package_requires(poetry): + packages = poetry.locker.locked_repository(with_dev_reqs=True).packages + poetry.package.requires = [ + pkg.to_dependency() for pkg in packages if pkg.category == "main" + ] + poetry.package.dev_requires = [ + pkg.to_dependency() for pkg in packages if pkg.category == "dev" + ] + + +def test_exporter_can_export_requirements_txt_with_standard_packages( + tmp_dir, poetry, mocker +): poetry.locker.mock_lock_data( { "package": [ @@ -73,6 +97,8 @@ def test_exporter_can_export_requirements_txt_with_standard_packages(tmp_dir, po }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -110,14 +136,24 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers "python-versions": "*", "marker": "extra =='foo'", }, + { + "name": "baz", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "sys_platform == 'win32'", + }, ], "metadata": { "python-versions": "*", "content-hash": "123456789", - "hashes": {"foo": [], "bar": []}, + "hashes": {"foo": [], "bar": [], "baz": []}, }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -126,7 +162,8 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers content = f.read() expected = """\ -bar==4.5.6; extra == "foo" +bar==4.5.6 +baz==7.8.9; sys_platform == "win32" foo==1.2.3; python_version < "3.7" """ @@ -161,6 +198,8 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -206,6 +245,8 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes_ }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export( @@ -251,6 +292,8 @@ def test_exporter_exports_requirements_txt_without_dev_packages_by_default( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -294,6 +337,8 @@ def test_exporter_exports_requirements_txt_with_dev_packages_if_opted_in( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) @@ -337,6 +382,8 @@ def test_exporter_exports_requirements_txt_without_optional_packages(tmp_dir, po }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) @@ -389,6 +436,8 @@ def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( "extras": {"feature_bar": ["bar"]}, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export( @@ -438,6 +487,8 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry) }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -446,7 +497,7 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry) content = f.read() expected = """\ --e git+https://github.com/foo/foo.git@123456#egg=foo +foo @ git+https://github.com/foo/foo.git@123456 """ assert expected == content @@ -479,6 +530,8 @@ def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -487,13 +540,15 @@ def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( content = f.read() expected = """\ --e git+https://github.com/foo/foo.git@123456#egg=foo; python_version < "3.7" +foo @ git+https://github.com/foo/foo.git@123456 ; python_version < "3.7" """ assert expected == content -def test_exporter_can_export_requirements_txt_with_directory_packages(tmp_dir, poetry): +def test_exporter_can_export_requirements_txt_with_directory_packages( + tmp_dir, poetry, working_directory +): poetry.locker.mock_lock_data( { "package": [ @@ -517,6 +572,8 @@ def test_exporter_can_export_requirements_txt_with_directory_packages(tmp_dir, p }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -525,14 +582,16 @@ def test_exporter_can_export_requirements_txt_with_directory_packages(tmp_dir, p content = f.read() expected = """\ --e tests/fixtures/sample_project -""" +foo @ {}/tests/fixtures/sample_project +""".format( + working_directory.as_posix() + ) assert expected == content def test_exporter_can_export_requirements_txt_with_directory_packages_and_markers( - tmp_dir, poetry + tmp_dir, poetry, working_directory ): poetry.locker.mock_lock_data( { @@ -558,6 +617,8 @@ def test_exporter_can_export_requirements_txt_with_directory_packages_and_marker }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -566,13 +627,17 @@ def test_exporter_can_export_requirements_txt_with_directory_packages_and_marker content = f.read() expected = """\ --e tests/fixtures/sample_project; python_version < "3.7" -""" +foo @ {}/tests/fixtures/sample_project; python_version < "3.7" +""".format( + working_directory.as_posix() + ) assert expected == content -def test_exporter_can_export_requirements_txt_with_file_packages(tmp_dir, poetry): +def test_exporter_can_export_requirements_txt_with_file_packages( + tmp_dir, poetry, working_directory +): poetry.locker.mock_lock_data( { "package": [ @@ -596,6 +661,8 @@ def test_exporter_can_export_requirements_txt_with_file_packages(tmp_dir, poetry }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -604,14 +671,16 @@ def test_exporter_can_export_requirements_txt_with_file_packages(tmp_dir, poetry content = f.read() expected = """\ -tests/fixtures/distributions/demo-0.1.0.tar.gz -""" +foo @ {}/tests/fixtures/distributions/demo-0.1.0.tar.gz +""".format( + working_directory.as_uri() + ) assert expected == content def test_exporter_can_export_requirements_txt_with_file_packages_and_markers( - tmp_dir, poetry + tmp_dir, poetry, working_directory ): poetry.locker.mock_lock_data( { @@ -637,6 +706,8 @@ def test_exporter_can_export_requirements_txt_with_file_packages_and_markers( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") @@ -645,8 +716,10 @@ def test_exporter_can_export_requirements_txt_with_file_packages_and_markers( content = f.read() expected = """\ -tests/fixtures/distributions/demo-0.1.0.tar.gz; python_version < "3.7" -""" +foo @ {}/tests/fixtures/distributions/demo-0.1.0.tar.gz; python_version < "3.7" +""".format( + working_directory.as_uri() + ) assert expected == content @@ -685,6 +758,8 @@ def test_exporter_exports_requirements_txt_with_legacy_packages(tmp_dir, poetry) }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) @@ -758,6 +833,8 @@ def test_exporter_exports_requirements_txt_with_legacy_packages_and_duplicate_so }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True) @@ -822,6 +899,8 @@ def test_exporter_exports_requirements_txt_with_legacy_packages_and_credentials( }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export( @@ -873,6 +952,8 @@ def test_exporter_exports_requirements_txt_to_standard_output(tmp_dir, poetry, c }, } ) + set_package_requires(poetry) + exporter = Exporter(poetry) exporter.export("requirements.txt", Path(tmp_dir), sys.stdout)