Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

locker: allow project dependencies to be retrieved #3002

Merged
merged 3 commits into from
Sep 30, 2020
Merged

locker: allow project dependencies to be retrieved #3002

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
14ff291
locker: allow project dependencies to be retrieved
abn Sep 30, 2020
b2b8b7d
export: ensure propagated markers are exported
abn Sep 30, 2020
6a9b4e3
Bump poetry-core to 1.0.0rc3
abn Sep 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 4 additions & 14 deletions poetry.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

82 changes: 82 additions & 0 deletions poetry/packages/locker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
import itertools
import json
import logging
import os
import re

from copy import deepcopy
from hashlib import sha256
from typing import Any
from typing import List
from typing import Optional

from tomlkit import array
from tomlkit import document
Expand Down Expand Up @@ -177,6 +181,84 @@ def locked_repository(

return packages

def get_project_dependencies(
self, project_requires, pinned_versions=False, with_nested=False
): # type: (List[Dependency], bool, bool) -> Any
packages = self.locked_repository().packages

# group packages entries by name, this is required because requirement might use
# different constraints
packages_by_name = {}
for pkg in packages:
if pkg.name not in packages_by_name:
packages_by_name[pkg.name] = []
packages_by_name[pkg.name].append(pkg)

def __get_locked_package(
_dependency,
): # type: (Dependency) -> Optional[Package]
"""
Internal helper to identify corresponding locked package using dependency
version constraints.
"""
for _package in packages_by_name.get(_dependency.name, []):
if _dependency.constraint.allows(_package.version):
return _package
return None

project_level_dependencies = set()
dependencies = []

for dependency in project_requires:
dependency = deepcopy(dependency)
if pinned_versions:
locked_package = __get_locked_package(dependency)
if locked_package:
dependency.set_constraint(locked_package.to_dependency().constraint)
project_level_dependencies.add(dependency.name)
dependencies.append(dependency)

if not with_nested:
# return only with project level dependencies
return dependencies

nested_dependencies = list()

for pkg in packages: # type: Package
for requirement in pkg.requires: # type: Dependency
if requirement.name in project_level_dependencies:
# project level dependencies take precedence
continue

if pinned_versions:
requirement.set_constraint(
__get_locked_package(requirement).to_dependency().constraint
)

# dependencies use extra to indicate that it was activated via parent
# package's extras
marker = requirement.marker.without_extras()
for project_requirement in project_requires:
if (
pkg.name == project_requirement.name
and project_requirement.constraint.allows(pkg.version)
):
requirement.marker = marker.intersect(
project_requirement.marker
)
break
else:
# this dependency was not from a project requirement
requirement.marker = marker.intersect(pkg.marker)

if requirement not in nested_dependencies:
nested_dependencies.append(requirement)

return sorted(
itertools.chain(dependencies, nested_dependencies),
key=lambda x: x.name.lower(),
)

def set_lock_data(self, root, packages): # type: (...) -> bool
files = table()
packages = self._lock_packages(packages)
Expand Down
96 changes: 35 additions & 61 deletions poetry/utils/exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
import os

from typing import Union

from clikit.api.io import IO

from poetry.core.packages.directory_dependency import DirectoryDependency
from poetry.core.packages.file_dependency import FileDependency
from poetry.core.packages.url_dependency import URLDependency
from poetry.core.packages.vcs_dependency import VCSDependency
from poetry.poetry import Poetry
from poetry.utils._compat import Path
from poetry.utils._compat import decode
Expand Down Expand Up @@ -60,75 +54,54 @@ def _export_requirements_txt(
): # type: (Path, Union[IO, str], bool, bool, bool) -> None
indexes = set()
content = ""
packages = self._poetry.locker.locked_repository(dev).packages
repository = self._poetry.locker.locked_repository(dev)

# Build a set of all packages required by our selected extras
extra_package_names = set(
get_extra_package_names(
packages, self._poetry.locker.lock_data.get("extras", {}), extras or ()
repository.packages,
self._poetry.locker.lock_data.get("extras", {}),
extras or (),
)
)

for package in sorted(packages, key=lambda p: p.name):
dependency_lines = set()

for dependency in self._poetry.locker.get_project_dependencies(
project_requires=self._poetry.package.requires
if not dev
else self._poetry.package.all_requires,
with_nested=True,
):
package = repository.find_packages(dependency=dependency)[0]

# If a package is optional and we haven't opted in to it, continue
if package.optional and package.name not in extra_package_names:
continue

if package.source_type == "git":
dependency = VCSDependency(
package.name,
package.source_type,
package.source_url,
package.source_reference,
)
dependency.marker = package.marker
line = "-e git+{}@{}#egg={}".format(
package.source_url, package.source_reference, package.name
)
elif package.source_type in ["directory", "file", "url"]:
url = package.source_url
if package.source_type == "file":
dependency = FileDependency(
package.name,
Path(package.source_url),
base=self._poetry.locker.lock.path.parent,
)
url = Path(
os.path.relpath(
url, self._poetry.locker.lock.path.parent.as_posix()
)
).as_posix()
elif package.source_type == "directory":
dependency = DirectoryDependency(
package.name,
Path(package.source_url),
base=self._poetry.locker.lock.path.parent,
)
url = Path(
os.path.relpath(
url, self._poetry.locker.lock.path.parent.as_posix()
)
).as_posix()
else:
dependency = URLDependency(package.name, package.source_url)
line = ""

dependency.marker = package.marker
if package.develop:
line += "-e "

line = "{}".format(url)
if package.develop and package.source_type == "directory":
line = "-e " + line
requirement = dependency.to_pep_508(with_extras=False)
is_direct_reference = (
dependency.is_vcs()
or dependency.is_url()
or dependency.is_file()
or dependency.is_directory()
)

if is_direct_reference:
line = requirement
else:
dependency = package.to_dependency()
line = "{}=={}".format(package.name, package.version)
if ";" in requirement:
markers = requirement.split(";", 1)[1].strip()
if markers:
line += "; {}".format(markers)

requirement = dependency.to_pep_508()
if ";" in requirement:
line += "; {}".format(requirement.split(";")[1].strip())

if (
package.source_type not in {"git", "directory", "file", "url"}
and package.source_url
):
if not is_direct_reference and package.source_url:
indexes.add(package.source_url)

if package.files and with_hashes:
Expand All @@ -150,9 +123,10 @@ def _export_requirements_txt(
line += " --hash={}{}".format(
h, " \\\n" if i < len(hashes) - 1 else ""
)
dependency_lines.add(line)

line += "\n"
content += line
content += "\n".join(sorted(dependency_lines))
content += "\n"

if indexes:
# If we have extra indexes, we add them to the beginning of the output
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ classifiers = [
[tool.poetry.dependencies]
python = "~2.7 || ^3.5"

poetry-core = "^1.0.0rc2"
poetry-core = "^1.0.0rc3"
cleo = "^0.8.1"
clikit = "^0.6.2"
crashtest = { version = "^0.3.0", python = "^3.6" }
Expand Down
Loading