Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support adding new requirements in a lock update. #1797

Merged
merged 2 commits into from
Jun 2, 2022
Merged

Support adding new requirements in a lock update. #1797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0566e89
Support adding new requirements ina l ock update.
jsirois Jun 1, 2022
b286e03
Prove `-p new-req` top-level requirements promotion is properly scoped.
jsirois Jun 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 59 additions & 49 deletions pex/cli/commands/lock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,15 @@

import sys
from argparse import ArgumentParser, _ActionsContainer
from collections import OrderedDict, defaultdict
from collections import defaultdict

from pex.argparse import HandleBoolAction
from pex.cli.command import BuildTimeCommand
from pex.commands.command import JsonMixin, OutputMixin
from pex.common import pluralize
from pex.dist_metadata import Requirement, RequirementParseError
from pex.enum import Enum
from pex.orderedset import OrderedSet
from pex.resolve import requirement_options, resolver_options, target_options
from pex.resolve.locked_resolve import LockConfiguration, LockedResolve, LockStyle
from pex.resolve.lockfile import json_codec
Expand Down Expand Up @@ -167,7 +168,11 @@ def _add_update_arguments(cls, update_parser):
action="append",
default=[],
type=str,
help="Just attempt to update these projects in the lock, leaving all others unchanged.",
help=(
"Just attempt to update these projects in the lock, leaving all others unchanged. "
"If the projects aren't already in the lock, attempt to add them as top-level"
"requirements leaving all others unchanged."
),
)
update_parser.add_argument(
"--strict",
Expand Down Expand Up @@ -373,28 +378,6 @@ def _update(self):
return Error("Failed to parse project requirement to update: {err}".format(err=e))

lock_file_path, lock_file = self._load_lockfile()

if updates:
updates_by_project_name = OrderedDict(
(update.project_name, update) for update in updates
)
for locked_resolve in lock_file.locked_resolves:
for locked_requirement in locked_resolve.locked_requirements:
updates_by_project_name.pop(locked_requirement.pin.project_name, None)
if not updates_by_project_name:
break
if updates_by_project_name:
return Error(
"The following updates were requested but there were no matching locked "
"requirements found in {lock_file}:\n{updates}".format(
lock_file=lock_file_path,
updates="\n".join(
"+ {update}".format(update=update)
for update in updates_by_project_name.values()
),
)
)

lock_updater = LockUpdater.create(
lock_file=lock_file,
repos_configuration=resolver_options.create_repos_configuration(self.options),
Expand Down Expand Up @@ -469,17 +452,29 @@ def _update(self):
for project_name, version_update in resolve_update.updates.items():
if version_update:
performed_update = True
print(
"{lead_in} {project_name} from {original_version} to {updated_version} in "
"lock generated by {platform}.".format(
lead_in="Would update" if dry_run else "Updated",
project_name=project_name,
original_version=version_update.original,
updated_version=version_update.updated,
platform=platform,
),
file=output,
)
if version_update.original:
print(
"{lead_in} {project_name} from {original_version} to {updated_version} "
"in lock generated by {platform}.".format(
lead_in="Would update" if dry_run else "Updated",
project_name=project_name,
original_version=version_update.original,
updated_version=version_update.updated,
platform=platform,
),
file=output,
)
else:
print(
"{lead_in} {project_name} {updated_version} to lock generated by "
"{platform}.".format(
lead_in="Would add" if dry_run else "Added",
project_name=project_name,
updated_version=version_update.updated,
platform=platform,
),
file=output,
)
else:
print(
"There {tense} no updates for {project_name} in lock generated by "
Expand All @@ -491,22 +486,37 @@ def _update(self):
file=output,
)
if performed_update:
original_locked_project_names = {
locked_requirement.pin.project_name
for locked_resolve in lock_file.locked_resolves
for locked_requirement in locked_resolve.locked_requirements
}
new_requirements = OrderedSet(
jsirois marked this conversation as resolved.
Show resolved Hide resolved
update
for update in updates
if update.project_name not in original_locked_project_names
)
constraints_by_project_name.update(
(constraint.project_name, constraint) for constraint in updates
)

if performed_update and not dry_run:
with open(lock_file_path, "w") as fp:
self._dump_lockfile(
lock_file=attr.evolve(
lock_file,
pex_version=__version__,
constraints=SortedTuple(constraints_by_project_name.values()),
locked_resolves=SortedTuple(
resolve_update.updated_resolve
for resolve_update in lock_update.resolves
for requirement in new_requirements:
constraints_by_project_name.pop(requirement.project_name, None)
requirements = OrderedSet(lock_file.requirements)
requirements.update(new_requirements)

if not dry_run:
with open(lock_file_path, "w") as fp:
self._dump_lockfile(
lock_file=attr.evolve(
lock_file,
pex_version=__version__,
requirements=SortedTuple(requirements, key=str),
constraints=SortedTuple(constraints_by_project_name.values(), key=str),
locked_resolves=SortedTuple(
resolve_update.updated_resolve
for resolve_update in lock_update.resolves
),
),
),
output=fp,
)
output=fp,
)
return Ok()
44 changes: 29 additions & 15 deletions pex/resolve/lockfile/updater.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from pex.common import pluralize
from pex.dist_metadata import Requirement
from pex.network_configuration import NetworkConfiguration
from pex.orderedset import OrderedSet
from pex.pep_440 import Version
from pex.pep_503 import ProjectName
from pex.resolve.locked_resolve import LockConfiguration, LockedRequirement, LockedResolve
Expand Down Expand Up @@ -38,7 +39,7 @@

@attr.s(frozen=True)
class VersionUpdate(object):
original = attr.ib() # type: Version
original = attr.ib() # type: Optional[Version]
updated = attr.ib() # type: Version


Expand Down Expand Up @@ -82,7 +83,9 @@ def create(
constraint.project_name: constraint for constraint in constraints
} # type: Mapping[ProjectName, Requirement]

update_constraints_by_project_name = {} # type: Dict[ProjectName, Requirement]
update_constraints_by_project_name = (
OrderedDict()
) # type: OrderedDict[ProjectName, Requirement]
for update in updates:
project_name = update.project_name
original_constraint = original_constraints.get(project_name)
Expand All @@ -107,28 +110,36 @@ def create(
pip_configuration = attr.ib() # type: PipConfiguration

@contextmanager
def _calculate_update_constraints(self, locked_resolve):
# type: (LockedResolve) -> Iterator[Optional[Iterable[str]]]
def _calculate_requirement_configuration(self, locked_resolve):
# type: (LockedResolve) -> Iterator[RequirementConfiguration]
if not self.update_constraints_by_project_name:
yield None
yield RequirementConfiguration(requirements=self.original_requirements)
return

requirements = OrderedSet(self.original_requirements)
constraints = []
update_constraints_by_project_name = OrderedDict(self.update_constraints_by_project_name)
for locked_requirement in locked_resolve.locked_requirements:
pin = locked_requirement.pin
constraint = self.update_constraints_by_project_name.get(
constraint = update_constraints_by_project_name.pop(
pin.project_name, pin.as_requirement()
)
constraints.append(str(constraint))

# Any update constraints remaining are new requirements to resolve.
requirements.update(str(req) for req in update_constraints_by_project_name.values())

if not constraints:
yield None
yield RequirementConfiguration(requirements=requirements)
return

with named_temporary_file(prefix="lock_update.", suffix=".constraints.txt", mode="w") as fp:
fp.write(os.linesep.join(constraints))
fp.flush()
try:
yield [fp.name]
yield RequirementConfiguration(
requirements=requirements, constraint_files=[fp.name]
)
except ResultError as e:
logger.error(
"The following lock update constraints could not be satisfied:\n"
Expand All @@ -143,14 +154,11 @@ def update_resolve(
):
# type: (...) -> Union[ResolveUpdate, Error]

with self._calculate_update_constraints(locked_resolve) as constraints_files:
with self._calculate_requirement_configuration(locked_resolve) as requirement_configuration:
updated_lock_file = try_(
create(
lock_configuration=self.lock_configuration,
requirement_configuration=RequirementConfiguration(
requirements=self.original_requirements,
constraint_files=constraints_files,
),
requirement_configuration=requirement_configuration,
targets=targets,
pip_configuration=self.pip_configuration,
)
Expand All @@ -168,7 +176,7 @@ def update_resolve(
for locked_requirement in locked_resolve.locked_requirements:
original_pin = locked_requirement.pin
project_name = original_pin.project_name
updated_requirement = updated_requirements_by_project_name.get(project_name)
updated_requirement = updated_requirements_by_project_name.pop(project_name, None)
if not updated_requirement:
continue
updated_pin = updated_requirement.pin
Expand All @@ -189,10 +197,16 @@ def update_resolve(
elif project_name in self.update_constraints_by_project_name:
updates[project_name] = None

# Anything left was an addition.
updates.update(
(project_name, VersionUpdate(original=None, updated=locked_requirement.pin.version))
for project_name, locked_requirement in updated_requirements_by_project_name.items()
)

return ResolveUpdate(
updated_resolve=attr.evolve(
locked_resolve,
locked_requirements=SortedTuple(updated_requirements_by_project_name.values()),
locked_requirements=SortedTuple(updated_resolve.locked_requirements),
),
updates=updates,
)
Expand Down
Loading