Skip to content

Commit

Permalink
feat: use ArtifactCache in get_package_from_url
Browse files Browse the repository at this point in the history
  • Loading branch information
ralbertazzi committed Apr 19, 2023
1 parent 6b4b539 commit 43895af
Show file tree
Hide file tree
Showing 7 changed files with 75 additions and 19 deletions.
3 changes: 3 additions & 0 deletions src/poetry/console/commands/init.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,14 +434,17 @@ def _parse_requirements(self, requirements: list[str]) -> list[dict[str, Any]]:

try:
cwd = self.poetry.file.parent
artifact_cache = self.poetry.pool.artifact_cache
except (PyProjectException, RuntimeError):
cwd = Path.cwd()
artifact_cache = None

return [
parse_dependency_specification(
requirement=requirement,
env=self.env if isinstance(self, EnvCommand) else None,
cwd=cwd,
artifact_cache=artifact_cache,
)
for requirement in requirements
]
Expand Down
14 changes: 12 additions & 2 deletions src/poetry/console/commands/show.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from poetry.console.commands.env_command import EnvCommand
from poetry.console.commands.group_command import GroupCommand
from poetry.utils.cache import ArtifactCache


if TYPE_CHECKING:
Expand Down Expand Up @@ -213,7 +214,12 @@ def _display_packages_information(
from poetry.utils.helpers import get_package_version_display_string

locked_packages = locked_repository.packages
pool = RepositoryPool(ignore_repository_names=True)
pool = RepositoryPool(
ignore_repository_names=True,
artifact_cache=ArtifactCache(
cache_dir=self.poetry.config.artifacts_cache_directory
),
)
pool.add_repository(locked_repository)
solver = Solver(
root,
Expand Down Expand Up @@ -547,7 +553,11 @@ def find_latest_package(
if package.is_direct_origin():
for dep in requires:
if dep.name == package.name and dep.source_type == package.source_type:
provider = Provider(root, self.poetry.pool, NullIO())
provider = Provider(
root,
self.poetry.pool,
NullIO(),
)
return provider.search_for_direct_origin_dependency(dep)

allow_prereleases = False
Expand Down
13 changes: 8 additions & 5 deletions src/poetry/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from poetry.plugins.plugin_manager import PluginManager
from poetry.poetry import Poetry
from poetry.toml.file import TOMLFile
from poetry.utils.cache import ArtifactCache


if TYPE_CHECKING:
Expand Down Expand Up @@ -118,7 +119,7 @@ def get_package(cls, name: str, version: str) -> ProjectPackage:
@classmethod
def create_pool(
cls,
auth_config: Config,
config: Config,
sources: Iterable[dict[str, Any]] = (),
io: IO | None = None,
disable_cache: bool = False,
Expand All @@ -132,11 +133,13 @@ def create_pool(
if disable_cache:
logger.debug("Disabling source caches")

pool = RepositoryPool()
pool = RepositoryPool(
artifact_cache=ArtifactCache(cache_dir=config.artifacts_cache_directory)
)

for source in sources:
repository = cls.create_package_source(
source, auth_config, disable_cache=disable_cache
source, config, disable_cache=disable_cache
)
priority = Priority[source.get("priority", Priority.PRIMARY.name).upper()]
if "default" in source or "secondary" in source:
Expand Down Expand Up @@ -184,7 +187,7 @@ def create_pool(

@classmethod
def create_package_source(
cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False
cls, source: dict[str, str], config: Config, disable_cache: bool = False
) -> LegacyRepository:
from poetry.repositories.legacy_repository import LegacyRepository
from poetry.repositories.single_page_repository import SinglePageRepository
Expand All @@ -206,7 +209,7 @@ def create_package_source(
return repository_class(
name,
url,
config=auth_config,
config=config,
disable_cache=disable_cache,
)

Expand Down
6 changes: 5 additions & 1 deletion src/poetry/installation/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from poetry.repositories import RepositoryPool
from poetry.repositories.installed_repository import InstalledRepository
from poetry.repositories.lockfile_repository import LockfileRepository
from poetry.utils.cache import ArtifactCache
from poetry.utils.extras import get_extra_package_names
from poetry.utils.helpers import pluralize

Expand Down Expand Up @@ -81,6 +82,7 @@ def __init__(
installed = self._get_installed()

self._installed_repository = installed
self._artifact_cache = ArtifactCache(cache_dir=config.artifacts_cache_directory)

@property
def executor(self) -> Executor:
Expand Down Expand Up @@ -314,7 +316,9 @@ def _do_install(self) -> int:
)

# We resolve again by only using the lock file
pool = RepositoryPool(ignore_repository_names=True)
pool = RepositoryPool(
ignore_repository_names=True, artifact_cache=self._artifact_cache
)

# Making a new repo containing the packages
# newly resolved and the ones from the current lock file
Expand Down
34 changes: 27 additions & 7 deletions src/poetry/puzzle/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from cleo.ui.progress_indicator import ProgressIndicator
from poetry.core.constraints.version import EmptyConstraint
from poetry.core.constraints.version import Version
from poetry.core.packages.utils.link import Link
from poetry.core.packages.utils.utils import get_python_constraint_from_marker
from poetry.core.version.markers import AnyMarker
from poetry.core.version.markers import EmptyMarker
Expand Down Expand Up @@ -56,6 +57,7 @@
from poetry.core.version.markers import BaseMarker

from poetry.repositories import RepositoryPool
from poetry.utils.cache import ArtifactCache
from poetry.utils.env import Env


Expand Down Expand Up @@ -439,7 +441,7 @@ def get_package_from_directory(cls, directory: Path) -> Package:
return PackageInfo.from_directory(path=directory).to_package(root_dir=directory)

def _search_for_url(self, dependency: URLDependency) -> Package:
package = self.get_package_from_url(dependency.url)
package = self.get_package_from_url(dependency.url, self._pool.artifact_cache)

self.validate_package_for_dependency(dependency=dependency, package=package)

Expand All @@ -454,15 +456,33 @@ def _search_for_url(self, dependency: URLDependency) -> Package:
return package

@classmethod
def get_package_from_url(cls, url: str) -> Package:
def get_package_from_url(
cls, url: str, artifact_cache: ArtifactCache | None = None
) -> Package:
file_name = os.path.basename(urllib.parse.urlparse(url).path)
with tempfile.TemporaryDirectory() as temp_dir:
dest = Path(temp_dir) / file_name
download_file(url, dest)
package = cls.get_package_from_file(dest)
link = Link(url)
artifact = (
artifact_cache.get_cached_archive_for_link(Link(url), strict=True)
if artifact_cache
else None
)

with tempfile.TemporaryDirectory() as tmp_dir:
if not artifact:
if artifact_cache:
artifact = (
artifact_cache.get_cache_directory_for_link(link) / file_name
)
artifact.parent.mkdir(parents=True, exist_ok=True)
else:
artifact = Path(tmp_dir) / file_name

download_file(url, artifact)

package = cls.get_package_from_file(artifact)

package.files = [
{"file": file_name, "hash": "sha256:" + get_file_hash(dest)}
{"file": file_name, "hash": "sha256:" + get_file_hash(artifact)}
]

package._source_type = "url"
Expand Down
8 changes: 8 additions & 0 deletions src/poetry/repositories/repository_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from poetry.core.packages.package import Package

from poetry.repositories.repository import Repository
from poetry.utils.cache import ArtifactCache


class Priority(IntEnum):
Expand All @@ -40,6 +41,7 @@ def __init__(
self,
repositories: list[Repository] | None = None,
ignore_repository_names: bool = False,
artifact_cache: ArtifactCache | None = None,
) -> None:
super().__init__("poetry-repository-pool")
self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict()
Expand All @@ -50,6 +52,8 @@ def __init__(
for repository in repositories:
self.add_repository(repository)

self._artifact_cache = artifact_cache

@property
def repositories(self) -> list[Repository]:
"""
Expand Down Expand Up @@ -77,6 +81,10 @@ def _sorted_repositories(self) -> list[PrioritizedRepository]:
self._repositories.values(), key=lambda prio_repo: prio_repo.priority
)

@property
def artifact_cache(self) -> ArtifactCache | None:
return self._artifact_cache

def has_default(self) -> bool:
return self._contains_priority(Priority.DEFAULT)

Expand Down
16 changes: 12 additions & 4 deletions src/poetry/utils/dependency_specification.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
if TYPE_CHECKING:
from poetry.core.packages.vcs_dependency import VCSDependency

from poetry.utils.cache import ArtifactCache
from poetry.utils.env import Env


Expand Down Expand Up @@ -58,7 +59,9 @@ def _parse_dependency_specification_git_url(


def _parse_dependency_specification_url(
requirement: str, env: Env | None = None
requirement: str,
env: Env | None = None,
artifact_cache: ArtifactCache | None = None,
) -> DependencySpec | None:
url_parsed = urllib.parse.urlparse(requirement)
if not (url_parsed.scheme and url_parsed.netloc):
Expand All @@ -68,7 +71,7 @@ def _parse_dependency_specification_url(
return _parse_dependency_specification_git_url(requirement, env)

if url_parsed.scheme in ["http", "https"]:
package = Provider.get_package_from_url(requirement)
package = Provider.get_package_from_url(requirement, artifact_cache)
assert package.source_url is not None
return {"name": package.name, "url": package.source_url}

Expand Down Expand Up @@ -196,7 +199,10 @@ def pep508_to_dependency_specification(requirement: str) -> DependencySpec | Non


def parse_dependency_specification(
requirement: str, env: Env | None = None, cwd: Path | None = None
requirement: str,
env: Env | None = None,
cwd: Path | None = None,
artifact_cache: ArtifactCache | None = None,
) -> DependencySpec:
requirement = requirement.strip()
cwd = cwd or Path.cwd()
Expand All @@ -213,7 +219,9 @@ def parse_dependency_specification(
requirement, _ = requirement.split("[")

specification = (
_parse_dependency_specification_url(requirement, env=env)
_parse_dependency_specification_url(
requirement, env=env, artifact_cache=artifact_cache
)
or _parse_dependency_specification_path(requirement, cwd=cwd)
or _parse_dependency_specification_simple(requirement)
)
Expand Down

0 comments on commit 43895af

Please sign in to comment.