Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix for url zip subdirectories #5811

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/poetry/inspection/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def __init__(
self._source_type: str | None = None
self._source_url: str | None = None
self._source_reference: str | None = None
self._source_subdirectory: str | None = None

@property
def cache_version(self) -> str | None:
Expand Down Expand Up @@ -167,6 +168,7 @@ def to_package(
source_type=self._source_type,
source_url=self._source_url,
source_reference=self._source_reference,
source_subdirectory=self._source_subdirectory,
yanked=self.yanked,
)
if self.summary is not None:
Expand Down Expand Up @@ -263,6 +265,9 @@ def _from_distribution(
info._source_type = "file"
info._source_url = Path(dist.filename).resolve().as_posix()

if hasattr(cls, "_source_subdirectory") and cls._source_subdirectory:
info._source_subdirectory = cls._source_subdirectory

return info

@classmethod
Expand Down Expand Up @@ -321,6 +326,13 @@ def _from_sdist_file(cls, path: Path) -> PackageInfo:
if not sdist_dir.is_dir():
sdist_dir = tmp

if (
hasattr(cls, "_source_subdirectory")
and cls._source_subdirectory
and sdist_dir.joinpath(cls._source_subdirectory).is_dir()
):
sdist_dir = sdist_dir.joinpath(cls._source_subdirectory)

# now this is an unpacked directory we know how to deal with
new_info = cls.from_directory(path=sdist_dir)

Expand Down Expand Up @@ -504,6 +516,9 @@ def from_directory(cls, path: Path, disable_build: bool = False) -> PackageInfo:
info._source_type = "directory"
info._source_url = path.as_posix()

if hasattr(cls, "_source_subdirectory") and cls._source_subdirectory:
info._source_subdirectory = cls._source_subdirectory

return info

@classmethod
Expand Down
34 changes: 29 additions & 5 deletions src/poetry/installation/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,20 @@ def verbose(self, verbose: bool = True) -> Executor:
return self

def pip_install(
self, req: Path, upgrade: bool = False, editable: bool = False
self,
req: Path,
upgrade: bool = False,
editable: bool = False,
subdirectory: str | None = None,
) -> int:
try:
pip_install(req, self._env, upgrade=upgrade, editable=editable)
pip_install(
req,
self._env,
upgrade=upgrade,
editable=editable,
subdirectory=subdirectory,
)
except EnvCommandError as e:
output = decode(e.e.output)
if (
Expand Down Expand Up @@ -492,7 +502,13 @@ def _install(self, operation: Install | Update) -> int:
" <info>Installing...</info>"
)
self._write(operation, message)
return self.pip_install(archive, upgrade=operation.job_type == "update")
if hasattr(package, "source_subdirectory"):
subdirectory = package.source_subdirectory
else:
subdirectory = None
return self.pip_install(
archive, upgrade=operation.job_type == "update", subdirectory=subdirectory
)

def _update(self, operation: Install | Update) -> int:
return self._install(operation)
Expand Down Expand Up @@ -803,6 +819,8 @@ def _create_url_url_reference(self, package: Package) -> dict[str, Any]:
archive_info["hash"] = self._hashes[package.name]

reference = {"url": package.source_url, "archive_info": archive_info}
if package.source_subdirectory:
reference["subdirectory"] = package.source_subdirectory

return reference

Expand All @@ -813,10 +831,13 @@ def _create_file_url_reference(self, package: Package) -> dict[str, Any]:
archive_info["hash"] = self._hashes[package.name]

assert package.source_url is not None
return {
reference = {
"url": Path(package.source_url).as_uri(),
"archive_info": archive_info,
}
if package.source_subdirectory:
reference["subdirectory"] = package.source_subdirectory
return reference

def _create_directory_url_reference(self, package: Package) -> dict[str, Any]:
dir_info = {}
Expand All @@ -825,7 +846,10 @@ def _create_directory_url_reference(self, package: Package) -> dict[str, Any]:
dir_info["editable"] = True

assert package.source_url is not None
return {
reference = {
"url": Path(package.source_url).as_uri(),
"dir_info": dir_info,
}
if package.source_subdirectory:
reference["subdirectory"] = package.source_subdirectory
return reference
5 changes: 4 additions & 1 deletion src/poetry/installation/pip_installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,10 @@ def requirement(self, package: Package, formatted: bool = False) -> str | list[s
return req

if package.source_type == "url":
return f"{package.source_url}#egg={package.name}"
req = f"{package.source_url}#egg={package.name}"
if package.source_subdirectory:
req += f"&subdirectory={package.source_subdirectory}"
return req

return f"{package.name}=={package.version}"

Expand Down
21 changes: 18 additions & 3 deletions src/poetry/puzzle/provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,11 @@ def _search_for_file(self, dependency: FileDependency) -> Package:
return package

@classmethod
def get_package_from_file(cls, file_path: Path) -> Package:
def get_package_from_file(
cls, file_path: Path, source_subdirectory: str | None = None
) -> Package:
try:
PackageInfo._source_subdirectory = source_subdirectory
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting the subdirectory as class attribute seems like a crude hack to me. I think that has to be refactored.

package = PackageInfo.from_path(path=file_path).to_package(
root_dir=file_path
)
Expand Down Expand Up @@ -419,7 +422,12 @@ def get_package_from_directory(cls, directory: Path) -> Package:
return PackageInfo.from_directory(path=directory).to_package(root_dir=directory)

def _search_for_url(self, dependency: URLDependency) -> Package:
package = self.get_package_from_url(dependency.url)
if dependency.directory:
url = f"{dependency.url}#subdirectory={dependency.directory}"
else:
url = dependency.url

package = self.get_package_from_url(url)

self.validate_package_for_dependency(dependency=dependency, package=package)

Expand All @@ -439,7 +447,14 @@ def get_package_from_url(cls, url: str) -> Package:
with tempfile.TemporaryDirectory() as temp_dir:
dest = Path(temp_dir) / file_name
download_file(url, dest)
package = cls.get_package_from_file(dest)
_subdirectory_fragment_re = re.compile(r"[#&]subdirectory=([^&]*)")
match = _subdirectory_fragment_re.search(url)
if not match:
source_subdirectory = None
else:
url = re.sub(r"[#&]subdirectory=([^&]*)", "", url)
source_subdirectory = match.group(1)
package = cls.get_package_from_file(dest, source_subdirectory)

package._source_type = "url"
package._source_url = url
Expand Down
9 changes: 8 additions & 1 deletion src/poetry/utils/dependency_specification.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,14 @@ def _parse_dependency_specification_url(
if url_parsed.scheme in ["http", "https"]:
package = Provider.get_package_from_url(requirement)
assert package.source_url is not None
return {"name": package.name, "url": package.source_url}
if package.source_subdirectory:
return {
"name": package.name,
"url": package.source_url,
"subdirectory": package.source_subdirectory,
}
else:
return {"name": package.name, "url": package.source_url}

return None

Expand Down
4 changes: 4 additions & 0 deletions src/poetry/utils/pip.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ def pip_install(
editable: bool = False,
deps: bool = False,
upgrade: bool = False,
subdirectory: str | None = None,
) -> int | str:
is_wheel = path.suffix == ".whl"

Expand Down Expand Up @@ -50,6 +51,9 @@ def pip_install(
)
args.append("-e")

if subdirectory:
path = f"file:{str(path)}#subdirectory={subdirectory}" # type: ignore[assignment] # noqa: E501

args.append(str(path))

try:
Expand Down
40 changes: 40 additions & 0 deletions tests/console/commands/test_add.py
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,46 @@ def test_add_url_constraint_wheel_with_extras(
}


def test_add_url_constraint_zip_with_subdir(
app: PoetryTestApplication,
repo: TestRepository,
tester: CommandTester,
mocker: MockerFixture,
):
p = mocker.patch("pathlib.Path.cwd")
p.return_value = Path(__file__) / ".."

repo.add_package(get_package("pendulum", "1.4.4"))

tester.execute(
"https://python-poetry.org/distributions/demo-0.1.0.zip#subdirectory=subdir"
)

expected = """\

Updating dependencies
Resolving dependencies...

Writing lock file

Package operations: 2 installs, 0 updates, 0 removals

• Installing pendulum (1.4.4)
• Installing demo\
(0.1.0 https://python-poetry.org/distributions/demo-0.1.0.zip)
"""
assert tester.io.fetch_output() == expected
assert tester.command.installer.executor.installations_count == 2

content = app.poetry.file.read()["tool"]["poetry"]

assert "demo" in content["dependencies"]
assert content["dependencies"]["demo"] == {
"url": "https://python-poetry.org/distributions/demo-0.1.0.zip",
"subdirectory": "subdir",
}


def test_add_constraint_with_python(
app: PoetryTestApplication, repo: TestRepository, tester: CommandTester
):
Expand Down
Binary file added tests/fixtures/distributions/demo-0.1.0.zip
Binary file not shown.
35 changes: 35 additions & 0 deletions tests/installation/test_pip_installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from cleo.io.null_io import NullIO
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link

from poetry.installation.pip_installer import PipInstaller
from poetry.repositories.legacy_repository import LegacyRepository
Expand Down Expand Up @@ -52,6 +53,19 @@ def package_git_with_subdirectory() -> Package:
return package


@pytest.fixture
def package_url_zip_with_subdirectory() -> Package:
package = Package(
"subdirectories",
"2.0.0",
source_type="url",
source_url="https://github.com/demo/subdirectories.zip",
source_subdirectory="two",
)

return package


@pytest.fixture
def pool() -> RepositoryPool:
return RepositoryPool()
Expand Down Expand Up @@ -117,6 +131,27 @@ def test_requirement_git_subdirectory(
assert Path(cmd[-1]).parts[-3:] == ("demo", "subdirectories", "two")


def test_requirement_url_zip_subdirectory(
pool: RepositoryPool, package_url_zip_with_subdirectory: Package
) -> None:
null_env = NullEnv()
installer = PipInstaller(null_env, NullIO(), pool)
result = installer.requirement(package_url_zip_with_subdirectory)
expected = (
"https://github.com/demo/subdirectories.zip#egg=subdirectories&subdirectory=two"
)

assert result == expected
installer.install(package_url_zip_with_subdirectory)
assert len(null_env.executed) == 1
cmd = null_env.executed[0]

link = Link(cmd[-1])
assert link.filename == "subdirectories.zip"
assert link.subdirectory_fragment == "two"
assert link.is_sdist


def test_requirement_git_develop_false(installer: PipInstaller, package_git: Package):
package_git.develop = False
result = installer.requirement(package_git)
Expand Down
Loading