From 8eb921e6f4de3fdc22b545fac2f1aca35e86c6b1 Mon Sep 17 00:00:00 2001 From: Ashwin Nair Date: Thu, 9 Jun 2022 11:45:02 +0400 Subject: [PATCH 1/2] Fix for zip subdirectories --- src/poetry/inspection/info.py | 15 +++++++++ src/poetry/installation/executor.py | 34 +++++++++++++++++--- src/poetry/installation/pip_installer.py | 5 ++- src/poetry/puzzle/provider.py | 21 ++++++++++-- src/poetry/utils/dependency_specification.py | 9 +++++- src/poetry/utils/pip.py | 4 +++ 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/src/poetry/inspection/info.py b/src/poetry/inspection/info.py index 665fd641c84..7da2ad6837d 100644 --- a/src/poetry/inspection/info.py +++ b/src/poetry/inspection/info.py @@ -92,6 +92,7 @@ def __init__( self._source_type: str | None = None self._source_url: str | None = None self._source_reference: str | None = None + self._source_subdirectory: str | None = None @property def cache_version(self) -> str | None: @@ -167,6 +168,7 @@ def to_package( source_type=self._source_type, source_url=self._source_url, source_reference=self._source_reference, + source_subdirectory=self._source_subdirectory, yanked=self.yanked, ) if self.summary is not None: @@ -263,6 +265,9 @@ def _from_distribution( info._source_type = "file" info._source_url = Path(dist.filename).resolve().as_posix() + if hasattr(cls, "_source_subdirectory") and cls._source_subdirectory: + info._source_subdirectory = cls._source_subdirectory + return info @classmethod @@ -321,6 +326,13 @@ def _from_sdist_file(cls, path: Path) -> PackageInfo: if not sdist_dir.is_dir(): sdist_dir = tmp + if ( + hasattr(cls, "_source_subdirectory") + and cls._source_subdirectory + and sdist_dir.joinpath(cls._source_subdirectory).is_dir() + ): + sdist_dir = sdist_dir.joinpath(cls._source_subdirectory) + # now this is an unpacked directory we know how to deal with new_info = cls.from_directory(path=sdist_dir) @@ -504,6 +516,9 @@ def from_directory(cls, path: Path, disable_build: bool = False) -> PackageInfo: info._source_type = "directory" info._source_url = path.as_posix() + if hasattr(cls, "_source_subdirectory") and cls._source_subdirectory: + info._source_subdirectory = cls._source_subdirectory + return info @classmethod diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 38e0065e31d..8d498084f99 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -118,10 +118,20 @@ def verbose(self, verbose: bool = True) -> Executor: return self def pip_install( - self, req: Path, upgrade: bool = False, editable: bool = False + self, + req: Path, + upgrade: bool = False, + editable: bool = False, + subdirectory: str | None = None, ) -> int: try: - pip_install(req, self._env, upgrade=upgrade, editable=editable) + pip_install( + req, + self._env, + upgrade=upgrade, + editable=editable, + subdirectory=subdirectory, + ) except EnvCommandError as e: output = decode(e.e.output) if ( @@ -492,7 +502,13 @@ def _install(self, operation: Install | Update) -> int: " Installing..." ) self._write(operation, message) - return self.pip_install(archive, upgrade=operation.job_type == "update") + if hasattr(package, "source_subdirectory"): + subdirectory = package.source_subdirectory + else: + subdirectory = None + return self.pip_install( + archive, upgrade=operation.job_type == "update", subdirectory=subdirectory + ) def _update(self, operation: Install | Update) -> int: return self._install(operation) @@ -803,6 +819,8 @@ def _create_url_url_reference(self, package: Package) -> dict[str, Any]: archive_info["hash"] = self._hashes[package.name] reference = {"url": package.source_url, "archive_info": archive_info} + if package.source_subdirectory: + reference["subdirectory"] = package.source_subdirectory return reference @@ -813,10 +831,13 @@ def _create_file_url_reference(self, package: Package) -> dict[str, Any]: archive_info["hash"] = self._hashes[package.name] assert package.source_url is not None - return { + reference = { "url": Path(package.source_url).as_uri(), "archive_info": archive_info, } + if package.source_subdirectory: + reference["subdirectory"] = package.source_subdirectory + return reference def _create_directory_url_reference(self, package: Package) -> dict[str, Any]: dir_info = {} @@ -825,7 +846,10 @@ def _create_directory_url_reference(self, package: Package) -> dict[str, Any]: dir_info["editable"] = True assert package.source_url is not None - return { + reference = { "url": Path(package.source_url).as_uri(), "dir_info": dir_info, } + if package.source_subdirectory: + reference["subdirectory"] = package.source_subdirectory + return reference diff --git a/src/poetry/installation/pip_installer.py b/src/poetry/installation/pip_installer.py index 884dd1895df..752bd1f9141 100644 --- a/src/poetry/installation/pip_installer.py +++ b/src/poetry/installation/pip_installer.py @@ -190,7 +190,10 @@ def requirement(self, package: Package, formatted: bool = False) -> str | list[s return req if package.source_type == "url": - return f"{package.source_url}#egg={package.name}" + req = f"{package.source_url}#egg={package.name}" + if package.source_subdirectory: + req += f"&subdirectory={package.source_subdirectory}" + return req return f"{package.name}=={package.version}" diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index 4d2e2d30b67..7ca77c5eb16 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -390,8 +390,11 @@ def _search_for_file(self, dependency: FileDependency) -> Package: return package @classmethod - def get_package_from_file(cls, file_path: Path) -> Package: + def get_package_from_file( + cls, file_path: Path, source_subdirectory: str | None = None + ) -> Package: try: + PackageInfo._source_subdirectory = source_subdirectory package = PackageInfo.from_path(path=file_path).to_package( root_dir=file_path ) @@ -419,7 +422,12 @@ def get_package_from_directory(cls, directory: Path) -> Package: return PackageInfo.from_directory(path=directory).to_package(root_dir=directory) def _search_for_url(self, dependency: URLDependency) -> Package: - package = self.get_package_from_url(dependency.url) + if dependency.directory: + url = f"{dependency.url}#subdirectory={dependency.directory}" + else: + url = dependency.url + + package = self.get_package_from_url(url) self.validate_package_for_dependency(dependency=dependency, package=package) @@ -439,7 +447,14 @@ def get_package_from_url(cls, url: str) -> Package: with tempfile.TemporaryDirectory() as temp_dir: dest = Path(temp_dir) / file_name download_file(url, dest) - package = cls.get_package_from_file(dest) + _subdirectory_fragment_re = re.compile(r"[#&]subdirectory=([^&]*)") + match = _subdirectory_fragment_re.search(url) + if not match: + source_subdirectory = None + else: + url = re.sub(r"[#&]subdirectory=([^&]*)", "", url) + source_subdirectory = match.group(1) + package = cls.get_package_from_file(dest, source_subdirectory) package._source_type = "url" package._source_url = url diff --git a/src/poetry/utils/dependency_specification.py b/src/poetry/utils/dependency_specification.py index e80a2b97019..97966c4795b 100644 --- a/src/poetry/utils/dependency_specification.py +++ b/src/poetry/utils/dependency_specification.py @@ -70,7 +70,14 @@ def _parse_dependency_specification_url( if url_parsed.scheme in ["http", "https"]: package = Provider.get_package_from_url(requirement) assert package.source_url is not None - return {"name": package.name, "url": package.source_url} + if package.source_subdirectory: + return { + "name": package.name, + "url": package.source_url, + "subdirectory": package.source_subdirectory, + } + else: + return {"name": package.name, "url": package.source_url} return None diff --git a/src/poetry/utils/pip.py b/src/poetry/utils/pip.py index 58b3504fba7..29e32e477db 100644 --- a/src/poetry/utils/pip.py +++ b/src/poetry/utils/pip.py @@ -18,6 +18,7 @@ def pip_install( editable: bool = False, deps: bool = False, upgrade: bool = False, + subdirectory: str | None = None, ) -> int | str: is_wheel = path.suffix == ".whl" @@ -50,6 +51,9 @@ def pip_install( ) args.append("-e") + if subdirectory: + path = f"file:{str(path)}#subdirectory={subdirectory}" + args.append(str(path)) try: From 11894feab38e10643b5a02351a963fce5ad607c6 Mon Sep 17 00:00:00 2001 From: Ashwin Nair Date: Thu, 9 Jun 2022 11:45:29 +0400 Subject: [PATCH 2/2] Add tests --- src/poetry/utils/pip.py | 2 +- tests/console/commands/test_add.py | 40 ++++++++ tests/fixtures/distributions/demo-0.1.0.zip | Bin 0 -> 2246 bytes tests/installation/test_pip_installer.py | 35 +++++++ tests/packages/test_locker.py | 95 ++++++++++++++++++ .../METADATA | 6 ++ .../direct_url.json | 5 + .../repositories/test_installed_repository.py | 19 ++++ tests/utils/test_dependency_specification.py | 9 ++ 9 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/distributions/demo-0.1.0.zip create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/direct_url.json diff --git a/src/poetry/utils/pip.py b/src/poetry/utils/pip.py index 29e32e477db..e229628320f 100644 --- a/src/poetry/utils/pip.py +++ b/src/poetry/utils/pip.py @@ -52,7 +52,7 @@ def pip_install( args.append("-e") if subdirectory: - path = f"file:{str(path)}#subdirectory={subdirectory}" + path = f"file:{str(path)}#subdirectory={subdirectory}" # type: ignore[assignment] # noqa: E501 args.append(str(path)) diff --git a/tests/console/commands/test_add.py b/tests/console/commands/test_add.py index ff34c4f8b20..5972270bff8 100644 --- a/tests/console/commands/test_add.py +++ b/tests/console/commands/test_add.py @@ -748,6 +748,46 @@ def test_add_url_constraint_wheel_with_extras( } +def test_add_url_constraint_zip_with_subdir( + app: PoetryTestApplication, + repo: TestRepository, + tester: CommandTester, + mocker: MockerFixture, +): + p = mocker.patch("pathlib.Path.cwd") + p.return_value = Path(__file__) / ".." + + repo.add_package(get_package("pendulum", "1.4.4")) + + tester.execute( + "https://python-poetry.org/distributions/demo-0.1.0.zip#subdirectory=subdir" + ) + + expected = """\ + +Updating dependencies +Resolving dependencies... + +Writing lock file + +Package operations: 2 installs, 0 updates, 0 removals + + • Installing pendulum (1.4.4) + • Installing demo\ + (0.1.0 https://python-poetry.org/distributions/demo-0.1.0.zip) +""" + assert tester.io.fetch_output() == expected + assert tester.command.installer.executor.installations_count == 2 + + content = app.poetry.file.read()["tool"]["poetry"] + + assert "demo" in content["dependencies"] + assert content["dependencies"]["demo"] == { + "url": "https://python-poetry.org/distributions/demo-0.1.0.zip", + "subdirectory": "subdir", + } + + def test_add_constraint_with_python( app: PoetryTestApplication, repo: TestRepository, tester: CommandTester ): diff --git a/tests/fixtures/distributions/demo-0.1.0.zip b/tests/fixtures/distributions/demo-0.1.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..e04197d6676bf309c6a6195af3dcf629d251d0d6 GIT binary patch literal 2246 zcmWIWW@Zs#00H;tlObRRl;CDiU`R>L&DS;1Gt@KC4-MgEV4t>NVG<0NR&X;gvbjBwG!4aJQ1c2Z3ySiyQj<&cO7e4a zz^S8q4_D*ojV>0A%{ch*_L{&>LK*fx;^s?{<8Q-M_$yZj)te9fK5pwJI z7n!{hCk?m*cE9*!AASG-ZlSd0)oPr}rS`n^SSz~iis7L~r!8-|XT01wPjQRL?1XAb z%_h%J#}v~7AAWmYoxUjI(<@b1L*A9kJ(R1KXE-9US8^L`*T`NvgWl5^EW+z z@PF$C?NYnd*WLHJXjq?Iy3WV!_0Bl48y1gv?D97GeQ^9Z;o^=psw%%&9-iA9{qy{* zS~Dvz1Iw%PDl;Pao-tVqvswQ77x>cPZ)kTCXTwgve@|HhyxBRb``-My4Gc3-a6uvl z6p#!Iim-5Gn83uqAcZv`i&INV3-k&q!9fU8KOK7z#!mF~J8U4*_T8>kklXo6)LfN- zr6D;tHYG-`I^i4F9Lj%?xmkIdk$Y?Yzc-J2oN0D+>AD@OuHV>w_{%=k z7YSRAh;Bb;t*4{AmGMmK;@-nzbMghUA}Ui%Bj!97-=Zq&;akS6r}ZpiieJl`kJD08 zzV`}*Uo%wXV$Pp8`(YvTs!xYb``Rtn-j((Fz;fd=YRU66&-?g1+alur?nGYGNypbG zycn&nb^H}Q&Y$+m>_6wj4}wQkSmLigUy}SjLFf9aqsCUtHy_d7u=HW?>#Z%VVWo!G z=51+;OTD-E)i&Ete|~RHzg8IV<6?WIw}P@i`55b^L(IMp zBRZ}Bhd<7LzvE7ILF}S?miMYZ^g6EkA<7o`{N)L?_zSx^_nn$v;JT>;+ix}JV+{_yC+k>aOL^z~wP z6lnck+hwH9>t|eVb-~E-Vgont!G_;n%Z$wY{Ez<=ELJ)H@aP)}cc+-I?>^r9zKZMB zL>aRe29;{bOW(0Ru}L-#QCXn4^yd8epL&+(Je=yZS@giWt&38ca~aZish2&z+ik`q z7C+y6i(e3baK7xiHVwtrO~MH;&G#>>J-0N?|I4$binGsbZ{&TgR(fuAvBgU6d7nPU zr8J*l5By-b`|X9Oozq07o$t-#)l&T}CbuEKb_c)9hx%pjLgY?=*4h+!ycidA zy~Wa^+*fwz91310e{Mm8>a6;EQSvtL{i4(pu1?&?bn9urbpx&wFJ{~fxc}pvw#D2l zlKPdEWj+>bcZI*?V{iSd{EZhB>Yci$-T8o_4hmaD>1B&n42omTB%p#3QiQ>R8&oW! z6*7pZlvudW2bT53fD9$vhQ`Nd=4F<|$HQ{q#4~)+oj|ie_Mn;H-LLPht99^f{yKtKB$fHN4F;9^F!0yW z6V+g%>i~%Puv!4QSOe7pFtDU?CRX!Fs}o?s1S=|$b0w&#gn=cEj~G#c3F;eS3r=)< zkkcWmJrjYID^heo?IAYD!|VaM8(XS|`RA`AJ1aJOh)m;%C<3K& Package: return package +@pytest.fixture +def package_url_zip_with_subdirectory() -> Package: + package = Package( + "subdirectories", + "2.0.0", + source_type="url", + source_url="https://github.com/demo/subdirectories.zip", + source_subdirectory="two", + ) + + return package + + @pytest.fixture def pool() -> RepositoryPool: return RepositoryPool() @@ -117,6 +131,27 @@ def test_requirement_git_subdirectory( assert Path(cmd[-1]).parts[-3:] == ("demo", "subdirectories", "two") +def test_requirement_url_zip_subdirectory( + pool: RepositoryPool, package_url_zip_with_subdirectory: Package +) -> None: + null_env = NullEnv() + installer = PipInstaller(null_env, NullIO(), pool) + result = installer.requirement(package_url_zip_with_subdirectory) + expected = ( + "https://github.com/demo/subdirectories.zip#egg=subdirectories&subdirectory=two" + ) + + assert result == expected + installer.install(package_url_zip_with_subdirectory) + assert len(null_env.executed) == 1 + cmd = null_env.executed[0] + + link = Link(cmd[-1]) + assert link.filename == "subdirectories.zip" + assert link.subdirectory_fragment == "two" + assert link.is_sdist + + def test_requirement_git_develop_false(installer: PipInstaller, package_git: Package): package_git.develop = False result = installer.requirement(package_git) diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 9da6c422f7c..897325078c2 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -79,6 +79,13 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): source_type="url", source_url="https://example.org/url-package-1.0-cp39-win_amd64.whl", ) + package_url_zip = Package( + "url-zip-subdir", + "1.0", + source_type="url", + source_url="https://example.org/archive/1.0.zip", + source_subdirectory="subdir", + ) packages = [ package_a2, package_a, @@ -87,6 +94,7 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): package_git_with_subdirectory, package_url_win32, package_url_linux, + package_url_zip, ] locker.set_lock_data(root, packages) @@ -191,6 +199,20 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): type = "url" url = "https://example.org/url-package-1.0-cp39-win_amd64.whl" +[[package]] +name = "url-zip-subdir" +version = "1.0" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[package.source] +type = "url" +url = "https://example.org/archive/1.0.zip" +subdirectory = "subdir" + [metadata] lock-version = "2.0" python-versions = "*" @@ -411,6 +433,39 @@ def test_locker_properly_loads_subdir(locker: Locker) -> None: assert package.source_subdirectory == "subdir" +def test_locker_properly_loads_url_zip_with_subdir(locker: Locker) -> None: + content = """\ +[[package]] +name = "url-zip-subdir" +version = "1.2.0" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[package.source] +type = "url" +url = "https://github.com/python-poetry/archive/1.2.0.zip" +subdirectory = "subdir" + +[metadata] +lock-version = "2.0" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" +""" + locker.lock.write(tomlkit.parse(content)) + + repository = locker.locked_repository() + assert len(repository.packages) == 1 + + packages = repository.find_packages(get_dependency("url-zip-subdir", "1.2.0")) + assert len(packages) == 1 + + package = packages[0] + assert package.source_subdirectory == "subdir" + + def test_locker_properly_assigns_metadata_files(locker: Locker) -> None: """ For multiple constraints dependencies, there is only one common entry in @@ -895,6 +950,46 @@ def test_locker_dumps_subdir(locker: Locker, root: ProjectPackage) -> None: assert content == expected +def test_locker_dumps_url_zip_with_subdir(locker: Locker, root: ProjectPackage) -> None: + package_git_with_subdirectory = Package( + "url-zip-subdir", + "1.2.0", + source_type="url", + source_url="https://github.com/python-poetry/archive/1.2.0.zip", + source_subdirectory="subdir", + ) + + locker.set_lock_data(root, [package_git_with_subdirectory]) + + with locker.lock.open(encoding="utf-8") as f: + content = f.read() + + expected = f"""\ +# {GENERATED_COMMENT} + +[[package]] +name = "url-zip-subdir" +version = "1.2.0" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[package.source] +type = "url" +url = "https://github.com/python-poetry/archive/1.2.0.zip" +subdirectory = "subdir" + +[metadata] +lock-version = "2.0" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" +""" # noqa: E800 + + assert content == expected + + def test_locker_dumps_dependency_extras_in_correct_order( locker: Locker, root: ProjectPackage ): diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..bb09c280747 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: url-pep-610-subdirectory +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..f716ac1e346 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610_subdirectory-1.2.3.dist-info/direct_url.json @@ -0,0 +1,5 @@ +{ + "url": "https://python-poetry.org/distributions/url-pep-610-subdirectory-1.2.3.zip", + "subdirectory": "subdir", + "archive_info": {} +} diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index 87dfe183c8a..05f7c5a017e 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -43,6 +43,9 @@ SITE_PURELIB / "git_pep_610_subdirectory-1.2.3.dist-info" ), metadata.PathDistribution(SITE_PURELIB / "url_pep_610-1.2.3.dist-info"), + metadata.PathDistribution( + SITE_PURELIB / "url_pep_610_subdirectory-1.2.3.dist-info" + ), metadata.PathDistribution(SITE_PURELIB / "file_pep_610-1.2.3.dist-info"), metadata.PathDistribution(SITE_PURELIB / "directory_pep_610-1.2.3.dist-info"), metadata.PathDistribution( @@ -268,6 +271,22 @@ def test_load_pep_610_compliant_url_packages(repository: InstalledRepository): ) +def test_load_pep_610_compliant_url_packages_with_subdirectory( + repository: InstalledRepository, +): + package = get_package_from_repository("url-pep-610-subdirectory", repository) + + assert package is not None + assert package.name == "url-pep-610-subdirectory" + assert package.version.text == "1.2.3" + assert package.source_type == "url" + assert package.source_subdirectory == "subdir" + assert ( + package.source_url + == "https://python-poetry.org/distributions/url-pep-610-subdirectory-1.2.3.zip" + ) + + def test_load_pep_610_compliant_file_packages(repository: InstalledRepository): package = get_package_from_repository("file-pep-610", repository) diff --git a/tests/utils/test_dependency_specification.py b/tests/utils/test_dependency_specification.py index 8b1c1dfd1b1..7665434d8fc 100644 --- a/tests/utils/test_dependency_specification.py +++ b/tests/utils/test_dependency_specification.py @@ -44,6 +44,15 @@ "subdirectory": "two", }, ), + ( + "https://python-poetry.org/distributions/demo-0.1.0.zip" + + "#subdirectory=subdir", + { + "url": "https://python-poetry.org/distributions/demo-0.1.0.zip", + "name": "demo", + "subdirectory": "subdir", + }, + ), ("demo", {"name": "demo"}), ("demo@1.0.0", {"name": "demo", "version": "1.0.0"}), ("demo@^1.0.0", {"name": "demo", "version": "^1.0.0"}),