From c4d63117d2ed2b91ba4854eee7eb82d3b77ab1a8 Mon Sep 17 00:00:00 2001 From: MarcDufresne Date: Sat, 14 Jul 2018 16:26:09 -0400 Subject: [PATCH 1/8] Adding support for HTTP Basic Auth when pulling from legacy repo --- poetry/installation/pip_installer.py | 7 ++++++- poetry/repositories/legacy_repository.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index 175d2bcedfa..077172dadc1 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -25,9 +25,14 @@ def install(self, package, update=False): if package.source_type == "legacy" and package.source_url: parsed = urlparse.urlparse(package.source_url) if parsed.scheme == "http": + if not parsed.username: + from_host = parsed.netloc + else: + from_host = parsed.netloc[parsed.netloc.index("@") + 1 :] + self._io.write_error( " Installing from unsecure host: {}".format( - parsed.netloc + from_host ) ) args += ["--trusted-host", parsed.netloc] diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 199472a15bd..b228fcf96e2 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -28,6 +28,7 @@ import poetry.packages +from poetry.config import Config from poetry.locations import CACHE_DIR from poetry.masonry.publishing.uploader import wheel_file_re from poetry.packages import Package @@ -159,6 +160,18 @@ def __init__(self, name, url, disable_cache=False): requests.session(), cache=FileCache(str(self._cache_dir / "_http")) ) + url_parts = urlparse.urlparse(self._url) + if not url_parts.username: + config = Config.create("auth.toml") + repo_auth = config.setting("http-basic.{}".format(self.name)) + if repo_auth: + netloc = "{}:{}@{}".format( + repo_auth["username"], repo_auth["password"], url_parts.netloc + ) + self._url = urlparse.urlunsplit( + (url_parts.scheme, netloc, url_parts.path, "", "") + ) + self._disable_cache = disable_cache @property From 1c5b9d792390798420ecd6cacf044339f07c2831 Mon Sep 17 00:00:00 2001 From: MarcDufresne Date: Sat, 14 Jul 2018 16:33:46 -0400 Subject: [PATCH 2/8] added docs --- docs/docs/repositories.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/docs/repositories.md b/docs/docs/repositories.md index 4aae8ccca94..238ddeec1a1 100644 --- a/docs/docs/repositories.md +++ b/docs/docs/repositories.md @@ -63,3 +63,8 @@ url = "https://foo.bar/simple/" ``` From now on, Poetry will also look for packages in your private repository. + +If your private repository requires HTTP Basic Auth be sure to add the username and +password to your `http-basic` config using the example above (be sure to use the +same name than in the `tool.poetry.source` section). Poetry will use these values +to authenticate to your private repository when downloading or looking for packages. From 6c62b457706163c92f406a6f79ea7450a91cdaa2 Mon Sep 17 00:00:00 2001 From: MarcDufresne Date: Sat, 14 Jul 2018 18:14:13 -0400 Subject: [PATCH 3/8] Adding unit test for http auth --- tests/repositories/test_legacy_repository.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 9fe5d5830bb..76c21756ec3 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -1,4 +1,5 @@ import pytest +from mock import MagicMock, mock from poetry.repositories.legacy_repository import LegacyRepository from poetry.repositories.legacy_repository import Page @@ -43,3 +44,15 @@ def test_page_absolute_links_path_are_correct(): for link in page.links: assert link.netloc == "files.pythonhosted.org" assert link.path.startswith("/packages/") + + +@mock.patch("poetry.repositories.legacy_repository.Config") +def test_http_basic_auth_repo(mock_config): + class MockConfig(object): + def setting(self, _, **__): + return {"username": "user1", "password": "p4ss"} + + mock_config.create = MagicMock(return_value=MockConfig()) + + repo = MockRepository() + assert "user1:p4ss@" in repo._url From 2d13177f8eaaf331306e8490495694e04b21f6f1 Mon Sep 17 00:00:00 2001 From: MarcDufresne Date: Sat, 14 Jul 2018 19:55:05 -0400 Subject: [PATCH 4/8] Adding mock to dev deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 698d1ac4397..ef14a79e46c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ pygments-github-lexers = "^0.0.5" black = { version = "^18.3-alpha.0", python = "^3.6" } pre-commit = "^1.10" tox = "^3.0" +mock = "^2.0" [tool.poetry.scripts] From 8c69c2b354137a50933cfb6aea92eff1b925e520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Dufresne?= Date: Mon, 16 Jul 2018 11:58:04 -0400 Subject: [PATCH 5/8] moved config reading logic to helper function, adapted test --- poetry/masonry/publishing/publisher.py | 14 ++------------ poetry/repositories/legacy_repository.py | 11 ++++------- poetry/utils/helpers.py | 9 +++++++++ tests/repositories/test_legacy_repository.py | 10 +++------- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/poetry/masonry/publishing/publisher.py b/poetry/masonry/publishing/publisher.py index 475841a631a..46ee7417c12 100644 --- a/poetry/masonry/publishing/publisher.py +++ b/poetry/masonry/publishing/publisher.py @@ -1,5 +1,6 @@ from poetry.locations import CONFIG_DIR from poetry.utils._compat import Path +from poetry.utils.helpers import get_http_basic_auth from poetry.utils.toml_file import TomlFile from .uploader import Uploader @@ -64,18 +65,7 @@ def publish(self, repository_name, username, password): url = config["repositories"][repository_name]["url"] if not (username and password): - auth_file = TomlFile(Path(CONFIG_DIR) / "auth.toml") - if auth_file.exists(): - auth_config = auth_file.read(raw=True) - - if ( - "http-basic" in auth_config - and repository_name in auth_config["http-basic"] - ): - config = auth_config["http-basic"][repository_name] - - username = config.get("username") - password = config.get("password") + username, password = get_http_basic_auth(repository_name) # Requesting missing credentials if not username: diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index b228fcf96e2..9ae26ed256b 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -38,7 +38,7 @@ from poetry.semver import Version from poetry.semver import VersionConstraint from poetry.utils._compat import Path -from poetry.utils.helpers import canonicalize_name +from poetry.utils.helpers import canonicalize_name, get_http_basic_auth from poetry.version.markers import InvalidMarker from .pypi_repository import PyPiRepository @@ -162,12 +162,9 @@ def __init__(self, name, url, disable_cache=False): url_parts = urlparse.urlparse(self._url) if not url_parts.username: - config = Config.create("auth.toml") - repo_auth = config.setting("http-basic.{}".format(self.name)) - if repo_auth: - netloc = "{}:{}@{}".format( - repo_auth["username"], repo_auth["password"], url_parts.netloc - ) + username, password = get_http_basic_auth(self.name) + if username and password: + netloc = "{}:{}@{}".format(username, password, url_parts.netloc) self._url = urlparse.urlunsplit( (url_parts.scheme, netloc, url_parts.path, "", "") ) diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index e085c05f0e1..393a996b4f7 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -5,6 +5,7 @@ from contextlib import contextmanager from typing import Union +from poetry.config import Config from poetry.version import Version _canonicalize_regex = re.compile("[-_]+") @@ -77,3 +78,11 @@ def parse_requires(requires): # type: (str) -> Union[list, None] if requires_dist: return requires_dist + + +def get_http_basic_auth(repository_name): # type: (str) -> (str, str) + config = Config.create("auth.toml") + repo_auth = config.setting("http-basic.{}".format(repository_name)) + if repo_auth: + return repo_auth["username"], repo_auth["password"] + return "", "" diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 76c21756ec3..676bc243e38 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -46,13 +46,9 @@ def test_page_absolute_links_path_are_correct(): assert link.path.startswith("/packages/") -@mock.patch("poetry.repositories.legacy_repository.Config") -def test_http_basic_auth_repo(mock_config): - class MockConfig(object): - def setting(self, _, **__): - return {"username": "user1", "password": "p4ss"} - - mock_config.create = MagicMock(return_value=MockConfig()) +@mock.patch("poetry.repositories.legacy_repository.get_http_basic_auth") +def test_http_basic_auth_repo(mock_get_auth): + mock_get_auth.return_value = ("user1", "p4ss") repo = MockRepository() assert "user1:p4ss@" in repo._url From 94ff4fecf1618156608b288bfaaafe6ed2f9c630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Dufresne?= Date: Tue, 24 Jul 2018 13:51:03 -0400 Subject: [PATCH 6/8] corrections --- .gitignore | 2 ++ poetry/masonry/publishing/publisher.py | 5 ++++- poetry/repositories/legacy_repository.py | 14 ++++++++------ poetry/utils/helpers.py | 4 ++-- pyproject.toml | 1 - tests/repositories/test_legacy_repository.py | 14 ++++++-------- 6 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 816cc97aab1..6ff12ce9b7d 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ MANIFEST.in pyproject.lock /tests/fixtures/simple_project/setup.py .mypy_cache + +.venv diff --git a/poetry/masonry/publishing/publisher.py b/poetry/masonry/publishing/publisher.py index 46ee7417c12..426bc4db7d6 100644 --- a/poetry/masonry/publishing/publisher.py +++ b/poetry/masonry/publishing/publisher.py @@ -65,7 +65,10 @@ def publish(self, repository_name, username, password): url = config["repositories"][repository_name]["url"] if not (username and password): - username, password = get_http_basic_auth(repository_name) + auth = get_http_basic_auth(repository_name) + if auth: + username = auth[0] + password = auth[1] # Requesting missing credentials if not username: diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 9ae26ed256b..6bbfd3af7fb 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -162,12 +162,7 @@ def __init__(self, name, url, disable_cache=False): url_parts = urlparse.urlparse(self._url) if not url_parts.username: - username, password = get_http_basic_auth(self.name) - if username and password: - netloc = "{}:{}@{}".format(username, password, url_parts.netloc) - self._url = urlparse.urlunsplit( - (url_parts.scheme, netloc, url_parts.path, "", "") - ) + self._session.auth = get_http_basic_auth(self.name) self._disable_cache = disable_cache @@ -343,6 +338,13 @@ def _get_release_info(self, name, version): # type: (str, str) -> dict return data + def _download(self, url, dest): # type: (str, str) -> None + r = self._session.get(url, stream=True) + with open(dest, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: + f.write(chunk) + def _get(self, endpoint): # type: (str) -> Union[Page, None] url = self._url + endpoint response = self._session.get(url) diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index 393a996b4f7..48b48a0a0ff 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -80,9 +80,9 @@ def parse_requires(requires): # type: (str) -> Union[list, None] return requires_dist -def get_http_basic_auth(repository_name): # type: (str) -> (str, str) +def get_http_basic_auth(repository_name): # type: (str) -> tuple config = Config.create("auth.toml") repo_auth = config.setting("http-basic.{}".format(repository_name)) if repo_auth: return repo_auth["username"], repo_auth["password"] - return "", "" + return None diff --git a/pyproject.toml b/pyproject.toml index ef14a79e46c..698d1ac4397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,6 @@ pygments-github-lexers = "^0.0.5" black = { version = "^18.3-alpha.0", python = "^3.6" } pre-commit = "^1.10" tox = "^3.0" -mock = "^2.0" [tool.poetry.scripts] diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 676bc243e38..77ffaed82c1 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -1,10 +1,6 @@ -import pytest -from mock import MagicMock, mock - from poetry.repositories.legacy_repository import LegacyRepository from poetry.repositories.legacy_repository import Page from poetry.utils._compat import Path -from poetry.utils._compat import decode class MockRepository(LegacyRepository): @@ -46,9 +42,11 @@ def test_page_absolute_links_path_are_correct(): assert link.path.startswith("/packages/") -@mock.patch("poetry.repositories.legacy_repository.get_http_basic_auth") -def test_http_basic_auth_repo(mock_get_auth): - mock_get_auth.return_value = ("user1", "p4ss") +def test_http_basic_auth_repo(mocker): + mock = mocker.patch("poetry.repositories.legacy_repository.get_http_basic_auth") + mock.return_value = ("user1", "p4ss") repo = MockRepository() - assert "user1:p4ss@" in repo._url + + mock.assert_called_once_with("legacy") + assert repo._session.auth == ("user1", "p4ss") From 3f82b3c90afc050cc056d77411cf50cf5b7f33ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Dufresne?= Date: Tue, 24 Jul 2018 14:44:26 -0400 Subject: [PATCH 7/8] rework pip_installer --- poetry/installation/pip_installer.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index 077172dadc1..16897d2ba9e 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -3,6 +3,9 @@ from subprocess import CalledProcessError +from poetry.utils.helpers import get_http_basic_auth + + try: import urllib.parse as urlparse except ImportError: @@ -25,19 +28,27 @@ def install(self, package, update=False): if package.source_type == "legacy" and package.source_url: parsed = urlparse.urlparse(package.source_url) if parsed.scheme == "http": - if not parsed.username: - from_host = parsed.netloc - else: - from_host = parsed.netloc[parsed.netloc.index("@") + 1 :] - self._io.write_error( " Installing from unsecure host: {}".format( - from_host + parsed.netloc ) ) args += ["--trusted-host", parsed.netloc] - args += ["--index-url", package.source_url] + # This here won't work, need some pointers + auth = get_http_basic_auth(package.source_url) + if auth: + index_url = "{scheme}://{username}:{password}@{netloc}{path}".format( + scheme=parsed.scheme, + username=auth[0], + password=auth[1], + netloc=parsed.netloc, + path=parsed.path, + ) + else: + index_url = package.source_url + + args += ["--index-url", index_url] if update: args.append("-U") From d599d4f2a93ea02be3458f4095f82bedb4127c8a Mon Sep 17 00:00:00 2001 From: Marc-Andre Dufresne Date: Tue, 24 Jul 2018 21:10:26 -0400 Subject: [PATCH 8/8] use source_reference as repo name when writing lock file and installing --- poetry/installation/pip_installer.py | 3 +-- poetry/repositories/legacy_repository.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index 16897d2ba9e..4343b5f9673 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -35,8 +35,7 @@ def install(self, package, update=False): ) args += ["--trusted-host", parsed.netloc] - # This here won't work, need some pointers - auth = get_http_basic_auth(package.source_url) + auth = get_http_basic_auth(package.source_reference) if auth: index_url = "{scheme}://{username}:{password}@{netloc}{path}".format( scheme=parsed.scheme, diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 6bbfd3af7fb..b0857a72152 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -242,6 +242,7 @@ def package( package = poetry.packages.Package(name, version, version) package.source_type = "legacy" package.source_url = self._url + package.source_reference = self.name requires_dist = release_info["requires_dist"] or [] for req in requires_dist: