From 1c0d57f45f0734bb062f7e39786bed70c17af1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Jan 2024 11:23:21 +0100 Subject: [PATCH 1/8] Initial work for recipe filters per remote --- conan/api/model.py | 3 +- conan/api/subapi/remotes.py | 4 +- conan/cli/commands/remote.py | 9 +- conans/client/cache/remote_registry.py | 15 +- conans/client/graph/proxy.py | 3 + conans/client/graph/range_resolver.py | 6 + .../test/integration/command/remote_test.py | 178 +++++++++++------- 7 files changed, 139 insertions(+), 79 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index d1687197e00..ca16f4158bf 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -12,11 +12,12 @@ class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False): + def __init__(self, name, url, verify_ssl=True, disabled=False, filters=None): self._name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled + self.filters = filters @property def name(self): diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index b5b1b23fb6c..531ec6f8f83 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -72,9 +72,9 @@ def remove(self, pattern: str): RemoteRegistry(self._remotes_file).remove(remote.name) users_clean(app.cache.localdb, remote.url) - def update(self, remote_name, url=None, secure=None, disabled=None, index=None): + def update(self, remote_name, url=None, secure=None, disabled=None, index=None, filters=None): RemoteRegistry(self._remotes_file).update(remote_name, url, secure, disabled=disabled, - index=index) + index=index, filters=filters) def rename(self, remote_name: str, new_name: str): RemoteRegistry(self._remotes_file).rename(remote_name, new_name) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index 9fea57cd530..e0f140242f8 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -72,9 +72,11 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") + subparser.add_argument("--filter", action="append", default=None, + help="Add recipe reference pattern filter to this remote") subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False) + r = Remote(args.name, args.url, args.secure, disabled=False, filters=args.filter) conan_api.remotes.add(r, force=args.force, index=args.index) @@ -102,11 +104,12 @@ def remote_update(conan_api, parser, subparser, *args): help="Allow insecure server connections when using SSL") subparser.add_argument("--index", action=OnceArgument, type=int, help="Insert the remote at a specific position in the remote list") + subparser.add_argument("--filter", action="append", default=None, help="Add recipe reference pattern filter to this remote") subparser.set_defaults(secure=None) args = parser.parse_args(*args) - if args.url is None and args.secure is None and args.index is None: + if args.url is None and args.secure is None and args.index is None and args.filter is None: subparser.error("Please add at least one argument to update") - conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index) + conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index, filters=args.filter) @conan_subcommand() diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index 4a906de2afc..56172ae3af9 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -33,8 +33,9 @@ def load(filename): data = json.loads(text) for r in data.get("remotes", []): disabled = r.get("disabled", False) + filters = r.get("filters", None) # TODO: Remote.serialize/deserialize - remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled) + remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled, filters) result._remotes[r["name"]] = remote return result @@ -44,6 +45,8 @@ def dumps(self): remote = {"name": r.name, "url": r.url, "verify_ssl": r.verify_ssl} if r.disabled: remote["disabled"] = True + if r.filters: + remote["filters"] = r.filters remote_list.append(remote) ret = {"remotes": remote_list} return json.dumps(ret, indent=True) @@ -94,7 +97,7 @@ def _check_urls(self, url, force, current): else: ConanOutput().warning(msg + " Adding duplicated remote url because '--force'.") - def update(self, remote_name, url=None, secure=None, disabled=None, index=None, force=False): + def update(self, remote_name, url=None, secure=None, disabled=None, index=None, force=False, filters=None): remote = self[remote_name] if url is not None: self._check_urls(url, force, remote) @@ -110,6 +113,10 @@ def update(self, remote_name, url=None, secure=None, disabled=None, index=None, remotes.insert(index, remote) self._remotes = {r.name: r for r in remotes} + if filters is not None: + remote.filters = remote.filters or [] + remote.filters.extend(filters) + def items(self): return list(self._remotes.values()) @@ -170,11 +177,11 @@ def remove(self, remote_name): remotes.remove(remote_name) self.save_remotes(remotes) - def update(self, remote_name, url, secure, disabled, index): + def update(self, remote_name, url, secure, disabled, index, filters): if url is not None: self._validate_url(url) remotes = self._load_remotes() - remotes.update(remote_name, url, secure, disabled, index) + remotes.update(remote_name, url, secure, disabled, index, filters=filters) self.save_remotes(remotes) def rename(self, remote, new_name): diff --git a/conans/client/graph/proxy.py b/conans/client/graph/proxy.py index 5adb86ad00a..8a223daaaed 100644 --- a/conans/client/graph/proxy.py +++ b/conans/client/graph/proxy.py @@ -97,6 +97,9 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat results = [] for remote in remotes: + if remote.filters and not any(reference.matches(f, is_consumer=False) for f in remote.filters): + output.info(f"Excluding remote {remote.name} because recipe is filtered out") + continue output.info(f"Checking remote: {remote.name}") try: if not reference.revision: diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py index cf6ba5fc049..8f0dd53076e 100644 --- a/conans/client/graph/range_resolver.py +++ b/conans/client/graph/range_resolver.py @@ -1,3 +1,4 @@ +from conan.api.output import ConanOutput from conans.errors import ConanException from conans.model.recipe_ref import RecipeReference from conans.model.version_range import VersionRange @@ -63,6 +64,11 @@ def _resolve_local(self, search_ref, version_range): return self._resolve_version(version_range, local_found, self._resolve_prereleases) def _search_remote_recipes(self, remote, search_ref): + (ConanOutput() + .info("Searching remote recipe: %s" % search_ref) + .info("For filters: %s" % remote.filters)) + if remote.filters and not any(search_ref.matches(f, is_consumer=False) for f in remote.filters): + return [] pattern = str(search_ref) pattern_cached = self._cached_remote_found.setdefault(pattern, {}) results = pattern_cached.get(remote.name) diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index 53a15c57db5..f5b6cb83e9e 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -2,12 +2,14 @@ import unittest from collections import OrderedDict +import pytest + from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient, TestServer from conans.util.files import load -class RemoteTest(unittest.TestCase): +class RemoteServerTest(unittest.TestCase): def setUp(self): self.servers = OrderedDict() @@ -15,7 +17,7 @@ def setUp(self): test_server = TestServer() self.servers["remote%d" % i] = test_server - self.client = TestClient(servers=self.servers, inputs=3*["admin", "password"]) + self.client = TestClient(servers=self.servers, inputs=3*["admin", "password"], light=True) def test_list_json(self): self.client.run("remote list --format=json") @@ -84,24 +86,6 @@ def test_remove_remote_no_user(self): self.client.run("remote list") self.assertNotIn("remote0", self.client.out) - def test_rename(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("export . --name=hello --version=0.1 --user=user --channel=testing") - client.run("remote add r1 https://r1") - client.run("remote add r2 https://r2") - client.run("remote add r3 https://r3") - client.run("remote rename r2 mynewr2") - client.run("remote list") - lines = str(client.out).splitlines() - self.assertIn("r1: https://r1", lines[0]) - self.assertIn("mynewr2: https://r2", lines[1]) - self.assertIn("r3: https://r3", lines[2]) - - # Rename to an existing one - client.run("remote rename mynewr2 r1", assert_error=True) - self.assertIn("Remote 'r1' already exists", client.out) - def test_insert(self): self.client.run("remote add origin https://myurl --index", assert_error=True) @@ -119,8 +103,69 @@ def test_insert(self): self.assertIn("origin3: https://myurl3", lines[1]) self.assertIn("origin: https://myurl", lines[2]) + def test_duplicated_error(self): + """ check remote name are not duplicated + """ + self.client.run("remote add remote1 http://otherurl", assert_error=True) + self.assertIn("ERROR: Remote 'remote1' already exists in remotes (use --force to continue)", + self.client.out) + + self.client.run("remote list") + assert "otherurl" not in self.client.out + self.client.run("remote add remote1 http://otherurl --force") + self.assertIn("WARN: Remote 'remote1' already exists in remotes", self.client.out) + + self.client.run("remote list") + assert "remote1: http://otherurl" in self.client.out + + def test_missing_subarguments(self): + self.client.run("remote", assert_error=True) + self.assertIn("ERROR: Exiting with code: 2", self.client.out) + + def test_invalid_url(self): + self.client.run("remote add foobar foobar.com") + self.assertIn("WARN: The URL 'foobar.com' is invalid. It must contain scheme and hostname.", + self.client.out) + self.client.run("remote list") + self.assertIn("foobar.com", self.client.out) + + self.client.run("remote update foobar --url pepe.org") + self.assertIn("WARN: The URL 'pepe.org' is invalid. It must contain scheme and hostname.", + self.client.out) + self.client.run("remote list") + self.assertIn("pepe.org", self.client.out) + + def test_errors(self): + self.client.run("remote update origin --url http://foo.com", assert_error=True) + self.assertIn("ERROR: Remote 'origin' doesn't exist", self.client.out) + + self.client.run("remote remove origin", assert_error=True) + self.assertIn("ERROR: Remote 'origin' can't be found or is disabled", self.client.out) + + + + +class RemoteModificationTest(unittest.TestCase): + def test_rename(self): + client = TestClient(light=True) + client.save({"conanfile.py": GenConanfile()}) + client.run("export . --name=hello --version=0.1 --user=user --channel=testing") + client.run("remote add r1 https://r1") + client.run("remote add r2 https://r2") + client.run("remote add r3 https://r3") + client.run("remote rename r2 mynewr2") + client.run("remote list") + lines = str(client.out).splitlines() + self.assertIn("r1: https://r1", lines[0]) + self.assertIn("mynewr2: https://r2", lines[1]) + self.assertIn("r3: https://r3", lines[2]) + + # Rename to an existing one + client.run("remote rename mynewr2 r1", assert_error=True) + self.assertIn("Remote 'r1' already exists", client.out) + def test_update_insert(self): - client = TestClient() + client = TestClient(light=True) client.run("remote add r1 https://r1") client.run("remote add r2 https://r2") client.run("remote add r3 https://r3") @@ -142,7 +187,7 @@ def test_update_insert(self): def test_update_insert_same_url(self): # https://github.com/conan-io/conan/issues/5107 - client = TestClient() + client = TestClient(light=True) client.run("remote add r1 https://r1") client.run("remote add r2 https://r2") client.run("remote add r3 https://r3") @@ -153,7 +198,7 @@ def test_update_insert_same_url(self): self.assertLess(str(client.out).find("r1"), str(client.out).find("r3")) def test_verify_ssl(self): - client = TestClient() + client = TestClient(light=True) client.run("remote add my-remote http://someurl") client.run("remote add my-remote2 http://someurl2 --insecure") client.run("remote add my-remote3 http://someurl3") @@ -173,7 +218,7 @@ def test_verify_ssl(self): self.assertEqual(data["remotes"][2]["verify_ssl"], True) def test_remote_disable(self): - client = TestClient() + client = TestClient(light=True) client.run("remote add my-remote0 http://someurl0") client.run("remote add my-remote1 http://someurl1") client.run("remote add my-remote2 http://someurl2") @@ -221,7 +266,7 @@ def test_remote_disable(self): assert "Enabled: False" not in client.out def test_invalid_remote_disable(self): - client = TestClient() + client = TestClient(light=True) client.run("remote disable invalid_remote", assert_error=True) msg = "ERROR: Remote 'invalid_remote' can't be found or is disabled" @@ -236,7 +281,7 @@ def test_remote_disable_already_set(self): """ Check that we don't raise an error if the remote is already in the required state """ - client = TestClient() + client = TestClient(light=True) client.run("remote add my-remote0 http://someurl0") client.run("remote enable my-remote0") @@ -246,57 +291,18 @@ def test_remote_disable_already_set(self): client.run("remote disable my-remote0") def test_verify_ssl_error(self): - client = TestClient() + client = TestClient(light=True) client.run("remote add my-remote http://someurl some_invalid_option=foo", assert_error=True) self.assertIn("unrecognized arguments: some_invalid_option=foo", client.out) data = json.loads(load(client.cache.remotes_path)) self.assertEqual(data["remotes"], []) - def test_errors(self): - self.client.run("remote update origin --url http://foo.com", assert_error=True) - self.assertIn("ERROR: Remote 'origin' doesn't exist", self.client.out) - - self.client.run("remote remove origin", assert_error=True) - self.assertIn("ERROR: Remote 'origin' can't be found or is disabled", self.client.out) - - def test_duplicated_error(self): - """ check remote name are not duplicated - """ - self.client.run("remote add remote1 http://otherurl", assert_error=True) - self.assertIn("ERROR: Remote 'remote1' already exists in remotes (use --force to continue)", - self.client.out) - - self.client.run("remote list") - assert "otherurl" not in self.client.out - self.client.run("remote add remote1 http://otherurl --force") - self.assertIn("WARN: Remote 'remote1' already exists in remotes", self.client.out) - - self.client.run("remote list") - assert "remote1: http://otherurl" in self.client.out - - def test_missing_subarguments(self): - self.client.run("remote", assert_error=True) - self.assertIn("ERROR: Exiting with code: 2", self.client.out) - - def test_invalid_url(self): - self.client.run("remote add foobar foobar.com") - self.assertIn("WARN: The URL 'foobar.com' is invalid. It must contain scheme and hostname.", - self.client.out) - self.client.run("remote list") - self.assertIn("foobar.com", self.client.out) - - self.client.run("remote update foobar --url pepe.org") - self.assertIn("WARN: The URL 'pepe.org' is invalid. It must contain scheme and hostname.", - self.client.out) - self.client.run("remote list") - self.assertIn("pepe.org", self.client.out) - def test_add_duplicated_url(): """ allow duplicated URL with --force """ - c = TestClient() + c = TestClient(light=True) c.run("remote add remote1 http://url") c.run("remote add remote2 http://url", assert_error=True) assert "ERROR: Remote url already existing in remote 'remote1'" in c.out @@ -320,7 +326,7 @@ def test_add_duplicated_name_url(): """ do not add extra remote with same name and same url # https://github.com/conan-io/conan/issues/13569 """ - c = TestClient() + c = TestClient(light=True) c.run("remote add remote1 http://url") c.run("remote add remote1 http://url --force") assert "WARN: Remote 'remote1' already exists in remotes" in c.out @@ -331,10 +337,44 @@ def test_add_duplicated_name_url(): def test_add_wrong_conancenter(): """ Avoid common error of adding the wrong ConanCenter URL """ - c = TestClient() + c = TestClient(light=True) c.run("remote add whatever https://conan.io/center", assert_error=True) assert "Wrong ConanCenter remote URL. You are adding the web https://conan.io/center" in c.out assert "the correct remote API is https://center.conan.io" in c.out c.run("remote update conancenter --url=https://conan.io/center", assert_error=True) assert "Wrong ConanCenter remote URL. You are adding the web https://conan.io/center" in c.out assert "the correct remote API is https://center.conan.io" in c.out + + +class TestRemoteRecipeFilter(unittest.TestCase): + def setUp(self): + self.client = TestClient(light=True, default_server_user=True) + self.client.save({"lib/conanfile.py": GenConanfile("lib", "1.0"), + "app/conanfile.py": GenConanfile("app", "1.0") + .with_requires("lib/1.0")}) + self.client.run("create lib") + self.client.run("create app") + self.client.run("upload * -r=default -c") + self.client.run("remove * -c") + + def test_filter_remotes(self): + self.client.run("install --requires=app/1.0 -r=default") + assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out + assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" in self.client.out + self.client.run("remove * -c") + + # lib/2.* pattern does not match the one being required by app, should not be found + self.client.run('remote update default --filter="app/*" --filter="lib/2.*"') + + self.client.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out + assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" not in self.client.out + assert "ERROR: Package 'lib/1.0' not resolved: Unable to find 'lib/1.0' in remotes" in self.client.out + self.client.run("remove * -c") + + self.client.run('remote update default --filter="lib/*"') + self.client.run("install --requires=app/1.0 -r=default") + assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out + assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" in self.client.out + self.client.run("remove * -c") + From 4c2129bdae0443b70da43cb974c9059f757649b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Jan 2024 11:26:11 +0100 Subject: [PATCH 2/8] Remove extra verbose logging --- conans/client/graph/range_resolver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py index 8f0dd53076e..ad81ecc3cd9 100644 --- a/conans/client/graph/range_resolver.py +++ b/conans/client/graph/range_resolver.py @@ -64,9 +64,6 @@ def _resolve_local(self, search_ref, version_range): return self._resolve_version(version_range, local_found, self._resolve_prereleases) def _search_remote_recipes(self, remote, search_ref): - (ConanOutput() - .info("Searching remote recipe: %s" % search_ref) - .info("For filters: %s" % remote.filters)) if remote.filters and not any(search_ref.matches(f, is_consumer=False) for f in remote.filters): return [] pattern = str(search_ref) From a0da5922805676a0632c4a1d42a41f03e613e8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Jan 2024 13:41:41 +0100 Subject: [PATCH 3/8] Address review comments --- conans/client/cache/remote_registry.py | 3 +-- conans/client/graph/proxy.py | 2 +- conans/test/integration/command/remote_test.py | 4 +--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index 56172ae3af9..e683c8195db 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -114,8 +114,7 @@ def update(self, remote_name, url=None, secure=None, disabled=None, index=None, self._remotes = {r.name: r for r in remotes} if filters is not None: - remote.filters = remote.filters or [] - remote.filters.extend(filters) + remote.filters = filters def items(self): return list(self._remotes.values()) diff --git a/conans/client/graph/proxy.py b/conans/client/graph/proxy.py index 8a223daaaed..84b03e1104d 100644 --- a/conans/client/graph/proxy.py +++ b/conans/client/graph/proxy.py @@ -98,7 +98,7 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat results = [] for remote in remotes: if remote.filters and not any(reference.matches(f, is_consumer=False) for f in remote.filters): - output.info(f"Excluding remote {remote.name} because recipe is filtered out") + output.debug(f"Excluding remote {remote.name} because recipe is filtered out") continue output.info(f"Checking remote: {remote.name}") try: diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index f5b6cb83e9e..c961f9c0305 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -372,9 +372,7 @@ def test_filter_remotes(self): assert "ERROR: Package 'lib/1.0' not resolved: Unable to find 'lib/1.0' in remotes" in self.client.out self.client.run("remove * -c") - self.client.run('remote update default --filter="lib/*"') + self.client.run('remote update default --filter="lib/*" --filter="app/*"') self.client.run("install --requires=app/1.0 -r=default") assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" in self.client.out - self.client.run("remove * -c") - From 0e8071ac238246bd84543f6f74425a95c39847ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Jan 2024 14:28:26 +0100 Subject: [PATCH 4/8] Address review comments --- conan/api/model.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index ca16f4158bf..64b0b8a12de 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -30,8 +30,11 @@ def __eq__(self, other): self.verify_ssl == other.verify_ssl and self.disabled == other.disabled) def __str__(self): - return "{}: {} [Verify SSL: {}, Enabled: {}]".format(self.name, self.url, self.verify_ssl, - not self.disabled) + filtered_msg = "" + if self.filters: + filtered_msg = ", Filtered: {}".format(", ".join(self.filters)) + return "{}: {} [Verify SSL: {}, Enabled: {}{}]".format(self.name, self.url, self.verify_ssl, + not self.disabled, filtered_msg) def __repr__(self): return str(self) From bb9dd2a3203d7b76b16d00a823cf67efe54fbacf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 17 Jan 2024 18:14:37 +0100 Subject: [PATCH 5/8] Add test for the resolve_range path --- .../test/integration/command/remote_test.py | 41 +++++++++++++------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index c961f9c0305..60b3c12e4e6 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -349,30 +349,47 @@ def test_add_wrong_conancenter(): class TestRemoteRecipeFilter(unittest.TestCase): def setUp(self): self.client = TestClient(light=True, default_server_user=True) - self.client.save({"lib/conanfile.py": GenConanfile("lib", "1.0"), + self.client.save({"conanfile.py": GenConanfile(), "app/conanfile.py": GenConanfile("app", "1.0") - .with_requires("lib/1.0")}) - self.client.run("create lib") + .with_requires("liba/1.0") + .with_requires("libb/[>=1.0]")}) + self.client.run("create . --name=liba --version=1.0") + self.client.run("create . --name=libb --version=1.0") self.client.run("create app") self.client.run("upload * -r=default -c") self.client.run("remove * -c") def test_filter_remotes(self): self.client.run("install --requires=app/1.0 -r=default") - assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out - assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" in self.client.out + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out self.client.run("remove * -c") # lib/2.* pattern does not match the one being required by app, should not be found - self.client.run('remote update default --filter="app/*" --filter="lib/2.*"') + self.client.run('remote update default --filter="app/*" --filter="liba/2.*"') self.client.run("install --requires=app/1.0 -r=default", assert_error=True) - assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out - assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" not in self.client.out - assert "ERROR: Package 'lib/1.0' not resolved: Unable to find 'lib/1.0' in remotes" in self.client.out + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" not in self.client.out + assert "ERROR: Package 'liba/1.0' not resolved: Unable to find 'liba/1.0' in remotes" in self.client.out self.client.run("remove * -c") - self.client.run('remote update default --filter="lib/*" --filter="app/*"') + self.client.run('remote update default --filter="liba/*" --filter="app/*"') + self.client.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out + assert "ERROR: Package 'libb/[>=1.0]' not resolved" in self.client.out + self.client.run("remove * -c") + + self.client.run('remote update default --filter="liba/*" --filter="app/*"') + self.client.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out + assert "ERROR: Package 'libb/[>=1.0]' not resolved" in self.client.out + self.client.run("remove * -c") + + self.client.run('remote update default --filter="*"') self.client.run("install --requires=app/1.0 -r=default") - assert "app/1.0: Downloaded recipe revision 6d87b1d33fd24eba1f2f71124fd07f9c" in self.client.out - assert "lib/1.0: Downloaded recipe revision 5abc78f3a076cc58852154c7546ff7e5" in self.client.out + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out + assert "libb/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out From 8fad20f588dc944d5a7426446d3c83c39474af65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Fri, 19 Jan 2024 09:47:51 +0100 Subject: [PATCH 6/8] Rename to --allowed-packages --- conan/api/model.py | 12 +- conan/api/subapi/remotes.py | 4 +- conan/cli/commands/remote.py | 13 ++- conans/client/cache/remote_registry.py | 18 +-- conans/client/graph/proxy.py | 3 +- conans/client/graph/range_resolver.py | 3 +- .../test/integration/command/remote_test.py | 105 +++++++++--------- 7 files changed, 83 insertions(+), 75 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 64b0b8a12de..0f02ad4ba93 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -12,12 +12,12 @@ class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False, filters=None): + def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None): self._name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled - self.filters = filters + self.allowed_packages = allowed_packages @property def name(self): @@ -30,11 +30,11 @@ def __eq__(self, other): self.verify_ssl == other.verify_ssl and self.disabled == other.disabled) def __str__(self): - filtered_msg = "" - if self.filters: - filtered_msg = ", Filtered: {}".format(", ".join(self.filters)) + allowed_msg = "" + if self.allowed_packages: + allowed_msg = ", Allowed packages: {}".format(", ".join(self.allowed_packages)) return "{}: {} [Verify SSL: {}, Enabled: {}{}]".format(self.name, self.url, self.verify_ssl, - not self.disabled, filtered_msg) + not self.disabled, allowed_msg) def __repr__(self): return str(self) diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 531ec6f8f83..d77cb9d252a 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -72,9 +72,9 @@ def remove(self, pattern: str): RemoteRegistry(self._remotes_file).remove(remote.name) users_clean(app.cache.localdb, remote.url) - def update(self, remote_name, url=None, secure=None, disabled=None, index=None, filters=None): + def update(self, remote_name, url=None, secure=None, disabled=None, index=None, allowed_packages=None): RemoteRegistry(self._remotes_file).update(remote_name, url, secure, disabled=disabled, - index=index, filters=filters) + index=index, allowed_packages=allowed_packages) def rename(self, remote_name: str, new_name: str): RemoteRegistry(self._remotes_file).rename(remote_name, new_name) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index e0f140242f8..a28b8c45083 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -72,11 +72,11 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") - subparser.add_argument("--filter", action="append", default=None, - help="Add recipe reference pattern filter to this remote") + subparser.add_argument("--allowed-packages", action="append", default=None, + help="Add recipe reference pattern to list of allowed packages for this remote") subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False, filters=args.filter) + r = Remote(args.name, args.url, args.secure, disabled=False, allowed_packages=args.allowed_packages) conan_api.remotes.add(r, force=args.force, index=args.index) @@ -104,12 +104,13 @@ def remote_update(conan_api, parser, subparser, *args): help="Allow insecure server connections when using SSL") subparser.add_argument("--index", action=OnceArgument, type=int, help="Insert the remote at a specific position in the remote list") - subparser.add_argument("--filter", action="append", default=None, help="Add recipe reference pattern filter to this remote") + subparser.add_argument("--allowed-packages", action="append", default=None, + help="Add recipe reference pattern to list of allowed packages for this remote") subparser.set_defaults(secure=None) args = parser.parse_args(*args) - if args.url is None and args.secure is None and args.index is None and args.filter is None: + if args.url is None and args.secure is None and args.index is None and args.allowed_packages is None: subparser.error("Please add at least one argument to update") - conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index, filters=args.filter) + conan_api.remotes.update(args.remote, args.url, args.secure, index=args.index, allowed_packages=args.allowed_packages) @conan_subcommand() diff --git a/conans/client/cache/remote_registry.py b/conans/client/cache/remote_registry.py index e683c8195db..860d39265eb 100644 --- a/conans/client/cache/remote_registry.py +++ b/conans/client/cache/remote_registry.py @@ -33,9 +33,9 @@ def load(filename): data = json.loads(text) for r in data.get("remotes", []): disabled = r.get("disabled", False) - filters = r.get("filters", None) + allowed_packages = r.get("allowed_packages", None) # TODO: Remote.serialize/deserialize - remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled, filters) + remote = Remote(r["name"], r["url"], r["verify_ssl"], disabled, allowed_packages) result._remotes[r["name"]] = remote return result @@ -45,8 +45,8 @@ def dumps(self): remote = {"name": r.name, "url": r.url, "verify_ssl": r.verify_ssl} if r.disabled: remote["disabled"] = True - if r.filters: - remote["filters"] = r.filters + if r.allowed_packages: + remote["allowed_packages"] = r.allowed_packages remote_list.append(remote) ret = {"remotes": remote_list} return json.dumps(ret, indent=True) @@ -97,7 +97,7 @@ def _check_urls(self, url, force, current): else: ConanOutput().warning(msg + " Adding duplicated remote url because '--force'.") - def update(self, remote_name, url=None, secure=None, disabled=None, index=None, force=False, filters=None): + def update(self, remote_name, url=None, secure=None, disabled=None, index=None, force=False, allowed_packages=None): remote = self[remote_name] if url is not None: self._check_urls(url, force, remote) @@ -113,8 +113,8 @@ def update(self, remote_name, url=None, secure=None, disabled=None, index=None, remotes.insert(index, remote) self._remotes = {r.name: r for r in remotes} - if filters is not None: - remote.filters = filters + if allowed_packages is not None: + remote.allowed_packages = allowed_packages def items(self): return list(self._remotes.values()) @@ -176,11 +176,11 @@ def remove(self, remote_name): remotes.remove(remote_name) self.save_remotes(remotes) - def update(self, remote_name, url, secure, disabled, index, filters): + def update(self, remote_name, url, secure, disabled, index, allowed_packages): if url is not None: self._validate_url(url) remotes = self._load_remotes() - remotes.update(remote_name, url, secure, disabled, index, filters=filters) + remotes.update(remote_name, url, secure, disabled, index, allowed_packages=allowed_packages) self.save_remotes(remotes) def rename(self, remote, new_name): diff --git a/conans/client/graph/proxy.py b/conans/client/graph/proxy.py index 84b03e1104d..324e88942f6 100644 --- a/conans/client/graph/proxy.py +++ b/conans/client/graph/proxy.py @@ -97,7 +97,8 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat results = [] for remote in remotes: - if remote.filters and not any(reference.matches(f, is_consumer=False) for f in remote.filters): + if remote.allowed_packages and not any(reference.matches(f, is_consumer=False) + for f in remote.allowed_packages): output.debug(f"Excluding remote {remote.name} because recipe is filtered out") continue output.info(f"Checking remote: {remote.name}") diff --git a/conans/client/graph/range_resolver.py b/conans/client/graph/range_resolver.py index ad81ecc3cd9..0c0756c86f7 100644 --- a/conans/client/graph/range_resolver.py +++ b/conans/client/graph/range_resolver.py @@ -64,7 +64,8 @@ def _resolve_local(self, search_ref, version_range): return self._resolve_version(version_range, local_found, self._resolve_prereleases) def _search_remote_recipes(self, remote, search_ref): - if remote.filters and not any(search_ref.matches(f, is_consumer=False) for f in remote.filters): + if remote.allowed_packages and not any(search_ref.matches(f, is_consumer=False) + for f in remote.allowed_packages): return [] pattern = str(search_ref) pattern_cached = self._cached_remote_found.setdefault(pattern, {}) diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index 60b3c12e4e6..97e7cc59551 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -1,4 +1,5 @@ import json +import os import unittest from collections import OrderedDict @@ -17,7 +18,7 @@ def setUp(self): test_server = TestServer() self.servers["remote%d" % i] = test_server - self.client = TestClient(servers=self.servers, inputs=3*["admin", "password"], light=True) + self.client = TestClient(servers=self.servers, inputs=3 * ["admin", "password"], light=True) def test_list_json(self): self.client.run("remote list --format=json") @@ -143,8 +144,6 @@ def test_errors(self): self.assertIn("ERROR: Remote 'origin' can't be found or is disabled", self.client.out) - - class RemoteModificationTest(unittest.TestCase): def test_rename(self): client = TestClient(light=True) @@ -346,50 +345,56 @@ def test_add_wrong_conancenter(): assert "the correct remote API is https://center.conan.io" in c.out -class TestRemoteRecipeFilter(unittest.TestCase): - def setUp(self): - self.client = TestClient(light=True, default_server_user=True) - self.client.save({"conanfile.py": GenConanfile(), - "app/conanfile.py": GenConanfile("app", "1.0") - .with_requires("liba/1.0") - .with_requires("libb/[>=1.0]")}) - self.client.run("create . --name=liba --version=1.0") - self.client.run("create . --name=libb --version=1.0") - self.client.run("create app") - self.client.run("upload * -r=default -c") - self.client.run("remove * -c") - - def test_filter_remotes(self): - self.client.run("install --requires=app/1.0 -r=default") - assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out - assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out - self.client.run("remove * -c") - - # lib/2.* pattern does not match the one being required by app, should not be found - self.client.run('remote update default --filter="app/*" --filter="liba/2.*"') - - self.client.run("install --requires=app/1.0 -r=default", assert_error=True) - assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out - assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" not in self.client.out - assert "ERROR: Package 'liba/1.0' not resolved: Unable to find 'liba/1.0' in remotes" in self.client.out - self.client.run("remove * -c") - - self.client.run('remote update default --filter="liba/*" --filter="app/*"') - self.client.run("install --requires=app/1.0 -r=default", assert_error=True) - assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out - assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out - assert "ERROR: Package 'libb/[>=1.0]' not resolved" in self.client.out - self.client.run("remove * -c") - - self.client.run('remote update default --filter="liba/*" --filter="app/*"') - self.client.run("install --requires=app/1.0 -r=default", assert_error=True) - assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out - assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out - assert "ERROR: Package 'libb/[>=1.0]' not resolved" in self.client.out - self.client.run("remove * -c") - - self.client.run('remote update default --filter="*"') - self.client.run("install --requires=app/1.0 -r=default") - assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in self.client.out - assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out - assert "libb/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in self.client.out +def test_allowed_packages_remotes(): + tc = TestClient(light=True, default_server_user=True) + tc.save({"conanfile.py": GenConanfile(), + "app/conanfile.py": GenConanfile("app", "1.0") + .with_requires("liba/1.0") + .with_requires("libb/[>=1.0]")}) + tc.run("create . --name=liba --version=1.0") + tc.run("create . --name=libb --version=1.0") + tc.run("create app") + tc.run("upload * -r=default -c") + tc.run("remove * -c") + + tc.run("install --requires=app/1.0 -r=default") + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in tc.out + tc.run("remove * -c") + + # lib/2.* pattern does not match the one being required by app, should not be found + tc.run('remote update default --allowed-packages="app/*" --allowed-packages="liba/2.*"') + + tc.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" not in tc.out + assert "ERROR: Package 'liba/1.0' not resolved: Unable to find 'liba/1.0' in remotes" in tc.out + tc.run("remove * -c") + + tc.run('remote update default --allowed-packages="liba/*" --allowed-packages="app/*"') + tc.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in tc.out + assert "ERROR: Package 'libb/[>=1.0]' not resolved" in tc.out + tc.run("remove * -c") + + tc.run('remote update default --allowed-packages="liba/*" --allowed-packages="app/*"') + tc.run("install --requires=app/1.0 -r=default", assert_error=True) + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in tc.out + assert "ERROR: Package 'libb/[>=1.0]' not resolved" in tc.out + tc.run("remove * -c") + + tc.run('remote update default --allowed-packages="*"') + tc.run("install --requires=app/1.0 -r=default") + assert "app/1.0: Downloaded recipe revision 04ab3bc4b945a2ee44285962c277906d" in tc.out + assert "liba/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in tc.out + assert "libb/1.0: Downloaded recipe revision 4d670581ccb765839f2239cc8dff8fbd" in tc.out + + +def test_remote_allowed_packages_new(): + """Test that the allowed packages are saved in the remotes.json file when adding a new remote""" + tc = TestClient(light=True) + tc.run("remote add foo https://foo --allowed-packages='liba/*'") + remotes = json.loads(load(os.path.join(tc.cache_folder, "remotes.json"))) + assert remotes[0]["allowed_packages"] == ["liba/*"] From a0817d286663e0d7636538b3a6fcd851b97e8cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Fri, 19 Jan 2024 10:06:37 +0100 Subject: [PATCH 7/8] Fix test --- conans/test/integration/command/remote_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index 97e7cc59551..4ea2c8f4492 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -397,4 +397,4 @@ def test_remote_allowed_packages_new(): tc = TestClient(light=True) tc.run("remote add foo https://foo --allowed-packages='liba/*'") remotes = json.loads(load(os.path.join(tc.cache_folder, "remotes.json"))) - assert remotes[0]["allowed_packages"] == ["liba/*"] + assert remotes["remotes"][0]["allowed_packages"] == ["liba/*"] From 9a8a18505d050fed2f2a00dcbc722f4ce18330b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Fri, 19 Jan 2024 10:15:06 +0100 Subject: [PATCH 8/8] Add shorthand syntax to --allowed-packages --- conan/cli/commands/remote.py | 4 ++-- conans/test/integration/command/remote_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index a28b8c45083..72ccc49b8fb 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -72,7 +72,7 @@ def remote_add(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") - subparser.add_argument("--allowed-packages", action="append", default=None, + subparser.add_argument("-ap", "--allowed-packages", action="append", default=None, help="Add recipe reference pattern to list of allowed packages for this remote") subparser.set_defaults(secure=True) args = parser.parse_args(*args) @@ -104,7 +104,7 @@ def remote_update(conan_api, parser, subparser, *args): help="Allow insecure server connections when using SSL") subparser.add_argument("--index", action=OnceArgument, type=int, help="Insert the remote at a specific position in the remote list") - subparser.add_argument("--allowed-packages", action="append", default=None, + subparser.add_argument("-ap", "--allowed-packages", action="append", default=None, help="Add recipe reference pattern to list of allowed packages for this remote") subparser.set_defaults(secure=None) args = parser.parse_args(*args) diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index 4ea2c8f4492..df291fbb187 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -395,6 +395,6 @@ def test_allowed_packages_remotes(): def test_remote_allowed_packages_new(): """Test that the allowed packages are saved in the remotes.json file when adding a new remote""" tc = TestClient(light=True) - tc.run("remote add foo https://foo --allowed-packages='liba/*'") + tc.run("remote add foo https://foo -ap='liba/*'") remotes = json.loads(load(os.path.join(tc.cache_folder, "remotes.json"))) assert remotes["remotes"][0]["allowed_packages"] == ["liba/*"]