From 618f9f1e35ea30811b14201b6710b0b7e1d85c80 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 21 May 2023 01:13:06 +0200 Subject: [PATCH 01/18] pkglist --- conan/api/model.py | 12 ++++ conan/cli/commands/list.py | 11 +++- conan/cli/commands/upload.py | 21 +++++-- .../command_v2/test_list_and_upload.py | 57 +++++++++++++++++++ 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 conans/test/integration/command_v2/test_list_and_upload.py diff --git a/conan/api/model.py b/conan/api/model.py index cb9420425bb..d6c80a6b009 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -1,8 +1,10 @@ import fnmatch +import json from conans.errors import ConanException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference +from conans.util.files import load class Remote: @@ -92,6 +94,16 @@ def prefs(ref, recipe_bundle): def serialize(self): return self.recipes + @staticmethod + def load(file): + return PackagesList.loads(load(file)) + + @staticmethod + def loads(content): + result = PackagesList() + result.recipes = json.loads(content) + return result + class ListPattern: diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index fc4453b1f7f..41d000777a2 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -86,9 +86,18 @@ def print_list_json(data): cli_out_write(myjson) +def print_pkg_list(data): + # FIXME: Workaround, to serialize only the 1st thing, 1 pkg-list, not the full + # remote-pkglist dict + results = next(iter(data["results"].items()))[1] + myjson = json.dumps(results, indent=4) + cli_out_write(myjson) + + @conan_command(group="Consumer", formatters={"text": print_list_text, "json": print_list_json, - "html": list_packages_html}) + "html": list_packages_html, + "pkglist": print_pkg_list}) def list(conan_api: ConanAPI, parser, *args): """ List existing recipes, revisions, or packages in the cache (by default) or the remotes. diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 27a04591e67..ec1354ffbd0 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -1,5 +1,6 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern +from conan.api.model import ListPattern, PackagesList +from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conans.client.userio import UserInput from conan.errors import ConanException @@ -17,7 +18,8 @@ def upload(conan_api: ConanAPI, parser, *args): """ parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " "wildcard at any reference field. If no revision is " - "specified, it is assumed to be the latest") + "specified, it is assumed to be the latest", + nargs="?") parser.add_argument('-p', '--package-query', default=None, action=OnceArgument, help="Only upload packages matching a specific query. e.g: os=Windows AND " "(arch=x86 OR compiler=gcc)") @@ -33,14 +35,23 @@ def upload(conan_api: ConanAPI, parser, *args): help='Perform an integrity check, using the manifests, before upload') parser.add_argument('-c', '--confirm', default=False, action='store_true', help='Upload all matching recipes without confirmation') + parser.add_argument("-l", "--list", help="Package list file") args = parser.parse_args(*args) remote = conan_api.remotes.get(args.remote) enabled_remotes = conan_api.remotes.list() - ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe) - package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) + if args.reference is None and args.list is None: + raise ConanException("Missing pattern or package list file") + if args.reference and args.list: + raise ConanException("Cannot define both the pattern and the package list file") + if args.list: + listfile = make_abs_path(args.list) + package_list = PackagesList.load(listfile) + else: + ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe) + package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) if not package_list.recipes: raise ConanException("No recipes found matching pattern '{}'".format(args.reference)) @@ -51,7 +62,7 @@ def upload(conan_api: ConanAPI, parser, *args): conan_api.upload.check_upstream(package_list, remote, args.force) # If only if search with "*" we ask for confirmation - if not args.confirm and "*" in args.reference: + if not args.list and not args.confirm and "*" in args.reference: _ask_confirm_upload(conan_api, package_list) conan_api.upload.prepare(package_list, enabled_remotes) diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_list_and_upload.py new file mode 100644 index 00000000000..f874f08dcd7 --- /dev/null +++ b/conans/test/integration/command_v2/test_list_and_upload.py @@ -0,0 +1,57 @@ +import time + +import pytest + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient +from conans.util.env import environment_update + + +# TODO: optimize this fixture +@pytest.fixture() +def client(): + c = TestClient(default_server_user=True) + c.save({ + "zlib.py": GenConanfile("zlib"), + "zlib_ng.py": GenConanfile("zlib_ng", "1.0.0"), + "zli.py": GenConanfile("zli", "1.0.0"), + "zli_rev2.py": GenConanfile("zli", "1.0.0").with_settings("os") + .with_package_file("f.txt", env_var="MYREV"), + "zlix.py": GenConanfile("zlix", "1.0.0") + }) + c.run("create zli.py") + c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") + c.run("create zlib.py --version=2.0.0 --user=user --channel=channel") + c.run("create zlix.py") + + time.sleep(1.0) + # We create and upload new revisions later, to avoid timestamp overlaps (low resolution) + with environment_update({"MYREV": "0"}): + c.run("create zli_rev2.py -s os=Windows") + c.run("create zli_rev2.py -s os=Linux") + with environment_update({"MYREV": "42"}): + c.run("create zli_rev2.py -s os=Windows") + return c + + +class TestListRefs: + + def test_list_upload_recipes(self, client): + pattern = "z*#latest" + client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") + client.run("upload --list=pkglist.json -r=default") + assert "Uploading recipe 'zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360'" in client.out + assert "Uploading recipe 'zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out + assert "Uploading recipe 'zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out + assert "Uploading recipe 'zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e'" in client.out + assert "Uploading package" not in client.out + + def test_list_upload_packages(self, client): + pattern = "z*#latest:*#latest" + client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") + client.run("upload --list=pkglist.json -r=default") + assert "Uploading recipe 'zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360'" in client.out + assert "Uploading recipe 'zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out + assert "Uploading recipe 'zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out + assert "Uploading recipe 'zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e'" in client.out + assert str(client.out).count("Uploading package") == 5 From 36735fad81eabaa90df3d8ab1f539974cdf74c96 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 21 May 2023 17:39:46 +0200 Subject: [PATCH 02/18] wip --- conan/cli/commands/list.py | 43 ++++++++++++++++++- .../command_v2/test_list_and_upload.py | 34 +++++++++++---- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 41d000777a2..008e4e8140c 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -1,13 +1,19 @@ import json from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern +from conan.api.model import ListPattern, PackagesList from conan.api.output import Color, cli_out_write +from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conan.cli.formatters.list import list_packages_html # Keep them so we don't break other commands that import them, but TODO: Remove later +from conans.client.graph.graph import BINARY_BUILD +from conans.errors import ConanException +from conans.model.package_ref import PkgReference +from conans.model.recipe_ref import RecipeReference from conans.util.dates import timestamp_to_str +from conans.util.files import load remote_color = Color.BRIGHT_BLUE recipe_name_color = Color.GREEN @@ -104,15 +110,48 @@ def list(conan_api: ConanAPI, parser, *args): """ parser.add_argument('reference', help="Recipe reference or package reference. " "Both can contain * as wildcard at any reference field. " - "If revision is not specified, it is assumed latest one.") + "If revision is not specified, it is assumed latest one.", + nargs="?") parser.add_argument('-p', '--package-query', default=None, action=OnceArgument, help="List only the packages matching a specific query, e.g, os=Windows AND " "(arch=x86 OR compiler=gcc)") parser.add_argument("-r", "--remote", default=None, action="append", help="Remote names. Accepts wildcards ('*' means all the remotes available)") parser.add_argument("-c", "--cache", action='store_true', help="Search in the local cache") + parser.add_argument("-g", "--graph", help="Graph json file") args = parser.parse_args(*args) + + if args.reference is None and args.graph is None: + raise ConanException("Missing pattern or graph json file") + if args.reference and args.graph: + raise ConanException("Cannot define both the pattern and the graph json file") + + if args.graph: + graphfile = make_abs_path(args.graph) + graph = json.loads(load(graphfile)) + pkglist = PackagesList() + for n in graph["graph"]["nodes"]: + ref = n["ref"] + if ref == "conanfile": + continue + ref = RecipeReference.loads(ref) + ref.timestamp = "1" # FIXME + binary = n["binary"] + if binary == BINARY_BUILD: + pkglist.add_refs([ref]) + + pref = PkgReference(ref, n["package_id"], n["prev"]) + pref.timestamp = "1" # FIXME + pkglist.add_prefs(ref, [pref]) + + return { + "results": {"Local Cache": pkglist.serialize()}, + "conan_api": conan_api, + "cli_args": " ".join( + [f"{arg}={getattr(args, arg)}" for arg in vars(args) if getattr(args, arg)]) + } + ref_pattern = ListPattern(args.reference, rrev=None, prev=None) # If neither remote nor cache are defined, show results only from cache remotes = [] diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_list_and_upload.py index f874f08dcd7..e33b71f39df 100644 --- a/conans/test/integration/command_v2/test_list_and_upload.py +++ b/conans/test/integration/command_v2/test_list_and_upload.py @@ -34,24 +34,40 @@ def client(): return c -class TestListRefs: +class TestListUpload: + refs = ["zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360", + "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb", + "zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb", + "zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e"] def test_list_upload_recipes(self, client): pattern = "z*#latest" client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") client.run("upload --list=pkglist.json -r=default") - assert "Uploading recipe 'zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360'" in client.out - assert "Uploading recipe 'zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out - assert "Uploading recipe 'zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out - assert "Uploading recipe 'zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e'" in client.out + for r in self.refs: + assert f"Uploading recipe '{r}'" in client.out assert "Uploading package" not in client.out def test_list_upload_packages(self, client): pattern = "z*#latest:*#latest" client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") client.run("upload --list=pkglist.json -r=default") - assert "Uploading recipe 'zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360'" in client.out - assert "Uploading recipe 'zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out - assert "Uploading recipe 'zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb'" in client.out - assert "Uploading recipe 'zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e'" in client.out + for r in self.refs: + assert f"Uploading recipe '{r}'" in client.out assert str(client.out).count("Uploading package") == 5 + + +class TestGraphCreatedUpload: + def test_graph_created_upload(self): + c = TestClient(default_server_user=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) + c.run("create zlib") + c.run("create app --format=json", redirect_stdout="graph.json") + c.run("list --graph=graph.json --format=pkglist", redirect_stdout="pkglist.json") + print(c.load("pkglist.json")) + c.run("upload --list=pkglist.json -r=default") + print(c.out) + assert "Uploading recipe 'app/1.0#0fa1ff1b90576bb782600e56df642e19'" in c.out + assert "zlib" not in c.out + assert "Uploading package 'app" in c.out From 8a525ecf256b2b2e268753741d6669d1114e30ea Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 21 May 2023 18:01:53 +0200 Subject: [PATCH 03/18] fix test --- conans/test/integration/command_v2/list_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conans/test/integration/command_v2/list_test.py b/conans/test/integration/command_v2/list_test.py index 6bd19fb6c9f..84ff71b3d12 100644 --- a/conans/test/integration/command_v2/list_test.py +++ b/conans/test/integration/command_v2/list_test.py @@ -20,16 +20,16 @@ class TestParamErrors: def test_query_param_is_required(self): c = TestClient() c.run("list", assert_error=True) - assert "error: the following arguments are required: reference" in c.out + assert "ERROR: Missing pattern or graph json file" in c.out c.run("list -c", assert_error=True) - assert "error: the following arguments are required: reference" in c.out + assert "ERROR: Missing pattern or graph json file" in c.out c.run('list -r="*"', assert_error=True) - assert "error: the following arguments are required: reference" in c.out + assert "ERROR: Missing pattern or graph json file" in c.out c.run("list --remote remote1 --cache", assert_error=True) - assert "error: the following arguments are required: reference" in c.out + assert "ERROR: Missing pattern or graph json file" in c.out @pytest.fixture(scope="module") From eb69a433ac3163daaa9eb270aa1cdd424b6b093e Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 23 May 2023 23:04:06 +0200 Subject: [PATCH 04/18] changed PackageList approach to include multiple remotes --- conan/api/model.py | 41 ++++++++++++++++--- conan/cli/commands/list.py | 38 +++++++---------- conan/cli/commands/upload.py | 5 ++- .../command_v2/test_list_and_upload.py | 8 ++-- 4 files changed, 55 insertions(+), 37 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index d6c80a6b009..e3543f83903 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -35,6 +35,39 @@ def __repr__(self): return str(self) +class MultiPackagesList: + def __init__(self): + self.lists = {} + + def __getitem__(self, name): + try: + return self.lists[name] + except KeyError: + raise ConanException(f"'{name}' doesn't exist is package list") + + def add(self, name, pkg_list): + self.lists[name] = pkg_list + + def add_error(self, remote_name, error): + self.lists[remote_name] = {"error": error} + + def serialize(self): + return {k: v.serialize() for k, v in self.lists.items()} + + @staticmethod + def load(file): + content = json.loads(load(file)) + result = {} + for remote, pkglist in content.items(): + if "error" in pkglist: + result[remote] = pkglist + else: + result[remote] = PackagesList.deserialize(pkglist) + pkglist = MultiPackagesList() + pkglist.lists = result + return result + + class PackagesList: def __init__(self): self.recipes = {} @@ -95,13 +128,9 @@ def serialize(self): return self.recipes @staticmethod - def load(file): - return PackagesList.loads(load(file)) - - @staticmethod - def loads(content): + def deserialize(data): result = PackagesList() - result.recipes = json.loads(content) + result.recipes = data return result diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 008e4e8140c..b8b3dba7e38 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -1,7 +1,7 @@ import json from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, PackagesList +from conan.api.model import ListPattern, PackagesList, MultiPackagesList from conan.api.output import Color, cli_out_write from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument @@ -92,18 +92,9 @@ def print_list_json(data): cli_out_write(myjson) -def print_pkg_list(data): - # FIXME: Workaround, to serialize only the 1st thing, 1 pkg-list, not the full - # remote-pkglist dict - results = next(iter(data["results"].items()))[1] - myjson = json.dumps(results, indent=4) - cli_out_write(myjson) - - @conan_command(group="Consumer", formatters={"text": print_list_text, "json": print_list_json, - "html": list_packages_html, - "pkglist": print_pkg_list}) + "html": list_packages_html}) def list(conan_api: ConanAPI, parser, *args): """ List existing recipes, revisions, or packages in the cache (by default) or the remotes. @@ -154,23 +145,22 @@ def list(conan_api: ConanAPI, parser, *args): ref_pattern = ListPattern(args.reference, rrev=None, prev=None) # If neither remote nor cache are defined, show results only from cache - remotes = [] + pkglist = MultiPackagesList() if args.cache or not args.remote: - remotes.append(None) + cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None) + pkglist.add("Local Cache", cache_list) if args.remote: - remotes.extend(conan_api.remotes.list(args.remote)) - results = {} - for remote in remotes: - name = getattr(remote, "name", "Local Cache") - try: - list_bundle = conan_api.list.select(ref_pattern, args.package_query, remote) - except Exception as e: - results[name] = {"error": str(e)} - else: - results[name] = list_bundle.serialize() + remotes = conan_api.remotes.list(args.remote) + for remote in remotes: + try: + remote_list = conan_api.list.select(ref_pattern, args.package_query, remote) + except Exception as e: + pkglist.add_error(remote.name, str(e)) + else: + pkglist.add(remote.name, remote_list) return { - "results": results, + "results": pkglist.serialize(), "conan_api": conan_api, "cli_args": " ".join([f"{arg}={getattr(args, arg)}" for arg in vars(args) if getattr(args, arg)]) } diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index ec1354ffbd0..39d748365de 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -1,5 +1,5 @@ from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, PackagesList +from conan.api.model import ListPattern, MultiPackagesList from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conans.client.userio import UserInput @@ -48,7 +48,8 @@ def upload(conan_api: ConanAPI, parser, *args): raise ConanException("Cannot define both the pattern and the package list file") if args.list: listfile = make_abs_path(args.list) - package_list = PackagesList.load(listfile) + multi_package_list = MultiPackagesList.load(listfile) + package_list = multi_package_list["Local Cache"] else: ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_list_and_upload.py index e33b71f39df..5ecdae73cb7 100644 --- a/conans/test/integration/command_v2/test_list_and_upload.py +++ b/conans/test/integration/command_v2/test_list_and_upload.py @@ -42,7 +42,7 @@ class TestListUpload: def test_list_upload_recipes(self, client): pattern = "z*#latest" - client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") + client.run(f"list {pattern} --format=json", redirect_stdout="pkglist.json") client.run("upload --list=pkglist.json -r=default") for r in self.refs: assert f"Uploading recipe '{r}'" in client.out @@ -50,7 +50,7 @@ def test_list_upload_recipes(self, client): def test_list_upload_packages(self, client): pattern = "z*#latest:*#latest" - client.run(f"list {pattern} --format=pkglist", redirect_stdout="pkglist.json") + client.run(f"list {pattern} --format=json", redirect_stdout="pkglist.json") client.run("upload --list=pkglist.json -r=default") for r in self.refs: assert f"Uploading recipe '{r}'" in client.out @@ -64,10 +64,8 @@ def test_graph_created_upload(self): "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) c.run("create zlib") c.run("create app --format=json", redirect_stdout="graph.json") - c.run("list --graph=graph.json --format=pkglist", redirect_stdout="pkglist.json") - print(c.load("pkglist.json")) + c.run("list --graph=graph.json --format=json", redirect_stdout="pkglist.json") c.run("upload --list=pkglist.json -r=default") - print(c.out) assert "Uploading recipe 'app/1.0#0fa1ff1b90576bb782600e56df642e19'" in c.out assert "zlib" not in c.out assert "Uploading package 'app" in c.out From 6d29cbce858bc9bd90c89a1421842bd83857b55c Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 May 2023 10:00:27 +0200 Subject: [PATCH 05/18] fixes --- conan/api/model.py | 3 ++- conan/cli/commands/list.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index e3543f83903..80eea169aab 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -52,7 +52,8 @@ def add_error(self, remote_name, error): self.lists[remote_name] = {"error": error} def serialize(self): - return {k: v.serialize() for k, v in self.lists.items()} + return {k: v.serialize() if isinstance(v, PackagesList) else v + for k, v in self.lists.items()} @staticmethod def load(file): diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index b8b3dba7e38..4050b4bd460 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -147,8 +147,12 @@ def list(conan_api: ConanAPI, parser, *args): # If neither remote nor cache are defined, show results only from cache pkglist = MultiPackagesList() if args.cache or not args.remote: - cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None) - pkglist.add("Local Cache", cache_list) + try: + cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None) + except Exception as e: + pkglist.add_error("Local Cache", str(e)) + else: + pkglist.add("Local Cache", cache_list) if args.remote: remotes = conan_api.remotes.list(args.remote) for remote in remotes: From ad7b1d8e9d0006dd143ed549855b2be18ab60da9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 May 2023 11:42:57 +0200 Subject: [PATCH 06/18] review --- conan/api/model.py | 20 ++++++++++++++++++++ conan/cli/commands/list.py | 25 ++++--------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 80eea169aab..2f93ea3a7ce 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -1,6 +1,7 @@ import fnmatch import json +from conans.client.graph.graph import BINARY_BUILD from conans.errors import ConanException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference @@ -68,6 +69,25 @@ def load(file): pkglist.lists = result return result + @staticmethod + def from_graph(graph): + pkglist = MultiPackagesList() + cache_list = PackagesList() + pkglist.lists["Local Cache"] = cache_list + for id_, node in graph["graph"]["nodes"].items(): + ref = node["ref"] + if ref == "conanfile": + continue + ref = RecipeReference.loads(ref) + ref.timestamp = "1" # FIXME + binary = node["binary"] + if binary == BINARY_BUILD: + cache_list.add_refs([ref]) + pref = PkgReference(ref, node["package_id"], node["prev"]) + pref.timestamp = "1" # FIXME + cache_list.add_prefs(ref, [pref]) + return pkglist + class PackagesList: def __init__(self): diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 4050b4bd460..ea9d4f7671c 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -6,15 +6,12 @@ from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument from conan.cli.formatters.list import list_packages_html - -# Keep them so we don't break other commands that import them, but TODO: Remove later -from conans.client.graph.graph import BINARY_BUILD from conans.errors import ConanException -from conans.model.package_ref import PkgReference -from conans.model.recipe_ref import RecipeReference from conans.util.dates import timestamp_to_str from conans.util.files import load + +# Keep them so we don't break other commands that import them, but TODO: Remove later remote_color = Color.BRIGHT_BLUE recipe_name_color = Color.GREEN recipe_color = Color.BRIGHT_WHITE @@ -121,23 +118,9 @@ def list(conan_api: ConanAPI, parser, *args): if args.graph: graphfile = make_abs_path(args.graph) graph = json.loads(load(graphfile)) - pkglist = PackagesList() - for n in graph["graph"]["nodes"]: - ref = n["ref"] - if ref == "conanfile": - continue - ref = RecipeReference.loads(ref) - ref.timestamp = "1" # FIXME - binary = n["binary"] - if binary == BINARY_BUILD: - pkglist.add_refs([ref]) - - pref = PkgReference(ref, n["package_id"], n["prev"]) - pref.timestamp = "1" # FIXME - pkglist.add_prefs(ref, [pref]) - + pkglist = MultiPackagesList.from_graph(graph) return { - "results": {"Local Cache": pkglist.serialize()}, + "results": pkglist.serialize(), "conan_api": conan_api, "cli_args": " ".join( [f"{arg}={getattr(args, arg)}" for arg in vars(args) if getattr(args, arg)]) From 6d2bc7c08f4257685890a32ef73c4cd528d1150f Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 25 May 2023 00:18:06 +0200 Subject: [PATCH 07/18] wip --- conan/api/model.py | 14 +++--- conan/cli/commands/list.py | 3 +- .../command_v2/test_list_and_upload.py | 46 ++++++++----------- 3 files changed, 29 insertions(+), 34 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 2f93ea3a7ce..a0335671f3e 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -1,7 +1,6 @@ import fnmatch import json -from conans.client.graph.graph import BINARY_BUILD from conans.errors import ConanException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference @@ -70,21 +69,20 @@ def load(file): return result @staticmethod - def from_graph(graph): + def from_graph(graph, graph_binaries=None): pkglist = MultiPackagesList() cache_list = PackagesList() + graph_binaries = ["*"] if graph_binaries is None else graph_binaries pkglist.lists["Local Cache"] = cache_list - for id_, node in graph["graph"]["nodes"].items(): + for node in graph["graph"]["nodes"].values(): ref = node["ref"] if ref == "conanfile": continue ref = RecipeReference.loads(ref) - ref.timestamp = "1" # FIXME binary = node["binary"] - if binary == BINARY_BUILD: + if any(fnmatch.fnmatch(binary, b) for b in graph_binaries): cache_list.add_refs([ref]) pref = PkgReference(ref, node["package_id"], node["prev"]) - pref.timestamp = "1" # FIXME cache_list.add_prefs(ref, [pref]) return pkglist @@ -130,7 +128,9 @@ def refs(self): for ref, ref_dict in self.recipes.items(): for rrev, rrev_dict in ref_dict.get("revisions", {}).items(): t = rrev_dict.get("timestamp") - recipe = RecipeReference.loads(f"{ref}#{rrev}%{t}") # TODO: optimize this + recipe = RecipeReference.loads(f"{ref}#{rrev}") # TODO: optimize this + if t is not None: + recipe.timestamp = t result[recipe] = rrev_dict return result.items() diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index ea9d4f7671c..d96118bf296 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -107,6 +107,7 @@ def list(conan_api: ConanAPI, parser, *args): help="Remote names. Accepts wildcards ('*' means all the remotes available)") parser.add_argument("-c", "--cache", action='store_true', help="Search in the local cache") parser.add_argument("-g", "--graph", help="Graph json file") + parser.add_argument("-gb", "--graph-binaries", help="Which binaries are listed", action="append") args = parser.parse_args(*args) @@ -118,7 +119,7 @@ def list(conan_api: ConanAPI, parser, *args): if args.graph: graphfile = make_abs_path(args.graph) graph = json.loads(load(graphfile)) - pkglist = MultiPackagesList.from_graph(graph) + pkglist = MultiPackagesList.from_graph(graph, args.graph_binaries) return { "results": pkglist.serialize(), "conan_api": conan_api, diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_list_and_upload.py index 5ecdae73cb7..60d41a27d30 100644 --- a/conans/test/integration/command_v2/test_list_and_upload.py +++ b/conans/test/integration/command_v2/test_list_and_upload.py @@ -1,44 +1,24 @@ -import time - import pytest from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient -from conans.util.env import environment_update -# TODO: optimize this fixture @pytest.fixture() def client(): c = TestClient(default_server_user=True) c.save({ "zlib.py": GenConanfile("zlib"), - "zlib_ng.py": GenConanfile("zlib_ng", "1.0.0"), - "zli.py": GenConanfile("zli", "1.0.0"), - "zli_rev2.py": GenConanfile("zli", "1.0.0").with_settings("os") - .with_package_file("f.txt", env_var="MYREV"), - "zlix.py": GenConanfile("zlix", "1.0.0") + "zli.py": GenConanfile("zli", "1.0.0") }) c.run("create zli.py") c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") - c.run("create zlib.py --version=2.0.0 --user=user --channel=channel") - c.run("create zlix.py") - - time.sleep(1.0) - # We create and upload new revisions later, to avoid timestamp overlaps (low resolution) - with environment_update({"MYREV": "0"}): - c.run("create zli_rev2.py -s os=Windows") - c.run("create zli_rev2.py -s os=Linux") - with environment_update({"MYREV": "42"}): - c.run("create zli_rev2.py -s os=Windows") return c class TestListUpload: - refs = ["zli/1.0.0#b58eeddfe2fd25ac3a105f72836b3360", - "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb", - "zlib/2.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb", - "zlix/1.0.0#81f598d1d8648389bb7d0494fffb654e"] + refs = ["zli/1.0.0#f034dc90894493961d92dd32a9ee3b78", + "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb"] def test_list_upload_recipes(self, client): pattern = "z*#latest" @@ -54,11 +34,11 @@ def test_list_upload_packages(self, client): client.run("upload --list=pkglist.json -r=default") for r in self.refs: assert f"Uploading recipe '{r}'" in client.out - assert str(client.out).count("Uploading package") == 5 + assert str(client.out).count("Uploading package") == 2 class TestGraphCreatedUpload: - def test_graph_created_upload(self): + def test_create_upload(self): c = TestClient(default_server_user=True) c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) @@ -67,5 +47,19 @@ def test_graph_created_upload(self): c.run("list --graph=graph.json --format=json", redirect_stdout="pkglist.json") c.run("upload --list=pkglist.json -r=default") assert "Uploading recipe 'app/1.0#0fa1ff1b90576bb782600e56df642e19'" in c.out - assert "zlib" not in c.out + assert "Uploading recipe 'zlib/1.0#c570d63921c5f2070567da4bf64ff261'" in c.out assert "Uploading package 'app" in c.out + + +class TestGraphPkgList: + def test_graph_pkg_list(self): + c = TestClient() + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) + c.run("create zlib") + c.run("create app --format=json", redirect_stdout="graph.json") + c.run("list --graph=graph.json --graph-binaries=build --format=json", + redirect_stdout="pkglist.json") + pkglist = c.load("pkglist.json") + assert "app/1.0" in pkglist + assert "zlib" not in pkglist From 50398de3c33d4fe666ce8d1a8bacf0301c25f59f Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 24 May 2023 16:00:59 +0200 Subject: [PATCH 08/18] Conan 2.0.6 --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index d3fa080bacb..08c8f67c8cd 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -2,4 +2,4 @@ REVISIONS = "revisions" # Only when enabled in config, not by default look at server_launcher.py OAUTH_TOKEN = "oauth_token" -__version__ = '2.0.5' +__version__ = '2.0.6' From ad22ad35c2cff5a63e20269666dde98bdbc6f3b0 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Thu, 25 May 2023 12:04:20 +0200 Subject: [PATCH 09/18] Fix format for id property in graph output json (#13964) * convert to str * franchus review --- conans/client/graph/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index 96170d65b97..e851093b3e7 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -381,7 +381,7 @@ def report_graph_error(self): def serialize(self): for i, n in enumerate(self.nodes): - n.id = i + n.id = str(i) result = OrderedDict() result["nodes"] = {n.id: n.serialize() for n in self.nodes} result["root"] = {self.root.id: repr(self.root.ref)} # TODO: ref of consumer/virtual From d3d9fbfc2d8b0a855c72165cdf6eb243ed63d540 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 25 May 2023 18:15:10 +0200 Subject: [PATCH 10/18] test_requires shouldn't affect package_id (#13966) * test_requires shouldn't affect package_id * test with transitive * remove prints --- conans/model/requires.py | 4 +- .../test_package_id_test_requires.py | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 conans/test/integration/package_id/test_package_id_test_requires.py diff --git a/conans/model/requires.py b/conans/model/requires.py index 89eb80a36ef..e2632dd0676 100644 --- a/conans/model/requires.py +++ b/conans/model/requires.py @@ -355,13 +355,15 @@ def deduce_package_id_mode(self, pkg_type, dep_node, non_embed_mode, embed_mode, if self.package_id_mode: return + if self.test: + return # test_requires never affect the binary_id dep_conanfile = dep_node.conanfile dep_pkg_type = dep_conanfile.package_type if self.build: build_mode = getattr(dep_conanfile, "build_mode", build_mode) if build_mode and self.direct: self.package_id_mode = build_mode - return # At the moment no defaults + return if pkg_type is PackageType.HEADER: self.package_id_mode = "unrelated_mode" diff --git a/conans/test/integration/package_id/test_package_id_test_requires.py b/conans/test/integration/package_id/test_package_id_test_requires.py new file mode 100644 index 00000000000..ff5703e621b --- /dev/null +++ b/conans/test/integration/package_id/test_package_id_test_requires.py @@ -0,0 +1,39 @@ +import pytest + +from conans.test.utils.tools import GenConanfile, TestClient +from conans.util.files import save + + +@pytest.mark.parametrize("build_mode", [None, "patch_mode"]) +def test_package_id_not_affected_test_requires(build_mode): + """ + By default, test_requires do not affect the package_id + """ + c = TestClient() + if build_mode is not None: + save(c.cache.new_config_path, "core.package_id:default_build_mode={build_mode}") + c.save({"gtest/conanfile.py": GenConanfile("gtest", "1.0"), + "engine/conanfile.py": GenConanfile("engine", "1.0").with_test_requires("gtest/1.0")}) + c.run("create gtest") + c.run("create engine") + c.run("list engine:*") + assert "engine/1.0" in c.out + assert "gtest" not in c.out + + +def test_package_id_not_affected_test_requires_transitive(): + """ + By default, transitive deps of test_requires do not affect the package_id + """ + c = TestClient() + + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "gtest/conanfile.py": GenConanfile("gtest", "1.0").with_requires("zlib/1.0"), + "engine/conanfile.py": GenConanfile("engine", "1.0").with_test_requires("gtest/1.0")}) + c.run("create zlib") + c.run("create gtest") + c.run("create engine") + c.run("list engine:*") + assert "engine/1.0" in c.out + assert "gtest" not in c.out + assert "zlib" not in c.out From 2609566dd8d4b5d0f535705bb6fe824ae87150a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Fri, 26 May 2023 10:38:04 +0200 Subject: [PATCH 11/18] [graph][json] Same json output for conan graph info, create, install and export-pkg (#13967) * Ensuring the output for the rest of commands * export-pkg too * graph as first output level * wip * typo * typos --- conan/cli/commands/create.py | 17 ++++++----------- conan/cli/commands/export_pkg.py | 18 +++++++----------- conan/cli/commands/install.py | 17 +++++++---------- conan/cli/formatters/graph/graph.py | 8 ++++---- conan/cli/formatters/graph/graph_info_text.py | 2 +- .../tools/system/package_manager_test.py | 5 +++-- conans/test/integration/command/create_test.py | 4 ++-- .../integration/command/export_pkg_test.py | 2 +- .../test/integration/command/info/info_test.py | 10 +++++----- .../command/info/test_info_folders.py | 12 ++++++------ .../integration/graph/test_validate_build.py | 4 ++-- .../version_range_override_test.py | 4 ++-- .../lockfile/test_graph_overrides.py | 16 ++++++++-------- .../integration/package_id/test_validate.py | 4 ++-- 14 files changed, 56 insertions(+), 67 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index f1d81814f51..361d60941d8 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -1,24 +1,18 @@ -import json import os import shutil -from conan.api.output import ConanOutput, cli_out_write +from conan.api.output import ConanOutput +from conan.cli.args import add_lockfile_args, add_common_install_arguments from conan.cli.command import conan_command, OnceArgument from conan.cli.commands.export import common_args_export -from conan.cli.args import add_lockfile_args, add_common_install_arguments +from conan.cli.formatters.graph import format_graph_json from conan.cli.printers import print_profiles from conan.cli.printers.graph import print_graph_packages, print_graph_basic from conan.errors import ConanException from conans.util.files import mkdir -def json_create(deps_graph): - if deps_graph is None: - return - cli_out_write(json.dumps({"graph": deps_graph.serialize()}, indent=4)) - - -@conan_command(group="Creator", formatters={"json": json_create}) +@conan_command(group="Creator", formatters={"json": format_graph_json}) def create(conan_api, parser, *args): """ Create a package. @@ -102,7 +96,8 @@ def create(conan_api, parser, *args): clean=args.lockfile_clean) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) - return deps_graph + return {"graph": deps_graph, + "conan_api": conan_api} def _check_tested_reference_matches(deps_graph, tested_ref, out): diff --git a/conan/cli/commands/export_pkg.py b/conan/cli/commands/export_pkg.py index b030ce165e2..673330836c6 100644 --- a/conan/cli/commands/export_pkg.py +++ b/conan/cli/commands/export_pkg.py @@ -1,20 +1,15 @@ -import json import os -from conan.api.output import cli_out_write, ConanOutput -from conan.cli.command import conan_command, OnceArgument -from conan.cli.args import add_lockfile_args, add_profiles_args, add_reference_args +from conan.api.output import ConanOutput from conan.cli import make_abs_path +from conan.cli.args import add_lockfile_args, add_profiles_args, add_reference_args +from conan.cli.command import conan_command, OnceArgument from conan.cli.commands.create import _get_test_conanfile_path +from conan.cli.formatters.graph import format_graph_json from conan.cli.printers.graph import print_graph_basic -def json_export_pkg(info): - deps_graph = info - cli_out_write(json.dumps({"graph": deps_graph.serialize()}, indent=4)) - - -@conan_command(group="Creator", formatters={"json": json_export_pkg}) +@conan_command(group="Creator", formatters={"json": format_graph_json}) def export_pkg(conan_api, parser, *args): """ Create a package directly from pre-compiled binaries. @@ -100,4 +95,5 @@ def export_pkg(conan_api, parser, *args): # TODO: Do something with lockfile, same as create() conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) - return deps_graph + return {"graph": deps_graph, + "conan_api": conan_api} diff --git a/conan/cli/commands/install.py b/conan/cli/commands/install.py index f6934fb0e48..74c514a5944 100644 --- a/conan/cli/commands/install.py +++ b/conan/cli/commands/install.py @@ -1,20 +1,15 @@ -import json import os -from conan.api.output import ConanOutput, cli_out_write +from conan.api.output import ConanOutput +from conan.cli import make_abs_path from conan.cli.args import common_graph_args, validate_common_graph_args from conan.cli.command import conan_command -from conan.cli import make_abs_path +from conan.cli.formatters.graph import format_graph_json from conan.cli.printers import print_profiles from conan.cli.printers.graph import print_graph_packages, print_graph_basic -def json_install(info): - deps_graph = info - cli_out_write(json.dumps({"graph": deps_graph.serialize()}, indent=4)) - - -@conan_command(group="Consumer", formatters={"json": json_install}) +@conan_command(group="Consumer", formatters={"json": format_graph_json}) def install(conan_api, parser, *args): """ Install the requirements specified in a recipe (conanfile.py or conanfile.txt). @@ -95,4 +90,6 @@ def install(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, clean=args.lockfile_clean) conan_api.lockfile.save_lockfile(lockfile, args.lockfile_out, cwd) - return deps_graph + return {"graph": deps_graph, + "conan_api": conan_api} + diff --git a/conan/cli/formatters/graph/graph.py b/conan/cli/formatters/graph/graph.py index 878a90758a0..92bb7635864 100644 --- a/conan/cli/formatters/graph/graph.py +++ b/conan/cli/formatters/graph/graph.py @@ -127,11 +127,11 @@ def format_graph_dot(result): def format_graph_json(result): graph = result["graph"] - field_filter = result["field_filter"] - package_filter = result["package_filter"] + field_filter = result.get("field_filter") + package_filter = result.get("package_filter") serial = graph.serialize() - serial = filter_graph(serial, package_filter, field_filter) - json_result = json.dumps(serial, indent=4) + serial = filter_graph(serial, package_filter=package_filter, field_filter=field_filter) + json_result = json.dumps({"graph": serial}, indent=4) cli_out_write(json_result) if graph.error: raise graph.error diff --git a/conan/cli/formatters/graph/graph_info_text.py b/conan/cli/formatters/graph/graph_info_text.py index 2da47f2172e..d7ad8d9c904 100644 --- a/conan/cli/formatters/graph/graph_info_text.py +++ b/conan/cli/formatters/graph/graph_info_text.py @@ -4,7 +4,7 @@ from conan.api.output import ConanOutput -def filter_graph(graph, package_filter, field_filter=None): +def filter_graph(graph, package_filter=None, field_filter=None): if package_filter is not None: graph["nodes"] = {id_: n for id_, n in graph["nodes"].items() if any(fnmatch.fnmatch(n["ref"] or "", p) for p in package_filter)} diff --git a/conans/test/functional/tools/system/package_manager_test.py b/conans/test/functional/tools/system/package_manager_test.py index d9b485020b5..8ec00fb5a50 100644 --- a/conans/test/functional/tools/system/package_manager_test.py +++ b/conans/test/functional/tools/system/package_manager_test.py @@ -186,7 +186,8 @@ def system_requirements(self): redirect_stdout="graph2.json") graph2 = json.loads(client.load("graph2.json")) # TODO: Unify format of ``graph info`` and ``install`` - assert {"apt-get": {"install": ["pkg1", "pkg2"]}} == graph2["nodes"]["0"]["system_requires"] + assert {"apt-get": {"install": ["pkg1", "pkg2"]}} == \ + graph2["graph"]["nodes"]["0"]["system_requires"] # Check report-installed with patch.object(_SystemPackageManagerTool, '_conanfile_run', MagicMock(return_value=True)): @@ -195,7 +196,7 @@ def system_requirements(self): redirect_stdout="graph2.json") graph2 = json.loads(client.load("graph2.json")) assert {"apt-get": {"install": ["pkg1", "pkg2"], - 'missing': ['pkg1', 'pkg2']}} == graph2["nodes"]["0"]["system_requires"] + 'missing': ['pkg1', 'pkg2']}} == graph2["graph"]["nodes"]["0"]["system_requires"] # Default "check" will fail, as dpkg-query not installed client.run("graph info . -c tools.system.package_manager:tool=apt-get " diff --git a/conans/test/integration/command/create_test.py b/conans/test/integration/command/create_test.py index 72716cae5e4..1749d850230 100644 --- a/conans/test/integration/command/create_test.py +++ b/conans/test/integration/command/create_test.py @@ -436,7 +436,7 @@ class MyTest(ConanFile): pkg_pkg_ref = 'pkg/0.2#db78b8d06a78af5c3ac56706f133098d' consumer_info = hello_pkg_info = pkg_pkg_info = None - for _, n in nodes.items(): + for n in nodes.values(): ref = n["ref"] if ref == consumer_ref: consumer_info = n @@ -586,7 +586,7 @@ def package_info(self): hello_pkg_ref = 'hello/0.1#18d5440ae45afc4c36139a160ac071c7' pkg_pkg_ref = 'pkg/0.2#926714b5fb0a994f47ec37e071eba1da' hello_cpp_info = pkg_cpp_info = None - for _, n in nodes.items(): + for n in nodes.values(): ref = n["ref"] if ref == hello_pkg_ref: assert n['binary'] == "Build" diff --git a/conans/test/integration/command/export_pkg_test.py b/conans/test/integration/command/export_pkg_test.py index 52beb043c6c..7e8d1d0d717 100644 --- a/conans/test/integration/command/export_pkg_test.py +++ b/conans/test/integration/command/export_pkg_test.py @@ -403,7 +403,7 @@ def package_info(self): hello_pkg_ref = 'hello/0.1#18d5440ae45afc4c36139a160ac071c7' pkg_pkg_ref = 'pkg/0.2#926714b5fb0a994f47ec37e071eba1da' hello_cpp_info = pkg_cpp_info = None - for _, n in nodes.items(): + for n in nodes.values(): ref = n["ref"] if ref == hello_pkg_ref: assert n['binary'] is None # The exported package has no binary status diff --git a/conans/test/integration/command/info/info_test.py b/conans/test/integration/command/info/info_test.py index c33e404d0cc..baa0212c7f5 100644 --- a/conans/test/integration/command/info/info_test.py +++ b/conans/test/integration/command/info/info_test.py @@ -70,7 +70,7 @@ class MyTest(ConanFile): """) client.save({"conanfile.py": conanfile}) client.run("graph info . --format=json") - recipe = json.loads(client.stdout)["nodes"]["0"] + recipe = json.loads(client.stdout)["graph"]["nodes"]["0"] assert type(recipe["topics"]) == list assert recipe["topics"] == ["foo"] assert type(recipe["provides"]) == list @@ -90,7 +90,7 @@ class MyTest(ConanFile): """) client2.save({"conanfile.py": conanfile2}) client2.run("graph info . --format=json") - recipe = json.loads(client2.stdout)["nodes"]["0"] + recipe = json.loads(client2.stdout)["graph"]["nodes"]["0"] assert type(recipe["topics"]) == list assert recipe["topics"] == ["foo"] assert type(recipe["provides"]) == list @@ -166,7 +166,7 @@ def test_json_package_filter(self): assert '"nodes": {}' in client.out client.run("graph info . --package-filter=pkg* --format=json") graph = json.loads(client.stdout) - assert graph["nodes"]["0"]["ref"] == "pkg/0.1" + assert graph["graph"]["nodes"]["0"]["ref"] == "pkg/0.1" def test_json_info_outputs(self): client = TestClient() @@ -174,7 +174,7 @@ def test_json_info_outputs(self): client.save({"conanfile.py": conanfile}) client.run("graph info . -s build_type=Debug --format=json") graph = json.loads(client.stdout) - assert graph["nodes"]["0"]["settings"]["build_type"] == "Debug" + assert graph["graph"]["nodes"]["0"]["settings"]["build_type"] == "Debug" class TestAdvancedCliOutput: @@ -199,7 +199,7 @@ class pkg(ConanFile): client.run("graph info . --format=json") info = json.loads(client.stdout) - assert info["nodes"]["0"]["python_requires"] == ['tool/0.1#4d670581ccb765839f2239cc8dff8fbd'] + assert info["graph"]["nodes"]["0"]["python_requires"] == ['tool/0.1#4d670581ccb765839f2239cc8dff8fbd'] def test_build_id_info(self): client = TestClient() diff --git a/conans/test/integration/command/info/test_info_folders.py b/conans/test/integration/command/info/test_info_folders.py index e8eda3928cd..8d27599afa0 100644 --- a/conans/test/integration/command/info/test_info_folders.py +++ b/conans/test/integration/command/info/test_info_folders.py @@ -47,7 +47,7 @@ def test_basic(): client.save({CONANFILE: conanfile_py}) client.run(f"export . --user=user --channel=testing") client.run(f"graph info --requires=package/0.1.0@user/testing --format=json") - nodes = json.loads(client.stdout)["nodes"] + nodes = json.loads(client.stdout)["graph"]["nodes"] assert client.cache_folder in nodes["1"]["recipe_folder"] assert os.path.basename(nodes["1"]["recipe_folder"]).strip() == EXPORT_FOLDER assert nodes["1"]["source_folder"] is None @@ -84,9 +84,9 @@ def test_deps_basic(client_deps): client_deps.run(f"graph info {ref} --format=json") nodes = json.loads(client_deps.stdout) found_ref = False - assert len(nodes["nodes"]) == 3 + assert len(nodes["graph"]["nodes"]) == 3 - for _, node in nodes["nodes"].items(): + for _, node in nodes["graph"]["nodes"].items(): if node["ref"] == "conanfile": assert node["source_folder"] is None else: @@ -101,7 +101,7 @@ def test_deps_basic(client_deps): def test_deps_specific_information(client_deps): client_deps.run("graph info . --package-filter package/* --format=json") - nodes = json.loads(client_deps.stdout)["nodes"] + nodes = json.loads(client_deps.stdout)["graph"]["nodes"] assert len(nodes) == 1 assert "package/0.1.0@user/testing" in nodes["2"]["ref"] assert nodes["2"]["source_folder"] is None @@ -109,7 +109,7 @@ def test_deps_specific_information(client_deps): assert nodes["2"]["package_folder"] is None client_deps.run("graph info . --package-filter package* --format=json") - nodes = json.loads(client_deps.stdout)["nodes"] + nodes = json.loads(client_deps.stdout)["graph"]["nodes"] assert len(nodes) == 2 assert "package2/0.2.0@user/testing" in nodes["1"]["ref"] assert nodes["1"]["source_folder"] is None @@ -126,7 +126,7 @@ def test_single_field(): client.save({CONANFILE: conanfile_py}) client.run(f"export . --user=user --channel=testing") client.run(f"graph info --requires package/0.1.0@user/testing --format=json") - nodes = json.loads(client.stdout)["nodes"] + nodes = json.loads(client.stdout)["graph"]["nodes"] assert len(nodes) == 2 assert "package/0.1.0@user/testing" in nodes["1"]["ref"] assert nodes["1"]["source_folder"] is None diff --git a/conans/test/integration/graph/test_validate_build.py b/conans/test/integration/graph/test_validate_build.py index 5e263e1d482..926362eac1f 100644 --- a/conans/test/integration/graph/test_validate_build.py +++ b/conans/test/integration/graph/test_validate_build.py @@ -45,11 +45,11 @@ def package_id(self): # What happens with a conan info? t.run(f"graph info --requires=foo/1.0 {settings_gcc} --format=json", redirect_stdout="myjson") - myjson = json.loads(t.load("myjson"))["nodes"] + myjson = json.loads(t.load("myjson"))["graph"]["nodes"] assert myjson["1"]["invalid_build"] == "This doesn't build in GCC" t.run(f"graph info --requires=foo/1.0 {settings_clang} --format=json", redirect_stdout="myjson") - myjson = json.loads(t.load("myjson"))["nodes"] + myjson = json.loads(t.load("myjson"))["graph"]["nodes"] assert myjson["1"]["invalid_build"] is False diff --git a/conans/test/integration/graph/version_ranges/version_range_override_test.py b/conans/test/integration/graph/version_ranges/version_range_override_test.py index e13cd51b158..134a086d1ba 100644 --- a/conans/test/integration/graph/version_ranges/version_range_override_test.py +++ b/conans/test/integration/graph/version_ranges/version_range_override_test.py @@ -139,9 +139,9 @@ def test_override(self): "ros_core/pr-53@3rdparty/snapshot#4d670581ccb765839f2239cc8dff8fbd" ] } - self.assertEqual(info["overrides"], expected_overrides) + self.assertEqual(info['graph']["overrides"], expected_overrides) expected_resolved_ranges = { "pkgb/[~0]@common/unstable": "pkgb/0.1@common/unstable", "ros_perception/[~1.1]@3rdparty/unstable": "ros_perception/1.1.4@3rdparty/unstable" } - self.assertEqual(info["resolved_ranges"], expected_resolved_ranges) + self.assertEqual(info['graph']["resolved_ranges"], expected_resolved_ranges) diff --git a/conans/test/integration/lockfile/test_graph_overrides.py b/conans/test/integration/lockfile/test_graph_overrides.py index cf2b6a9dcae..bae9f6817a4 100644 --- a/conans/test/integration/lockfile/test_graph_overrides.py +++ b/conans/test/integration/lockfile/test_graph_overrides.py @@ -29,12 +29,12 @@ def test_overrides_half_diamond(override, force): assert "pkga/0.2" in requires assert "pkga/0.1" not in requires c.run("graph info pkgc --lockfile=pkgc/conan.lock --format=json") - dependencies = json.loads(c.stdout)["nodes"]["0"]["dependencies"] + dependencies = json.loads(c.stdout)["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.2" in str(dependencies) assert "pkga/0.1" not in str(dependencies) # apply the lockfile to pkgb, should it lock to pkga/0.2 c.run("graph info pkgb --lockfile=pkgc/conan.lock --format=json") - dependencies = json.loads(c.stdout)["nodes"]["0"]["dependencies"] + dependencies = json.loads(c.stdout)["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.2" in str(dependencies) assert "pkga/0.1" not in str(dependencies) @@ -126,12 +126,12 @@ def test_overrides_diamond(override, force): assert "pkga/0.1" not in requires c.run("graph info pkgd --lockfile=pkgd/conan.lock --format=json") json_graph = json.loads(c.stdout) - deps = json_graph["nodes"]["0"]["dependencies"] + deps = json_graph["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.3" in str(deps) assert "pkga/0.2" not in str(deps) assert "pkga/0.1" not in str(deps) # Redundant assert, but checking "overrides" summary - overrides = json_graph["overrides"] + overrides = json_graph['graph']["overrides"] assert len(overrides) == 2 assert overrides['pkga/0.1'] == ['pkga/0.3'] assert overrides['pkga/0.2'] == ['pkga/0.3#e3afb975277bc245212d6e8de88f4f8f'] @@ -139,12 +139,12 @@ def test_overrides_diamond(override, force): # apply the lockfile to pkgb, should it lock to pkga/0.3 c.run("graph info pkgb --lockfile=pkgd/conan.lock --format=json") json_graph = json.loads(c.stdout) - deps = json_graph["nodes"]["0"]["dependencies"] + deps = json_graph["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.3" in str(deps) assert "pkga/0.2" not in str(deps) assert "pkga/0.1" not in str(deps) # Redundant assert, but checking "overrides" summary - overrides = json_graph["overrides"] + overrides = json_graph['graph']["overrides"] assert len(overrides) == 1 assert overrides["pkga/0.1"] == ["pkga/0.3"] @@ -178,13 +178,13 @@ def test_overrides_diamond_ranges(override, force): assert "pkga/0.2" not in requires assert "pkga/0.1" not in requires c.run("graph info pkgd --lockfile=pkgd/conan.lock --format=json") - dependencies = json.loads(c.stdout)["nodes"]["0"]["dependencies"] + dependencies = json.loads(c.stdout)["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.3" in str(dependencies) assert "pkga/0.2" not in str(dependencies) assert "pkga/0.1" not in str(dependencies) # apply the lockfile to pkgb, should it lock to pkga/0.3 c.run("graph info pkgb --lockfile=pkgd/conan.lock --format=json") - dependencies = json.loads(c.stdout)["nodes"]["0"]["dependencies"] + dependencies = json.loads(c.stdout)["graph"]["nodes"]["0"]["dependencies"] assert "pkga/0.3" in str(dependencies) assert "pkga/0.2" not in str(dependencies) assert "pkga/0.1" not in str(dependencies) diff --git a/conans/test/integration/package_id/test_validate.py b/conans/test/integration/package_id/test_validate.py index 13d3ca3a6a2..e2d43a1c551 100644 --- a/conans/test/integration/package_id/test_validate.py +++ b/conans/test/integration/package_id/test_validate.py @@ -44,8 +44,8 @@ def validate(self): client.run("graph info --require pkg/0.1 -s os=Windows --format json") myjson = json.loads(client.stdout) - self.assertEqual(myjson["nodes"]["1"]["binary"], BINARY_INVALID) - assert myjson["nodes"]["1"]["info_invalid"] == "Windows not supported" in client.out + self.assertEqual(myjson["graph"]["nodes"]["1"]["binary"], BINARY_INVALID) + assert myjson["graph"]["nodes"]["1"]["info_invalid"] == "Windows not supported" in client.out def test_validate_header_only(self): client = TestClient() From 299169f791ff33b5e5e0877606f7a252c7fc2aa6 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 May 2023 13:02:07 +0200 Subject: [PATCH 12/18] wip --- conan/api/model.py | 19 +++++++++--- conan/cli/commands/list.py | 5 ++-- conans/client/graph/graph.py | 1 - .../command_v2/test_list_and_upload.py | 30 +++++++++++++++---- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index a0335671f3e..aa491750b52 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -69,19 +69,30 @@ def load(file): return result @staticmethod - def from_graph(graph, graph_binaries=None): + def from_graph(graph, graph_recipes=None, graph_binaries=None): pkglist = MultiPackagesList() cache_list = PackagesList() - graph_binaries = ["*"] if graph_binaries is None else graph_binaries + if graph_recipes is None and graph_binaries is None: + recipes = ["*"] + binaries = ["*"] + else: + recipes = [r.lower() for r in graph_recipes or []] + binaries = [b.lower() for b in graph_binaries or []] + pkglist.lists["Local Cache"] = cache_list for node in graph["graph"]["nodes"].values(): ref = node["ref"] if ref == "conanfile": continue ref = RecipeReference.loads(ref) - binary = node["binary"] - if any(fnmatch.fnmatch(binary, b) for b in graph_binaries): + recipe = node["recipe"].lower() + if any(r == recipe or r == "*" for r in recipes): cache_list.add_refs([ref]) + + binary = node["binary"].lower() + if any(b == binary or b == "*" for b in binaries): + # TODO: Improve add_prefs so it automatically add_ref + cache_list.add_refs([ref]) # Binary listed forces recipe listed pref = PkgReference(ref, node["package_id"], node["prev"]) cache_list.add_prefs(ref, [pref]) return pkglist diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index d96118bf296..4930f150c50 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -1,7 +1,7 @@ import json from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern, PackagesList, MultiPackagesList +from conan.api.model import ListPattern, MultiPackagesList from conan.api.output import Color, cli_out_write from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument @@ -108,6 +108,7 @@ def list(conan_api: ConanAPI, parser, *args): parser.add_argument("-c", "--cache", action='store_true', help="Search in the local cache") parser.add_argument("-g", "--graph", help="Graph json file") parser.add_argument("-gb", "--graph-binaries", help="Which binaries are listed", action="append") + parser.add_argument("-gr", "--graph-recipes", help="Which recipes are listed", action="append") args = parser.parse_args(*args) @@ -119,7 +120,7 @@ def list(conan_api: ConanAPI, parser, *args): if args.graph: graphfile = make_abs_path(args.graph) graph = json.loads(load(graphfile)) - pkglist = MultiPackagesList.from_graph(graph, args.graph_binaries) + pkglist = MultiPackagesList.from_graph(graph, args.graph_recipes, args.graph_binaries) return { "results": pkglist.serialize(), "conan_api": conan_api, diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index e851093b3e7..a17ff0c4254 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -14,7 +14,6 @@ RECIPE_EDITABLE = "Editable" RECIPE_CONSUMER = "Consumer" # A conanfile from the user RECIPE_VIRTUAL = "Cli" # A virtual conanfile (dynamic in memory conanfile) -RECIPE_MISSING = "Missing recipe" # Impossible to find a recipe for this reference RECIPE_SYSTEM_TOOL = "System tool" BINARY_CACHE = "Cache" diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_list_and_upload.py index 60d41a27d30..1751f6299b9 100644 --- a/conans/test/integration/command_v2/test_list_and_upload.py +++ b/conans/test/integration/command_v2/test_list_and_upload.py @@ -1,3 +1,5 @@ +import json + import pytest from conans.test.assets.genconanfile import GenConanfile @@ -49,17 +51,33 @@ def test_create_upload(self): assert "Uploading recipe 'app/1.0#0fa1ff1b90576bb782600e56df642e19'" in c.out assert "Uploading recipe 'zlib/1.0#c570d63921c5f2070567da4bf64ff261'" in c.out assert "Uploading package 'app" in c.out + assert "Uploading package 'zlib" in c.out class TestGraphPkgList: - def test_graph_pkg_list(self): + def test_graph_pkg_list_only_built(self): + c = TestClient() + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) + c.run("create zlib") + c.run("create app --format=json", redirect_stdout="graph.json") + c.run("list --graph=graph.json --graph-binaries=build --format=json") + pkglist = json.loads(c.stdout)["Local Cache"] + assert len(pkglist) == 1 + assert len(pkglist["app/1.0"]["revisions"] + ["0fa1ff1b90576bb782600e56df642e19"]["packages"]) == 1 + + def test_graph_pkg_list_all_recipes_only(self): + """ + --graph-recipes=* selects all the recipes in the graph + """ c = TestClient() c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) c.run("create zlib") c.run("create app --format=json", redirect_stdout="graph.json") - c.run("list --graph=graph.json --graph-binaries=build --format=json", - redirect_stdout="pkglist.json") - pkglist = c.load("pkglist.json") - assert "app/1.0" in pkglist - assert "zlib" not in pkglist + c.run("list --graph=graph.json --graph-recipes=* --format=json") + pkglist = json.loads(c.stdout)["Local Cache"] + assert len(pkglist) == 2 + assert len(pkglist["app/1.0"]["revisions"]["0fa1ff1b90576bb782600e56df642e19"]) == 0 + assert len(pkglist["zlib/1.0"]["revisions"]["c570d63921c5f2070567da4bf64ff261"]) == 0 From 0beed4237c8214fbd8f73c819f5d92ead3e490da Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 May 2023 16:09:57 +0200 Subject: [PATCH 13/18] wip --- conan/cli/commands/list.py | 42 +++++++++---------- .../test/integration/command_v2/list_test.py | 9 ++++ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 4930f150c50..df4a9c92766 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -116,37 +116,33 @@ def list(conan_api: ConanAPI, parser, *args): raise ConanException("Missing pattern or graph json file") if args.reference and args.graph: raise ConanException("Cannot define both the pattern and the graph json file") + if (args.graph_recipes or args.graph_binaries) and not args.graph: + raise ConanException("--graph-recipes and --graph-binaries require a --graph input") if args.graph: graphfile = make_abs_path(args.graph) graph = json.loads(load(graphfile)) pkglist = MultiPackagesList.from_graph(graph, args.graph_recipes, args.graph_binaries) - return { - "results": pkglist.serialize(), - "conan_api": conan_api, - "cli_args": " ".join( - [f"{arg}={getattr(args, arg)}" for arg in vars(args) if getattr(args, arg)]) - } - - ref_pattern = ListPattern(args.reference, rrev=None, prev=None) - # If neither remote nor cache are defined, show results only from cache - pkglist = MultiPackagesList() - if args.cache or not args.remote: - try: - cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None) - except Exception as e: - pkglist.add_error("Local Cache", str(e)) - else: - pkglist.add("Local Cache", cache_list) - if args.remote: - remotes = conan_api.remotes.list(args.remote) - for remote in remotes: + else: + ref_pattern = ListPattern(args.reference, rrev=None, prev=None) + # If neither remote nor cache are defined, show results only from cache + pkglist = MultiPackagesList() + if args.cache or not args.remote: try: - remote_list = conan_api.list.select(ref_pattern, args.package_query, remote) + cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None) except Exception as e: - pkglist.add_error(remote.name, str(e)) + pkglist.add_error("Local Cache", str(e)) else: - pkglist.add(remote.name, remote_list) + pkglist.add("Local Cache", cache_list) + if args.remote: + remotes = conan_api.remotes.list(args.remote) + for remote in remotes: + try: + remote_list = conan_api.list.select(ref_pattern, args.package_query, remote) + except Exception as e: + pkglist.add_error(remote.name, str(e)) + else: + pkglist.add(remote.name, remote_list) return { "results": pkglist.serialize(), diff --git a/conans/test/integration/command_v2/list_test.py b/conans/test/integration/command_v2/list_test.py index 84ff71b3d12..5a03ffcbc78 100644 --- a/conans/test/integration/command_v2/list_test.py +++ b/conans/test/integration/command_v2/list_test.py @@ -31,6 +31,15 @@ def test_query_param_is_required(self): c.run("list --remote remote1 --cache", assert_error=True) assert "ERROR: Missing pattern or graph json file" in c.out + c.run("list * --graph=myjson", assert_error=True) + assert "ERROR: Cannot define both the pattern and the graph json file" in c.out + + c.run("list * --graph-binaries=x", assert_error=True) + assert "ERROR: --graph-recipes and --graph-binaries require a --graph input" in c.out + + c.run("list * --graph-recipes=x", assert_error=True) + assert "ERROR: --graph-recipes and --graph-binaries require a --graph input" in c.out + @pytest.fixture(scope="module") def client(): From 1556da421c67c5ea278a56c640d16363acb32b6e Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 May 2023 16:20:44 +0200 Subject: [PATCH 14/18] fixes --- conan/cli/commands/download.py | 9 +++++---- conan/cli/commands/list.py | 14 +++++++------- conan/cli/commands/remove.py | 8 +++++--- conan/cli/commands/upload.py | 18 +++++++++--------- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index 016a6e4415a..3bdacaf87c4 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -17,9 +17,10 @@ def download(conan_api: ConanAPI, parser, *args): queries over the package binaries. """ - parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " - "wildcard at any reference field. If revision is not " - "specified, it is assumed latest one.") + parser.add_argument('pattern', + help="A pattern in the form 'pkg/version#revision:package_id#revision', " + "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. " + "If revision is not specified, it is assumed latest one.") parser.add_argument("--only-recipe", action='store_true', default=False, help='Download only the recipe/s, not the binary packages.') parser.add_argument('-p', '--package-query', default=None, action=OnceArgument, @@ -31,7 +32,7 @@ def download(conan_api: ConanAPI, parser, *args): args = parser.parse_args(*args) remote = conan_api.remotes.get(args.remote) parallel = conan_api.config.get("core.download:parallel", default=1, check_type=int) - ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe) + ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) select_bundle = conan_api.list.select(ref_pattern, args.package_query, remote) refs = [] prefs = [] diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index df4a9c92766..50cd0e66d98 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -96,10 +96,10 @@ def list(conan_api: ConanAPI, parser, *args): """ List existing recipes, revisions, or packages in the cache (by default) or the remotes. """ - parser.add_argument('reference', help="Recipe reference or package reference. " - "Both can contain * as wildcard at any reference field. " - "If revision is not specified, it is assumed latest one.", - nargs="?") + parser.add_argument('pattern', nargs="?", + help="A pattern in the form 'pkg/version#revision:package_id#revision', " + "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. " + "If revision is not specified, it is assumed latest one.") parser.add_argument('-p', '--package-query', default=None, action=OnceArgument, help="List only the packages matching a specific query, e.g, os=Windows AND " "(arch=x86 OR compiler=gcc)") @@ -112,9 +112,9 @@ def list(conan_api: ConanAPI, parser, *args): args = parser.parse_args(*args) - if args.reference is None and args.graph is None: + if args.pattern is None and args.graph is None: raise ConanException("Missing pattern or graph json file") - if args.reference and args.graph: + if args.pattern and args.graph: raise ConanException("Cannot define both the pattern and the graph json file") if (args.graph_recipes or args.graph_binaries) and not args.graph: raise ConanException("--graph-recipes and --graph-binaries require a --graph input") @@ -124,7 +124,7 @@ def list(conan_api: ConanAPI, parser, *args): graph = json.loads(load(graphfile)) pkglist = MultiPackagesList.from_graph(graph, args.graph_recipes, args.graph_binaries) else: - ref_pattern = ListPattern(args.reference, rrev=None, prev=None) + ref_pattern = ListPattern(args.pattern, rrev=None, prev=None) # If neither remote nor cache are defined, show results only from cache pkglist = MultiPackagesList() if args.cache or not args.remote: diff --git a/conan/cli/commands/remove.py b/conan/cli/commands/remove.py index 0fb4289eb4a..b4fe4fb4f29 100644 --- a/conan/cli/commands/remove.py +++ b/conan/cli/commands/remove.py @@ -16,8 +16,10 @@ def remove(conan_api: ConanAPI, parser, *args): will be removed. - If a package reference is specified, it will remove only the package. """ - parser.add_argument('reference', help="Recipe reference or package reference, can contain * as" - "wildcard at any reference field. e.g: lib/*") + parser.add_argument('pattern', + help="A pattern in the form 'pkg/version#revision:package_id#revision', " + "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. " + "If revision is not specified, it is assumed latest one.") parser.add_argument('-c', '--confirm', default=False, action='store_true', help='Remove without requesting a confirmation') parser.add_argument('-p', '--package-query', action=OnceArgument, @@ -33,7 +35,7 @@ def remove(conan_api: ConanAPI, parser, *args): def confirmation(message): return args.confirm or ui.request_boolean(message) - ref_pattern = ListPattern(args.reference, rrev="*", prev="*") + ref_pattern = ListPattern(args.pattern, rrev="*", prev="*") select_bundle = conan_api.list.select(ref_pattern, args.package_query, remote) if ref_pattern.package_id is None: diff --git a/conan/cli/commands/upload.py b/conan/cli/commands/upload.py index 39d748365de..4a89e49c8f9 100644 --- a/conan/cli/commands/upload.py +++ b/conan/cli/commands/upload.py @@ -16,10 +16,10 @@ def upload(conan_api: ConanAPI, parser, *args): binary packages, unless --only-recipe is specified. You can use the "latest" placeholder at the "reference" argument to specify the latest revision of the recipe or the package. """ - parser.add_argument('reference', help="Recipe reference or package reference, can contain * as " - "wildcard at any reference field. If no revision is " - "specified, it is assumed to be the latest", - nargs="?") + parser.add_argument('pattern', nargs="?", + help="A pattern in the form 'pkg/version#revision:package_id#revision', " + "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. " + "If revision is not specified, it is assumed latest one.") parser.add_argument('-p', '--package-query', default=None, action=OnceArgument, help="Only upload packages matching a specific query. e.g: os=Windows AND " "(arch=x86 OR compiler=gcc)") @@ -42,20 +42,20 @@ def upload(conan_api: ConanAPI, parser, *args): remote = conan_api.remotes.get(args.remote) enabled_remotes = conan_api.remotes.list() - if args.reference is None and args.list is None: + if args.pattern is None and args.list is None: raise ConanException("Missing pattern or package list file") - if args.reference and args.list: + if args.pattern and args.list: raise ConanException("Cannot define both the pattern and the package list file") if args.list: listfile = make_abs_path(args.list) multi_package_list = MultiPackagesList.load(listfile) package_list = multi_package_list["Local Cache"] else: - ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe) + ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, package_query=args.package_query) if not package_list.recipes: - raise ConanException("No recipes found matching pattern '{}'".format(args.reference)) + raise ConanException("No recipes found matching pattern '{}'".format(args.pattern)) if args.check: conan_api.cache.check_integrity(package_list) @@ -63,7 +63,7 @@ def upload(conan_api: ConanAPI, parser, *args): conan_api.upload.check_upstream(package_list, remote, args.force) # If only if search with "*" we ask for confirmation - if not args.list and not args.confirm and "*" in args.reference: + if not args.list and not args.confirm and "*" in args.pattern: _ask_confirm_upload(conan_api, package_list) conan_api.upload.prepare(package_list, enabled_remotes) From 5261c8d6b2aa2482e46e7c74a4cfd65fdb6d2b87 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 May 2023 16:28:20 +0200 Subject: [PATCH 15/18] review --- conan/api/model.py | 3 ++- conan/cli/commands/list.py | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index aa491750b52..7f5f34498f2 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -69,7 +69,8 @@ def load(file): return result @staticmethod - def from_graph(graph, graph_recipes=None, graph_binaries=None): + def load_graph(graphfile, graph_recipes=None, graph_binaries=None): + graph = json.loads(load(graphfile)) pkglist = MultiPackagesList() cache_list = PackagesList() if graph_recipes is None and graph_binaries is None: diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 50cd0e66d98..009b80a765a 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -8,7 +8,6 @@ from conan.cli.formatters.list import list_packages_html from conans.errors import ConanException from conans.util.dates import timestamp_to_str -from conans.util.files import load # Keep them so we don't break other commands that import them, but TODO: Remove later @@ -121,8 +120,7 @@ def list(conan_api: ConanAPI, parser, *args): if args.graph: graphfile = make_abs_path(args.graph) - graph = json.loads(load(graphfile)) - pkglist = MultiPackagesList.from_graph(graph, args.graph_recipes, args.graph_binaries) + pkglist = MultiPackagesList.load_graph(graphfile, args.graph_recipes, args.graph_binaries) else: ref_pattern = ListPattern(args.pattern, rrev=None, prev=None) # If neither remote nor cache are defined, show results only from cache From 659ab7c3d816b4013f2dc394720ee64fc6720e79 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 30 May 2023 19:01:47 +0200 Subject: [PATCH 16/18] added download of pkg-list --- conan/cli/commands/download.py | 37 +++++++-- ...load.py => test_combined_pkglist_flows.py} | 80 ++++++++++++++++--- 2 files changed, 98 insertions(+), 19 deletions(-) rename conans/test/integration/command_v2/{test_list_and_upload.py => test_combined_pkglist_flows.py} (51%) diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index 3bdacaf87c4..a80b213463d 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -1,12 +1,16 @@ from multiprocessing.pool import ThreadPool from conan.api.conan_api import ConanAPI -from conan.api.model import ListPattern +from conan.api.model import ListPattern, MultiPackagesList from conan.api.output import ConanOutput +from conan.cli import make_abs_path from conan.cli.command import conan_command, OnceArgument +from conan.cli.commands.list import print_list_text, print_list_json +from conans.errors import ConanException -@conan_command(group="Creator") +@conan_command(group="Creator", formatters={"text": print_list_text, + "json": print_list_json}) def download(conan_api: ConanAPI, parser, *args): """ Download (without installing) a single conan package from a remote server. @@ -17,7 +21,7 @@ def download(conan_api: ConanAPI, parser, *args): queries over the package binaries. """ - parser.add_argument('pattern', + parser.add_argument('pattern', nargs="?", help="A pattern in the form 'pkg/version#revision:package_id#revision', " "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. " "If revision is not specified, it is assumed latest one.") @@ -28,17 +32,34 @@ def download(conan_api: ConanAPI, parser, *args): "(arch=x86 OR compiler=gcc)") parser.add_argument("-r", "--remote", action=OnceArgument, required=True, help='Download from this specific remote') + parser.add_argument("-l", "--list", help="Package list file") args = parser.parse_args(*args) + if args.pattern is None and args.list is None: + raise ConanException("Missing pattern or package list file") + if args.pattern and args.list: + raise ConanException("Cannot define both the pattern and the package list file") + remote = conan_api.remotes.get(args.remote) + + if args.list: + listfile = make_abs_path(args.list) + multi_package_list = MultiPackagesList.load(listfile) + try: + package_list = multi_package_list[remote.name] + except KeyError: + raise ConanException(f"The current package list do not contain remote '{remote.name}'") + else: + ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) + package_list = conan_api.list.select(ref_pattern, args.package_query, remote) + parallel = conan_api.config.get("core.download:parallel", default=1, check_type=int) - ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) - select_bundle = conan_api.list.select(ref_pattern, args.package_query, remote) + refs = [] prefs = [] - for ref, recipe_bundle in select_bundle.refs(): + for ref, recipe_bundle in package_list.refs(): refs.append(ref) - for pref, _ in select_bundle.prefs(ref, recipe_bundle): + for pref, _ in package_list.prefs(ref, recipe_bundle): prefs.append(pref) if parallel <= 1: @@ -49,6 +70,8 @@ def download(conan_api: ConanAPI, parser, *args): else: _download_parallel(parallel, conan_api, refs, prefs, remote) + return {"results": {"Local Cache": package_list.serialize()}} + def _download_parallel(parallel, conan_api, refs, prefs, remote): diff --git a/conans/test/integration/command_v2/test_list_and_upload.py b/conans/test/integration/command_v2/test_combined_pkglist_flows.py similarity index 51% rename from conans/test/integration/command_v2/test_list_and_upload.py rename to conans/test/integration/command_v2/test_combined_pkglist_flows.py index 1751f6299b9..050a631cb14 100644 --- a/conans/test/integration/command_v2/test_list_and_upload.py +++ b/conans/test/integration/command_v2/test_combined_pkglist_flows.py @@ -6,22 +6,21 @@ from conans.test.utils.tools import TestClient -@pytest.fixture() -def client(): - c = TestClient(default_server_user=True) - c.save({ - "zlib.py": GenConanfile("zlib"), - "zli.py": GenConanfile("zli", "1.0.0") - }) - c.run("create zli.py") - c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") - return c - - class TestListUpload: refs = ["zli/1.0.0#f034dc90894493961d92dd32a9ee3b78", "zlib/1.0.0@user/channel#ffd4bc45820ddb320ab224685b9ba3fb"] + @pytest.fixture() + def client(self): + c = TestClient(default_server_user=True) + c.save({ + "zlib.py": GenConanfile("zlib"), + "zli.py": GenConanfile("zli", "1.0.0") + }) + c.run("create zli.py") + c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") + return c + def test_list_upload_recipes(self, client): pattern = "z*#latest" client.run(f"list {pattern} --format=json", redirect_stdout="pkglist.json") @@ -81,3 +80,60 @@ def test_graph_pkg_list_all_recipes_only(self): assert len(pkglist) == 2 assert len(pkglist["app/1.0"]["revisions"]["0fa1ff1b90576bb782600e56df642e19"]) == 0 assert len(pkglist["zlib/1.0"]["revisions"]["c570d63921c5f2070567da4bf64ff261"]) == 0 + + +class TestDownloadUpload: + @pytest.fixture() + def client(self): + c = TestClient(default_server_user=True) + c.save({ + "zlib.py": GenConanfile("zlib"), + "zli.py": GenConanfile("zli", "1.0.0") + }) + c.run("create zli.py") + c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") + c.run("upload * -r=default -c") + c.run("remove * -c") + return c + + @pytest.mark.parametrize("prev_list", [False, True]) + def test_download_upload_all(self, client, prev_list): + # We need to be consequeent with the pattern, it is not the same defaults for + # download and for list + pattern = "zlib/*#latest:*#latest" + if prev_list: + client.run(f"list {pattern} -r=default --format=json", redirect_stdout="pkglist.json") + # Overwriting previous pkglist.json + pattern = "--list=pkglist.json" + + client.run(f"download {pattern} -r=default --format=json", redirect_stdout="pkglist.json") + # TODO: Discuss "origin" + assert "Local Cache" in client.load("pkglist.json") + client.run("remove * -r=default -c") + client.run("upload --list=pkglist.json -r=default") + assert f"Uploading recipe 'zlib/1.0.0" in client.out + assert f"Uploading recipe 'zli/" not in client.out + assert "Uploading package 'zlib/1.0.0" in client.out + assert "Uploading package 'zli/" not in client.out + + @pytest.mark.parametrize("prev_list", [False, True]) + def test_download_upload_only_recipes(self, client, prev_list): + if prev_list: + pattern = "zlib/*#latest" + client.run(f"list {pattern} -r=default --format=json", redirect_stdout="pkglist.json") + # Overwriting previous pkglist.json + pattern = "--list=pkglist.json" + else: + pattern = "zlib/*#latest --only-recipe" + client.run(f"download {pattern} -r=default --format=json", redirect_stdout="pkglist.json") + # TODO: Discuss "origin" + assert "Local Cache" in client.load("pkglist.json") + # Download binary too! Just to make sure it is in the cache, but not uploaded + # because it is not in the orignal list of only recipes + client.run(f"download * -r=default") + client.run("remove * -r=default -c") + client.run("upload --list=pkglist.json -r=default") + assert f"Uploading recipe 'zlib/1.0.0" in client.out + assert f"Uploading recipe 'zli/" not in client.out + assert "Uploading package 'zlib/1.0.0" not in client.out + assert "Uploading package 'zli/" not in client.out From dcc3a5411c8b6dfb3a98552ccdedd18e155cc30d Mon Sep 17 00:00:00 2001 From: James Date: Wed, 31 May 2023 20:38:06 +0200 Subject: [PATCH 17/18] Update conan/cli/commands/download.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Francisco Ramírez --- conan/cli/commands/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/cli/commands/download.py b/conan/cli/commands/download.py index a80b213463d..8fa7cd1e80f 100644 --- a/conan/cli/commands/download.py +++ b/conan/cli/commands/download.py @@ -48,7 +48,7 @@ def download(conan_api: ConanAPI, parser, *args): try: package_list = multi_package_list[remote.name] except KeyError: - raise ConanException(f"The current package list do not contain remote '{remote.name}'") + raise ConanException(f"The current package list does not contain remote '{remote.name}'") else: ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe) package_list = conan_api.list.select(ref_pattern, args.package_query, remote) From 9b39a4aa3dfad5b6ee357acf57a6dd4591b0c197 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 1 Jun 2023 01:07:06 +0200 Subject: [PATCH 18/18] review --- conan/api/model.py | 33 ++++++++++++++----- conans/client/graph/graph.py | 2 ++ .../command_v2/test_combined_pkglist_flows.py | 27 ++++++++++++++- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 7f5f34498f2..6a6f7bb6f03 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -1,6 +1,8 @@ import fnmatch import json +from conans.client.graph.graph import RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_SYSTEM_TOOL, \ + RECIPE_VIRTUAL, BINARY_SKIP, BINARY_MISSING, BINARY_INVALID from conans.errors import ConanException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference @@ -82,19 +84,34 @@ def load_graph(graphfile, graph_recipes=None, graph_binaries=None): pkglist.lists["Local Cache"] = cache_list for node in graph["graph"]["nodes"].values(): - ref = node["ref"] - if ref == "conanfile": + recipe = node["recipe"] + if recipe in (RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_VIRTUAL, RECIPE_SYSTEM_TOOL): continue + + ref = node["ref"] ref = RecipeReference.loads(ref) - recipe = node["recipe"].lower() - if any(r == recipe or r == "*" for r in recipes): + recipe = recipe.lower() + if any(r == "*" or r == recipe for r in recipes): cache_list.add_refs([ref]) - binary = node["binary"].lower() - if any(b == binary or b == "*" for b in binaries): - # TODO: Improve add_prefs so it automatically add_ref + remote = node["remote"] + if remote: + remote_list = pkglist.lists.setdefault(remote, PackagesList()) + remote_list.add_refs([ref]) + pref = PkgReference(ref, node["package_id"], node["prev"]) + binary_remote = node["binary_remote"] + if binary_remote: + remote_list = pkglist.lists.setdefault(binary_remote, PackagesList()) + remote_list.add_refs([ref]) # Binary listed forces recipe listed + remote_list.add_prefs(ref, [pref]) + + binary = node["binary"] + if binary in (BINARY_SKIP, BINARY_INVALID, BINARY_MISSING): + continue + + binary = binary.lower() + if any(b == "*" or b == binary for b in binaries): cache_list.add_refs([ref]) # Binary listed forces recipe listed - pref = PkgReference(ref, node["package_id"], node["prev"]) cache_list.add_prefs(ref, [pref]) return pkglist diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index a17ff0c4254..ff2aa3b0c12 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -213,6 +213,8 @@ def serialize(self): result["recipe"] = self.recipe result["package_id"] = self.package_id result["prev"] = self.prev + result["remote"] = self.remote.name if self.remote else None + result["binary_remote"] = self.binary_remote.name if self.binary_remote else None from conans.client.installer import build_id result["build_id"] = build_id(self.conanfile) result["binary"] = self.binary diff --git a/conans/test/integration/command_v2/test_combined_pkglist_flows.py b/conans/test/integration/command_v2/test_combined_pkglist_flows.py index 050a631cb14..a0cc613b45d 100644 --- a/conans/test/integration/command_v2/test_combined_pkglist_flows.py +++ b/conans/test/integration/command_v2/test_combined_pkglist_flows.py @@ -53,7 +53,7 @@ def test_create_upload(self): assert "Uploading package 'zlib" in c.out -class TestGraphPkgList: +class TestCreateGraphToPkgList: def test_graph_pkg_list_only_built(self): c = TestClient() c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), @@ -82,6 +82,31 @@ def test_graph_pkg_list_all_recipes_only(self): assert len(pkglist["zlib/1.0"]["revisions"]["c570d63921c5f2070567da4bf64ff261"]) == 0 +class TestGraphInfoToPkgList: + def test_graph_pkg_list_only_built(self): + c = TestClient(default_server_user=True) + c.save({"zlib/conanfile.py": GenConanfile("zlib", "1.0"), + "app/conanfile.py": GenConanfile("app", "1.0").with_requires("zlib/1.0")}) + c.run("create zlib") + c.run("create app --format=json") + c.run("upload * -c -r=default") + c.run("remove * -c") + c.run("graph info --requires=app/1.0 --format=json", redirect_stdout="graph.json") + c.run("list --graph=graph.json --graph-binaries=build --format=json") + pkglist = json.loads(c.stdout) + assert len(pkglist["Local Cache"]) == 0 + assert len(pkglist["default"]) == 2 + c.run("install --requires=app/1.0 --format=json", redirect_stdout="graph.json") + c.run("list --graph=graph.json --graph-binaries=download --format=json") + pkglist = json.loads(c.stdout) + assert len(pkglist["Local Cache"]) == 2 + assert len(pkglist["default"]) == 2 + c.run("list --graph=graph.json --graph-binaries=build --format=json") + pkglist = json.loads(c.stdout) + assert len(pkglist["Local Cache"]) == 0 + assert len(pkglist["default"]) == 2 + + class TestDownloadUpload: @pytest.fixture() def client(self):