Feature/pkg list (#13928)

* pkglist

* wip

* fix test

* changed PackageList approach to include multiple remotes

* fixes

* review

* wip

* Conan 2.0.6

* Fix format for id property in graph output json (#13964)

* convert to str

* franchus review

* test_requires shouldn't affect package_id (#13966)

* test_requires shouldn't affect package_id

* test with transitive

* remove prints

* [graph][json] Same json output for conan graph info, create, install and export-pkg (#13967)

* Ensuring the output for the rest of commands

* export-pkg too

* graph as first output level

* wip

* typo

* typos

* wip

* wip

* fixes

* review

* added download of pkg-list

* Update conan/cli/commands/download.py

Co-authored-by: Francisco Ramírez <franchuti688@gmail.com>

* review

---------

Co-authored-by: czoido <mrgalleta@gmail.com>
Co-authored-by: Francisco Ramírez <franchuti688@gmail.com>

conan/api/model.py

@@ -1,8 +1,12 @@
 import fnmatch
+import json
 
+from conans.client.graph.graph import RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_SYSTEM_TOOL, \
+    RECIPE_VIRTUAL, BINARY_SKIP, BINARY_MISSING, BINARY_INVALID
 from conans.errors import ConanException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
+from conans.util.files import load
 from conans.model.version_range import VersionRange
 
 
@@ -34,6 +38,85 @@ def __repr__(self):
         return str(self)
 
 
+class MultiPackagesList:
+    def __init__(self):
+        self.lists = {}
+
+    def __getitem__(self, name):
+        try:
+            return self.lists[name]
+        except KeyError:
+            raise ConanException(f"'{name}' doesn't exist is package list")
+
+    def add(self, name, pkg_list):
+        self.lists[name] = pkg_list
+
+    def add_error(self, remote_name, error):
+        self.lists[remote_name] = {"error": error}
+
+    def serialize(self):
+        return {k: v.serialize() if isinstance(v, PackagesList) else v
+                for k, v in self.lists.items()}
+
+    @staticmethod
+    def load(file):
+        content = json.loads(load(file))
+        result = {}
+        for remote, pkglist in content.items():
+            if "error" in pkglist:
+                result[remote] = pkglist
+            else:
+                result[remote] = PackagesList.deserialize(pkglist)
+        pkglist = MultiPackagesList()
+        pkglist.lists = result
+        return result
+
+    @staticmethod
+    def load_graph(graphfile, graph_recipes=None, graph_binaries=None):
+        graph = json.loads(load(graphfile))
+        pkglist = MultiPackagesList()
+        cache_list = PackagesList()
+        if graph_recipes is None and graph_binaries is None:
+            recipes = ["*"]
+            binaries = ["*"]
+        else:
+            recipes = [r.lower() for r in graph_recipes or []]
+            binaries = [b.lower() for b in graph_binaries or []]
+
+        pkglist.lists["Local Cache"] = cache_list
+        for node in graph["graph"]["nodes"].values():
+            recipe = node["recipe"]
+            if recipe in (RECIPE_EDITABLE, RECIPE_CONSUMER, RECIPE_VIRTUAL, RECIPE_SYSTEM_TOOL):
+                continue
+
+            ref = node["ref"]
+            ref = RecipeReference.loads(ref)
+            recipe = recipe.lower()
+            if any(r == "*" or r == recipe for r in recipes):
+                cache_list.add_refs([ref])
+
+            remote = node["remote"]
+            if remote:
+                remote_list = pkglist.lists.setdefault(remote, PackagesList())
+                remote_list.add_refs([ref])
+            pref = PkgReference(ref, node["package_id"], node["prev"])
+            binary_remote = node["binary_remote"]
+            if binary_remote:
+                remote_list = pkglist.lists.setdefault(binary_remote, PackagesList())
+                remote_list.add_refs([ref])  # Binary listed forces recipe listed
+                remote_list.add_prefs(ref, [pref])
+
+            binary = node["binary"]
+            if binary in (BINARY_SKIP, BINARY_INVALID, BINARY_MISSING):
+                continue
+
+            binary = binary.lower()
+            if any(b == "*" or b == binary for b in binaries):
+                cache_list.add_refs([ref])  # Binary listed forces recipe listed
+                cache_list.add_prefs(ref, [pref])
+        return pkglist
+
+
 class PackagesList:
     def __init__(self):
         self.recipes = {}
@@ -75,7 +158,9 @@ def refs(self):
         for ref, ref_dict in self.recipes.items():
             for rrev, rrev_dict in ref_dict.get("revisions", {}).items():
                 t = rrev_dict.get("timestamp")
-                recipe = RecipeReference.loads(f"{ref}#{rrev}%{t}")  # TODO: optimize this
+                recipe = RecipeReference.loads(f"{ref}#{rrev}")  # TODO: optimize this
+                if t is not None:
+                    recipe.timestamp = t
                 result[recipe] = rrev_dict
         return result.items()
 
@@ -93,6 +178,12 @@ def prefs(ref, recipe_bundle):
     def serialize(self):
         return self.recipes
 
+    @staticmethod
+    def deserialize(data):
+        result = PackagesList()
+        result.recipes = data
+        return result
+
 
 class ListPattern:
 

conan/cli/commands/download.py

@@ -1,12 +1,16 @@
 from multiprocessing.pool import ThreadPool
 
 from conan.api.conan_api import ConanAPI
-from conan.api.model import ListPattern
+from conan.api.model import ListPattern, MultiPackagesList
 from conan.api.output import ConanOutput
+from conan.cli import make_abs_path
 from conan.cli.command import conan_command, OnceArgument
+from conan.cli.commands.list import print_list_text, print_list_json
+from conans.errors import ConanException
 
 
-@conan_command(group="Creator")
+@conan_command(group="Creator", formatters={"text": print_list_text,
+                                            "json": print_list_json})
 def download(conan_api: ConanAPI, parser, *args):
     """
     Download (without installing) a single conan package from a remote server.
@@ -17,27 +21,45 @@ def download(conan_api: ConanAPI, parser, *args):
     queries over the package binaries.
     """
 
-    parser.add_argument('reference', help="Recipe reference or package reference, can contain * as "
-                                          "wildcard at any reference field. If revision is not "
-                                          "specified, it is assumed latest one.")
+    parser.add_argument('pattern', nargs="?",
+                        help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                             "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                             "If revision is not specified, it is assumed latest one.")
     parser.add_argument("--only-recipe", action='store_true', default=False,
                         help='Download only the recipe/s, not the binary packages.')
     parser.add_argument('-p', '--package-query', default=None, action=OnceArgument,
                         help="Only download packages matching a specific query. e.g: os=Windows AND "
                              "(arch=x86 OR compiler=gcc)")
     parser.add_argument("-r", "--remote", action=OnceArgument, required=True,
                         help='Download from this specific remote')
+    parser.add_argument("-l", "--list", help="Package list file")
 
     args = parser.parse_args(*args)
+    if args.pattern is None and args.list is None:
+        raise ConanException("Missing pattern or package list file")
+    if args.pattern and args.list:
+        raise ConanException("Cannot define both the pattern and the package list file")
+
     remote = conan_api.remotes.get(args.remote)
+
+    if args.list:
+        listfile = make_abs_path(args.list)
+        multi_package_list = MultiPackagesList.load(listfile)
+        try:
+            package_list = multi_package_list[remote.name]
+        except KeyError:
+            raise ConanException(f"The current package list does not contain remote '{remote.name}'")
+    else:
+        ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe)
+        package_list = conan_api.list.select(ref_pattern, args.package_query, remote)
+
     parallel = conan_api.config.get("core.download:parallel", default=1, check_type=int)
-    ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe)
-    select_bundle = conan_api.list.select(ref_pattern, args.package_query, remote)
+
     refs = []
     prefs = []
-    for ref, recipe_bundle in select_bundle.refs():
+    for ref, recipe_bundle in package_list.refs():
         refs.append(ref)
-        for pref, _ in select_bundle.prefs(ref, recipe_bundle):
+        for pref, _ in package_list.prefs(ref, recipe_bundle):
             prefs.append(pref)
 
     if parallel <= 1:
@@ -48,6 +70,8 @@ def download(conan_api: ConanAPI, parser, *args):
     else:
         _download_parallel(parallel, conan_api, refs, prefs, remote)
 
+    return {"results": {"Local Cache": package_list.serialize()}}
+
 
 def _download_parallel(parallel, conan_api, refs, prefs, remote):
 

conan/cli/commands/list.py

@@ -1,14 +1,16 @@
 import json
 
 from conan.api.conan_api import ConanAPI
-from conan.api.model import ListPattern
+from conan.api.model import ListPattern, MultiPackagesList
 from conan.api.output import Color, cli_out_write
+from conan.cli import make_abs_path
 from conan.cli.command import conan_command, OnceArgument
 from conan.cli.formatters.list import list_packages_html
-
-# Keep them so we don't break other commands that import them, but TODO: Remove later
+from conans.errors import ConanException
 from conans.util.dates import timestamp_to_str
 
+
+# Keep them so we don't break other commands that import them, but TODO: Remove later
 remote_color = Color.BRIGHT_BLUE
 recipe_name_color = Color.GREEN
 recipe_color = Color.BRIGHT_WHITE
@@ -93,36 +95,55 @@ def list(conan_api: ConanAPI, parser, *args):
     """
     List existing recipes, revisions, or packages in the cache (by default) or the remotes.
     """
-    parser.add_argument('reference', help="Recipe reference or package reference. "
-                                          "Both can contain * as wildcard at any reference field. "
-                                          "If revision is not specified, it is assumed latest one.")
+    parser.add_argument('pattern', nargs="?",
+                        help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                             "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                             "If revision is not specified, it is assumed latest one.")
     parser.add_argument('-p', '--package-query', default=None, action=OnceArgument,
                         help="List only the packages matching a specific query, e.g, os=Windows AND "
                              "(arch=x86 OR compiler=gcc)")
     parser.add_argument("-r", "--remote", default=None, action="append",
                         help="Remote names. Accepts wildcards ('*' means all the remotes available)")
     parser.add_argument("-c", "--cache", action='store_true', help="Search in the local cache")
+    parser.add_argument("-g", "--graph", help="Graph json file")
+    parser.add_argument("-gb", "--graph-binaries", help="Which binaries are listed", action="append")
+    parser.add_argument("-gr", "--graph-recipes", help="Which recipes are listed", action="append")
 
     args = parser.parse_args(*args)
-    ref_pattern = ListPattern(args.reference, rrev=None, prev=None)
-    # If neither remote nor cache are defined, show results only from cache
-    remotes = []
-    if args.cache or not args.remote:
-        remotes.append(None)
-    if args.remote:
-        remotes.extend(conan_api.remotes.list(args.remote))
-    results = {}
-    for remote in remotes:
-        name = getattr(remote, "name", "Local Cache")
-        try:
-            list_bundle = conan_api.list.select(ref_pattern, args.package_query, remote)
-        except Exception as e:
-            results[name] = {"error": str(e)}
-        else:
-            results[name] = list_bundle.serialize()
+
+    if args.pattern is None and args.graph is None:
+        raise ConanException("Missing pattern or graph json file")
+    if args.pattern and args.graph:
+        raise ConanException("Cannot define both the pattern and the graph json file")
+    if (args.graph_recipes or args.graph_binaries) and not args.graph:
+        raise ConanException("--graph-recipes and --graph-binaries require a --graph input")
+
+    if args.graph:
+        graphfile = make_abs_path(args.graph)
+        pkglist = MultiPackagesList.load_graph(graphfile, args.graph_recipes, args.graph_binaries)
+    else:
+        ref_pattern = ListPattern(args.pattern, rrev=None, prev=None)
+        # If neither remote nor cache are defined, show results only from cache
+        pkglist = MultiPackagesList()
+        if args.cache or not args.remote:
+            try:
+                cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None)
+            except Exception as e:
+                pkglist.add_error("Local Cache", str(e))
+            else:
+                pkglist.add("Local Cache", cache_list)
+        if args.remote:
+            remotes = conan_api.remotes.list(args.remote)
+            for remote in remotes:
+                try:
+                    remote_list = conan_api.list.select(ref_pattern, args.package_query, remote)
+                except Exception as e:
+                    pkglist.add_error(remote.name, str(e))
+                else:
+                    pkglist.add(remote.name, remote_list)
 
     return {
-        "results": results,
+        "results": pkglist.serialize(),
         "conan_api": conan_api,
         "cli_args": " ".join([f"{arg}={getattr(args, arg)}" for arg in vars(args) if getattr(args, arg)])
     }

conan/cli/commands/remove.py

@@ -16,8 +16,10 @@ def remove(conan_api: ConanAPI, parser, *args):
       will be removed.
     - If a package reference is specified, it will remove only the package.
     """
-    parser.add_argument('reference', help="Recipe reference or package reference, can contain * as"
-                                          "wildcard at any reference field. e.g: lib/*")
+    parser.add_argument('pattern',
+                        help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                             "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                             "If revision is not specified, it is assumed latest one.")
     parser.add_argument('-c', '--confirm', default=False, action='store_true',
                         help='Remove without requesting a confirmation')
     parser.add_argument('-p', '--package-query', action=OnceArgument,
@@ -33,7 +35,7 @@ def remove(conan_api: ConanAPI, parser, *args):
     def confirmation(message):
         return args.confirm or ui.request_boolean(message)
 
-    ref_pattern = ListPattern(args.reference, rrev="*", prev="*")
+    ref_pattern = ListPattern(args.pattern, rrev="*", prev="*")
     select_bundle = conan_api.list.select(ref_pattern, args.package_query, remote)
 
     if ref_pattern.package_id is None:

conan/cli/commands/upload.py

@@ -1,5 +1,6 @@
 from conan.api.conan_api import ConanAPI
-from conan.api.model import ListPattern
+from conan.api.model import ListPattern, MultiPackagesList
+from conan.cli import make_abs_path
 from conan.cli.command import conan_command, OnceArgument
 from conans.client.userio import UserInput
 from conan.errors import ConanException
@@ -15,9 +16,10 @@ def upload(conan_api: ConanAPI, parser, *args):
     binary packages, unless --only-recipe is specified. You can use the "latest" placeholder at the
     "reference" argument to specify the latest revision of the recipe or the package.
     """
-    parser.add_argument('reference', help="Recipe reference or package reference, can contain * as "
-                                          "wildcard at any reference field. If no revision is "
-                                          "specified, it is assumed to be the latest")
+    parser.add_argument('pattern', nargs="?",
+                        help="A pattern in the form 'pkg/version#revision:package_id#revision', "
+                             "e.g: zlib/1.2.13:* means all binaries for zlib/1.2.13. "
+                             "If revision is not specified, it is assumed latest one.")
     parser.add_argument('-p', '--package-query', default=None, action=OnceArgument,
                         help="Only upload packages matching a specific query. e.g: os=Windows AND "
                              "(arch=x86 OR compiler=gcc)")
@@ -33,25 +35,35 @@ def upload(conan_api: ConanAPI, parser, *args):
                         help='Perform an integrity check, using the manifests, before upload')
     parser.add_argument('-c', '--confirm', default=False, action='store_true',
                         help='Upload all matching recipes without confirmation')
+    parser.add_argument("-l", "--list", help="Package list file")
 
     args = parser.parse_args(*args)
 
     remote = conan_api.remotes.get(args.remote)
     enabled_remotes = conan_api.remotes.list()
 
-    ref_pattern = ListPattern(args.reference, package_id="*", only_recipe=args.only_recipe)
-    package_list = conan_api.list.select(ref_pattern, package_query=args.package_query)
+    if args.pattern is None and args.list is None:
+        raise ConanException("Missing pattern or package list file")
+    if args.pattern and args.list:
+        raise ConanException("Cannot define both the pattern and the package list file")
+    if args.list:
+        listfile = make_abs_path(args.list)
+        multi_package_list = MultiPackagesList.load(listfile)
+        package_list = multi_package_list["Local Cache"]
+    else:
+        ref_pattern = ListPattern(args.pattern, package_id="*", only_recipe=args.only_recipe)
+        package_list = conan_api.list.select(ref_pattern, package_query=args.package_query)
 
     if not package_list.recipes:
-        raise ConanException("No recipes found matching pattern '{}'".format(args.reference))
+        raise ConanException("No recipes found matching pattern '{}'".format(args.pattern))
 
     if args.check:
         conan_api.cache.check_integrity(package_list)
     # Check if the recipes/packages are in the remote
     conan_api.upload.check_upstream(package_list, remote, args.force)
 
     # If only if search with "*" we ask for confirmation
-    if not args.confirm and "*" in args.reference:
+    if not args.list and not args.confirm and "*" in args.pattern:
         _ask_confirm_upload(conan_api, package_list)
 
     conan_api.upload.prepare(package_list, enabled_remotes)