InstallGraph.reduce

conan/cli/commands/graph.py

@@ -63,6 +63,9 @@ def graph_build_order(conan_api, parser, subparser, *args):
     common_graph_args(subparser)
     subparser.add_argument("--order-by", choices=['recipe', 'configuration'],
                            help='Select how to order the output, "recipe" by default if not set.')
+    subparser.add_argument("--reduce", action='store_true', default=False,
+                           help='Reduce the build order, output only those to build. Use this '
+                                'only if the result will not be merged later with other build-order')
     args = parser.parse_args(*args)
 
     # parameter validation
@@ -102,7 +105,12 @@ def graph_build_order(conan_api, parser, subparser, *args):
 
     out = ConanOutput()
     out.title("Computing the build order")
+
     install_graph = InstallGraph(deps_graph, order_by=args.order_by)
+    if args.reduce:
+        if args.order_by is None:
+            raise ConanException("--reduce needs --order-by argument defined")
+        install_graph.reduce()
     install_order_serialized = install_graph.install_build_order()
     if args.order_by is None:  # legacy
         install_order_serialized = install_order_serialized["order"]
@@ -120,15 +128,24 @@ def graph_build_order_merge(conan_api, parser, subparser, *args):
     Merge more than 1 build-order file.
     """
     subparser.add_argument("--file", nargs="?", action="append", help="Files to be merged")
+    subparser.add_argument("--reduce", action='store_true', default=False,
+                           help='Reduce the build order, output only those to build. Use this '
+                                'only if the result will not be merged later with other build-order')
     args = parser.parse_args(*args)
     if not args.file or len(args.file) < 2:
         raise ConanException("At least 2 files are needed to be merged")
 
     result = InstallGraph.load(make_abs_path(args.file[0]))
+    if result.reduced:
+        raise ConanException(f"Reduced build-order file cannot be merged: {args.file[0]}")
     for f in args.file[1:]:
         install_graph = InstallGraph.load(make_abs_path(f))
+        if install_graph.reduced:
+            raise ConanException(f"Reduced build-order file cannot be merged: {f}")
         result.merge(install_graph)
 
+    if args.reduce:
+        result.reduce()
     install_order_serialized = result.install_build_order()
     if getattr(result, "legacy"):
         install_order_serialized = install_order_serialized["order"]

conans/client/graph/install_graph.py

@@ -4,7 +4,7 @@
 
 from conan.api.output import ConanOutput
 from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_SKIP, \
-    BINARY_MISSING, BINARY_INVALID, Overrides, BINARY_BUILD
+    BINARY_MISSING, BINARY_INVALID, Overrides, BINARY_BUILD, BINARY_EDITABLE_BUILD
 from conans.errors import ConanInvalidConfiguration, ConanException
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
@@ -111,6 +111,13 @@ def __init__(self):
         self.packages = {}  # {package_id: _InstallPackageReference}
         self.depends = []  # Other REFs, defines the graph topology and operation ordering
 
+    @property
+    def need_build(self):
+        for package in self.packages.values():
+            if package.binary in (BINARY_BUILD, BINARY_EDITABLE_BUILD):
+                return True
+        return False
+
     @property
     def node(self):
         return self._node
@@ -212,6 +219,10 @@ def __init__(self):
         self.depends = []  # List of full prefs
         self.overrides = Overrides()
 
+    @property
+    def need_build(self):
+        return self.binary in (BINARY_BUILD, BINARY_EDITABLE_BUILD)
+
     @property
     def pref(self):
         return PkgReference(self.ref, self.package_id, self.prev)
@@ -292,6 +303,8 @@ def deserialize(data, filename):
         return result
 
     def merge(self, other):
+        assert self.binary == other.binary, f"Binary for {self.ref}: {self.binary}!={other.binary}"
+
         assert self.ref == other.ref
         for d in other.depends:
             if d not in self.depends:
@@ -312,6 +325,7 @@ def __init__(self, deps_graph, order_by=None):
         self._order = order_by
         self._node_cls = _InstallRecipeReference if order_by == "recipe" else _InstallConfiguration
         self._is_test_package = False
+        self.reduced = False
         if deps_graph is not None:
             self._initialize_deps_graph(deps_graph)
             self._is_test_package = deps_graph.root.conanfile.tested_reference_str is not None
@@ -328,8 +342,10 @@ def merge(self, other):
         """
         @type other: InstallGraph
         """
+        if self.reduced or other.reduced:
+            raise ConanException("Reduced build-order files cannot be merged")
         if self._order != other._order:
-            raise ConanException(f"Cannot merge build-orders of `{self._order}!={other._order}")
+            raise ConanException(f"Cannot merge build-orders of {self._order}!={other._order}")
         for ref, install_node in other._nodes.items():
             existing = self._nodes.get(ref)
             if existing is None:
@@ -340,8 +356,10 @@ def merge(self, other):
     @staticmethod
     def deserialize(data, filename):
         legacy = isinstance(data, list)
-        order, data = ("recipe", data) if legacy else (data["order_by"], data["order"])
+        order, data, reduced = ("recipe", data, False) if legacy else \
+            (data["order_by"], data["order"], data["reduced"])
         result = InstallGraph(None, order_by=order)
+        result.reduced = reduced
         result.legacy = legacy
         for level in data:
             for item in level:
@@ -362,6 +380,22 @@ def _initialize_deps_graph(self, deps_graph):
             else:
                 existing.add(node)
 
+    def reduce(self):
+        result = {}
+        for k, node in self._nodes.items():
+            if node.need_build:
+                result[k] = node
+            else:  # Eliminate this element from the graph
+                dependencies = node.depends
+                # Find all consumers
+                for n in self._nodes.values():
+                    if k in n.depends:
+                        n.depends = [d for d in n.depends if d != k]  # Discard the removed node
+                        # Add new edges, without repetition
+                        n.depends.extend(d for d in dependencies if d not in n.depends)
+        self._nodes = result
+        self.reduced = True
+
     def install_order(self, flat=False):
         # a topological order by levels, returns a list of list, in order of processing
         levels = []
@@ -390,6 +424,7 @@ def install_build_order(self):
         """
         install_order = self.install_order()
         result = {"order_by": self._order,
+                  "reduced": self.reduced,
                   "order": [[n.serialize() for n in level] for level in install_order]}
         return result
 

conans/test/integration/command_v2/test_info_build_order.py

@@ -2,6 +2,8 @@
 import os
 import textwrap
 
+import pytest
+
 from conans.test.assets.genconanfile import GenConanfile
 from conans.test.utils.tools import TestClient
 
@@ -66,6 +68,11 @@ def test_info_build_order():
     assert bo_json["order_by"] == "recipe"
     assert bo_json["order"] == result
 
+    c.run("graph build-order consumer --build=missing --order-by=recipe --reduce --format=json")
+    bo_json = json.loads(c.stdout)
+    assert bo_json["order_by"] == "recipe"
+    assert bo_json["order"] == result
+
 
 def test_info_build_order_configuration():
     c = TestClient()
@@ -116,6 +123,10 @@ def test_info_build_order_configuration():
 
     assert bo_json["order"] == result
 
+    c.run("graph build-order consumer --build=missing --order=configuration --reduce --format=json")
+    bo_json = json.loads(c.stdout)
+    assert bo_json["order"] == result
+
 
 def test_info_build_order_configuration_text_formatter():
     c = TestClient()
@@ -533,3 +544,111 @@ def export(self):
     c.run("graph build-order --requires=dep/0.1 --format=json", assert_error=True)
     assert "ImportError" in c.out
     assert "It is possible that this recipe is not Conan 2.0 ready" in c.out
+
+
+class TestBuildOrderReduce:
+    @pytest.mark.parametrize("order", ["recipe", "configuration"])
+    def test_build_order_reduce(self, order):
+        c = TestClient()
+        c.save({"liba/conanfile.py": GenConanfile("liba", "0.1"),
+                "libb/conanfile.py": GenConanfile("libb", "0.1").with_requires("liba/0.1"),
+                "libc/conanfile.py": GenConanfile("libc", "0.1").with_requires("libb/0.1"),
+                "consumer/conanfile.txt": "[requires]\nlibc/0.1"})
+        c.run("create liba")
+        c.run("create libb")
+        c.run("create libc")
+        c.run("remove liba:* -c")
+        c.run("remove libc:* -c")
+        c.run(f"graph build-order consumer --order={order} --build=missing --reduce --format=json")
+        bo_json = json.loads(c.stdout)
+        order_json = bo_json["order"]
+        assert len(order_json) == 2  # 2 levels
+        level0, level1 = order_json
+        assert len(level0) == 1
+        assert level0[0]["ref"] == "liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2"
+        # then libc -> directly on liba, no libb involved
+        assert len(level1) == 1
+        assert level1[0]["ref"] == "libc/0.1#c04c370ad966390e67388565b56f019a"
+        depends = "liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2"
+        if order == "configuration":
+            depends += ":da39a3ee5e6b4b0d3255bfef95601890afd80709"
+        assert level1[0]["depends"] == [depends]
+
+    @pytest.mark.parametrize("order", ["recipe", "configuration"])
+    def test_build_order_merge_reduce(self, order):
+        c = TestClient()
+        c.save({"liba/conanfile.py": GenConanfile("liba", "0.1").with_settings("os"),
+                "libb/conanfile.py": GenConanfile("libb", "0.1").with_settings("os")
+                                                                .with_requires("liba/0.1"),
+                "libc/conanfile.py": GenConanfile("libc", "0.1").with_settings("os")
+                                                                .with_requires("libb/0.1"),
+                "consumer/conanfile.txt": "[requires]\nlibc/0.1"})
+        for _os in ("Windows", "Linux"):
+            c.run(f"create liba -s os={_os}")
+            c.run(f"create libb -s os={_os}")
+            c.run(f"create libc -s os={_os}")
+
+        c.run("remove liba:* -c")
+        c.run("remove libc:* -c")
+        c.run(f"graph build-order consumer --order={order} --build=missing -s os=Windows "
+              "--format=json", redirect_stdout="windows.json")
+        c.run(f"graph build-order consumer --order={order} --build=missing -s os=Linux "
+              "--format=json", redirect_stdout="linux.json")
+
+        c.run(f"graph build-order-merge --file=windows.json --file=linux.json --reduce "
+              "--format=json")
+        bo_json = json.loads(c.stdout)
+        order_json = bo_json["order"]
+        assert len(order_json) == 2  # 2 levels
+        level0, level1 = order_json
+        if order == "recipe":
+            assert len(level0) == 1
+            assert level0[0]["ref"] == "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e"
+            # then libc -> directly on liba, no libb involved
+            assert len(level1) == 1
+            assert level1[0]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3"
+            depends = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e"
+            assert level1[0]["depends"] == [depends]
+        else:
+            assert len(level0) == 2
+            liba1 = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e:" \
+                    "ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715"
+            liba2 = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e:" \
+                    "9a4eb3c8701508aa9458b1a73d0633783ecc2270"
+            assert level0[0]["pref"] == liba1
+            assert level0[1]["pref"] == liba2
+            # then libc -> directly on liba, no libb involved
+            assert len(level1) == 2
+            assert level1[0]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3"
+            assert level1[0]["depends"] == [liba1]
+            assert level1[1]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3"
+            assert level1[1]["depends"] == [liba2]
+
+    def test_error_reduced(self):
+        c = TestClient()
+        c.save({"conanfile.py": GenConanfile("liba", "0.1")})
+        c.run("graph build-order . --format=json", redirect_stdout="bo1.json")
+        c.run("graph build-order . --order-by=recipe --reduce --format=json",
+              redirect_stdout="bo2.json")
+        c.run(f"graph build-order-merge --file=bo1.json --file=bo2.json", assert_error=True)
+        assert "ERROR: Reduced build-order file cannot be merged: bo2.json"
+        # different order
+        c.run(f"graph build-order-merge --file=bo2.json --file=bo1.json", assert_error=True)
+        assert "ERROR: Reduced build-order file cannot be merged: bo2.json"
+
+    def test_error_different_orders(self):
+        c = TestClient()
+        c.save({"conanfile.py": GenConanfile("liba", "0.1")})
+        c.run("graph build-order . --format=json", redirect_stdout="bo1.json")
+        c.run("graph build-order . --order-by=recipe --format=json", redirect_stdout="bo2.json")
+        c.run("graph build-order . --order-by=configuration --format=json",
+              redirect_stdout="bo3.json")
+        c.run(f"graph build-order-merge --file=bo1.json --file=bo2.json")
+        # Not error
+        c.run(f"graph build-order-merge --file=bo1.json --file=bo3.json", assert_error=True)
+        assert "ERROR: Cannot merge build-orders of recipe!=configuration" in c.out
+        c.run(f"graph build-order-merge --file=bo2.json --file=bo3.json", assert_error=True)
+        assert "ERROR: Cannot merge build-orders of recipe!=configuration" in c.out
+        # different order
+        c.run(f"graph build-order-merge --file=bo3.json --file=bo2.json", assert_error=True)
+        assert "ERROR: Cannot merge build-orders of configuration!=recipe" in c.out