From a6e3ac46268e524c9990eada8327ef8092982744 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 31 Jan 2024 20:03:50 +0100 Subject: [PATCH 1/3] InstallGraph.reduce --- conan/cli/commands/graph.py | 4 +++ conans/client/graph/install_graph.py | 26 ++++++++++++++++- .../command_v2/test_info_build_order.py | 29 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 47c396a5b1b..782a18c19ca 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -62,6 +62,8 @@ def graph_build_order(conan_api, parser, subparser, *args): common_graph_args(subparser) subparser.add_argument("--order", choices=['recipe', 'configuration'], default="recipe", help='Select how to order the output, "recipe" by default if not set.') + subparser.add_argument("--reduce", action='store_true', default=False, + help='Reduce the build order, output only those to build') args = parser.parse_args(*args) # parameter validation @@ -100,6 +102,8 @@ def graph_build_order(conan_api, parser, subparser, *args): out = ConanOutput() out.title("Computing the build order") install_graph = InstallGraph(deps_graph, order=args.order) + if args.reduce: + install_graph.reduce() install_order_serialized = install_graph.install_build_order() lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index e34c525bf03..75a1b0fd681 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -4,7 +4,7 @@ from conan.api.output import ConanOutput from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_SKIP, \ - BINARY_MISSING, BINARY_INVALID, Overrides, BINARY_BUILD + BINARY_MISSING, BINARY_INVALID, Overrides, BINARY_BUILD, BINARY_EDITABLE_BUILD from conans.errors import ConanInvalidConfiguration, ConanException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference @@ -111,6 +111,13 @@ def __init__(self): self.packages = {} # {package_id: _InstallPackageReference} self.depends = [] # Other REFs, defines the graph topology and operation ordering + @property + def need_build(self): + for package in self.packages.values(): + if package.binary in (BINARY_BUILD, BINARY_EDITABLE_BUILD): + return True + return False + @property def node(self): return self._node @@ -212,6 +219,10 @@ def __init__(self): self.depends = [] # List of full prefs self.overrides = Overrides() + @property + def need_build(self): + return self.binary in (BINARY_BUILD, BINARY_EDITABLE_BUILD) + @property def pref(self): return PkgReference(self.ref, self.package_id, self.prev) @@ -363,6 +374,19 @@ def _initialize_deps_graph(self, deps_graph): else: existing.add(node) + def reduce(self): + result = {} + for k, node in self._nodes.items(): + if node.need_build: + result[k] = node + else: # Eliminate this element from the graph + dependencies = node.depends + # Find all consumers + for n in self._nodes.values(): + if k in n.depends: + n.depends = [d for d in n.depends if d != k] + dependencies + self._nodes = result + def install_order(self, flat=False): # a topological order by levels, returns a list of list, in order of processing levels = [] diff --git a/conans/test/integration/command_v2/test_info_build_order.py b/conans/test/integration/command_v2/test_info_build_order.py index bee61443026..8b20165075a 100644 --- a/conans/test/integration/command_v2/test_info_build_order.py +++ b/conans/test/integration/command_v2/test_info_build_order.py @@ -60,6 +60,9 @@ def test_info_build_order(): ] assert bo_json == result + c.run("graph build-order consumer --build=missing --reduce --format=json") + bo_json = json.loads(c.stdout) + assert bo_json == result def test_info_build_order_configuration(): @@ -109,6 +112,9 @@ def test_info_build_order_configuration(): ] assert bo_json == result + c.run("graph build-order consumer --build=missing --order=configuration --reduce --format=json") + bo_json = json.loads(c.stdout) + assert bo_json == result def test_info_build_order_configuration_text_formatter(): @@ -526,3 +532,26 @@ def export(self): c.run("graph build-order --requires=dep/0.1 --format=json", assert_error=True) assert "ImportError" in c.out assert "It is possible that this recipe is not Conan 2.0 ready" in c.out + + +class TestBuildOrderReduce: + def test_reduce(self): + c = TestClient() + c.save({"liba/conanfile.py": GenConanfile("liba", "0.1"), + "libb/conanfile.py": GenConanfile("libb", "0.1").with_requires("liba/0.1"), + "libc/conanfile.py": GenConanfile("libc", "0.1").with_requires("libb/0.1"), + "consumer/conanfile.txt": "[requires]\nlibc/0.1"}) + c.run("create liba") + c.run("create libb") + c.run("create libc") + c.run("remove liba:* -c") + c.run("remove libc:* -c") + c.run("graph build-order consumer --build=missing --reduce --format=json") + bo_json = json.loads(c.stdout) + assert len(bo_json) == 2 # 2 levels + level0 = bo_json[0] + assert len(level0) == 1 + assert level0[0]["ref"] == "liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2" + level1 = bo_json[1] + assert len(level1) == 1 + assert level1[0]["depends"] == ["liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2"] From daf90acee9968cc4549ba9686c7406253a210468 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 5 Feb 2024 18:20:46 +0100 Subject: [PATCH 2/3] wip --- conan/cli/commands/graph.py | 8 ++- conans/client/graph/install_graph.py | 4 +- .../command_v2/test_info_build_order.py | 66 +++++++++++++++++-- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 782a18c19ca..c687e560aa3 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -63,7 +63,8 @@ def graph_build_order(conan_api, parser, subparser, *args): subparser.add_argument("--order", choices=['recipe', 'configuration'], default="recipe", help='Select how to order the output, "recipe" by default if not set.') subparser.add_argument("--reduce", action='store_true', default=False, - help='Reduce the build order, output only those to build') + help='Reduce the build order, output only those to build. Use this ' + 'only if the result will not be merged later with other build-order') args = parser.parse_args(*args) # parameter validation @@ -119,6 +120,9 @@ def graph_build_order_merge(conan_api, parser, subparser, *args): Merge more than 1 build-order file. """ subparser.add_argument("--file", nargs="?", action="append", help="Files to be merged") + subparser.add_argument("--reduce", action='store_true', default=False, + help='Reduce the build order, output only those to build. Use this ' + 'only if the result will not be merged later with other build-order') args = parser.parse_args(*args) if not args.file or len(args.file) < 2: raise ConanException("At least 2 files are needed to be merged") @@ -128,6 +132,8 @@ def graph_build_order_merge(conan_api, parser, subparser, *args): install_graph = InstallGraph.load(make_abs_path(f)) result.merge(install_graph) + if args.reduce: + result.reduce() install_order_serialized = result.install_build_order() return install_order_serialized diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index 75a1b0fd681..35ccdd9a17b 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -384,7 +384,9 @@ def reduce(self): # Find all consumers for n in self._nodes.values(): if k in n.depends: - n.depends = [d for d in n.depends if d != k] + dependencies + n.depends = [d for d in n.depends if d != k] # Discard the removed node + # Add new edges, without repetition + n.depends.extend(d for d in dependencies if d not in n.depends) self._nodes = result def install_order(self, flat=False): diff --git a/conans/test/integration/command_v2/test_info_build_order.py b/conans/test/integration/command_v2/test_info_build_order.py index 8b20165075a..c5848f61e4d 100644 --- a/conans/test/integration/command_v2/test_info_build_order.py +++ b/conans/test/integration/command_v2/test_info_build_order.py @@ -2,6 +2,8 @@ import os import textwrap +import pytest + from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -535,7 +537,8 @@ def export(self): class TestBuildOrderReduce: - def test_reduce(self): + @pytest.mark.parametrize("order", ["recipe", "configuration"]) + def test_build_order_reduce(self, order): c = TestClient() c.save({"liba/conanfile.py": GenConanfile("liba", "0.1"), "libb/conanfile.py": GenConanfile("libb", "0.1").with_requires("liba/0.1"), @@ -546,12 +549,65 @@ def test_reduce(self): c.run("create libc") c.run("remove liba:* -c") c.run("remove libc:* -c") - c.run("graph build-order consumer --build=missing --reduce --format=json") + c.run(f"graph build-order consumer --order={order} --build=missing --reduce --format=json") bo_json = json.loads(c.stdout) assert len(bo_json) == 2 # 2 levels - level0 = bo_json[0] + level0, level1 = bo_json assert len(level0) == 1 assert level0[0]["ref"] == "liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2" - level1 = bo_json[1] + # then libc -> directly on liba, no libb involved assert len(level1) == 1 - assert level1[0]["depends"] == ["liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2"] + assert level1[0]["ref"] == "libc/0.1#c04c370ad966390e67388565b56f019a" + depends = "liba/0.1#a658e7beaaae5d6be0b6f67dcc9859e2" + if order == "configuration": + depends += ":da39a3ee5e6b4b0d3255bfef95601890afd80709" + assert level1[0]["depends"] == [depends] + + @pytest.mark.parametrize("order", ["recipe", "configuration"]) + def test_build_order_merge_reduce(self, order): + c = TestClient() + c.save({"liba/conanfile.py": GenConanfile("liba", "0.1").with_settings("os"), + "libb/conanfile.py": GenConanfile("libb", "0.1").with_settings("os") + .with_requires("liba/0.1"), + "libc/conanfile.py": GenConanfile("libc", "0.1").with_settings("os") + .with_requires("libb/0.1"), + "consumer/conanfile.txt": "[requires]\nlibc/0.1"}) + for _os in ("Windows", "Linux"): + c.run(f"create liba -s os={_os}") + c.run(f"create libb -s os={_os}") + c.run(f"create libc -s os={_os}") + + c.run("remove liba:* -c") + c.run("remove libc:* -c") + c.run(f"graph build-order consumer --order={order} --build=missing -s os=Windows " + "--format=json", redirect_stdout="windows.json") + c.run(f"graph build-order consumer --order={order} --build=missing -s os=Linux " + "--format=json", redirect_stdout="linux.json") + + c.run(f"graph build-order-merge --file=windows.json --file=linux.json --reduce " + "--format=json") + bo_json = json.loads(c.stdout) + assert len(bo_json) == 2 # 2 levels + level0, level1 = bo_json + if order == "recipe": + assert len(level0) == 1 + assert level0[0]["ref"] == "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e" + # then libc -> directly on liba, no libb involved + assert len(level1) == 1 + assert level1[0]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3" + depends = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e" + assert level1[0]["depends"] == [depends] + else: + assert len(level0) == 2 + liba1 = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e:" \ + "ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715" + liba2 = "liba/0.1#8c6ed89c12ab2ce78b239224bd7cb79e:" \ + "9a4eb3c8701508aa9458b1a73d0633783ecc2270" + assert level0[0]["pref"] == liba1 + assert level0[1]["pref"] == liba2 + # then libc -> directly on liba, no libb involved + assert len(level1) == 2 + assert level1[0]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3" + assert level1[0]["depends"] == [liba1] + assert level1[1]["ref"] == "libc/0.1#66db2600b9d6a2a61c9051fcf47da4a3" + assert level1[1]["depends"] == [liba2] From bb8bada40652447c2b1f28abe76fa9f65cfc0e60 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 7 Feb 2024 00:27:55 +0100 Subject: [PATCH 3/3] add assert --- conans/client/graph/install_graph.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conans/client/graph/install_graph.py b/conans/client/graph/install_graph.py index 5a82fc69e83..c044f4fb176 100644 --- a/conans/client/graph/install_graph.py +++ b/conans/client/graph/install_graph.py @@ -303,6 +303,8 @@ def deserialize(data, filename): return result def merge(self, other): + assert self.binary == other.binary, f"Binary for {self.ref}: {self.binary}!={other.binary}" + assert self.ref == other.ref for d in other.depends: if d not in self.depends: