[develop2][poc] Checking the [system_tools] idea

conans/client/graph/graph.py

@@ -14,6 +14,7 @@
 RECIPE_CONSUMER = "Consumer"  # A conanfile from the user
 RECIPE_VIRTUAL = "Virtual"  # A virtual conanfile (dynamic in memory conanfile)
 RECIPE_MISSING = "Missing recipe"  # Impossible to find a recipe for this reference
+RECIPE_SYSTEM_TOOL = "System tool"
 
 BINARY_CACHE = "Cache"
 BINARY_DOWNLOAD = "Download"
@@ -24,6 +25,7 @@
 BINARY_EDITABLE = "Editable"
 BINARY_EDITABLE_BUILD = "EditableBuild"
 BINARY_INVALID = "Invalid"
+BINARY_SYSTEM_TOOL = "System tool"
 
 CONTEXT_HOST = "host"
 CONTEXT_BUILD = "build"

conans/client/graph/graph_binaries.py

@@ -4,7 +4,8 @@
 from conans.client.graph.graph import (BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_MISSING,
                                        BINARY_UPDATE, RECIPE_EDITABLE, BINARY_EDITABLE,
                                        RECIPE_CONSUMER, RECIPE_VIRTUAL, BINARY_SKIP,
-                                       BINARY_INVALID, BINARY_EDITABLE_BUILD)
+                                       BINARY_INVALID, BINARY_EDITABLE_BUILD, RECIPE_SYSTEM_TOOL,
+                                       BINARY_SYSTEM_TOOL)
 from conans.errors import NoRemoteAvailable, NotFoundException, \
     PackageNotFoundException, conanfile_exception_formatter
 
@@ -159,6 +160,9 @@ def _process_node(self, node, build_mode):
         if node.conanfile.info.invalid:
             node.binary = BINARY_INVALID
             return
+        if node.recipe == RECIPE_SYSTEM_TOOL:
+            node.binary = BINARY_SYSTEM_TOOL
+            return
 
         if node.recipe == RECIPE_EDITABLE:
             # TODO: Check what happens when editable is passed an Invalid configuration

conans/client/graph/graph_builder.py

@@ -4,10 +4,12 @@
 from conans.client.conanfile.configure import run_configure_method
 from conans.client.graph.graph import DepsGraph, Node, CONTEXT_HOST, \
     CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL
+from conans.client.graph.graph import RECIPE_SYSTEM_TOOL
 from conans.client.graph.graph_error import GraphError
 from conans.client.graph.profile_node_definer import initialize_conanfile_profile
 from conans.client.graph.provides import check_graph_provides
 from conans.errors import ConanException
+from conans.model.conan_file import ConanFile
 from conans.model.options import Options
 from conans.model.recipe_ref import RecipeReference, ref_matches
 from conans.model.requires import Requirement
@@ -212,15 +214,33 @@ def _resolve_recipe(self, ref, graph_lock):
                                                     check_update=self._check_update)
         return new_ref, dep_conanfile, recipe_status, remote
 
+    @staticmethod
+    def _resolved_system_tool(node, require, profile_build, profile_host):
+        if node.context == CONTEXT_HOST and not require.build:
+            return
+        system_tool = profile_build.system_tool_requires if node.context == CONTEXT_BUILD \
+            else profile_host.system_tool_requires
+        if system_tool:
+            version_range = require.version_range
+            for d in system_tool:
+                if version_range:
+                    if require.ref.name == d.name and d.version in version_range:
+                        return d, ConanFile(str(d)), RECIPE_SYSTEM_TOOL, None
+                elif require.ref.name == d.name and require.ref.version == d.version:
+                    return d, ConanFile( str(d)), RECIPE_SYSTEM_TOOL, None
+
     def _create_new_node(self, node, require, graph, profile_host, profile_build, graph_lock):
-        try:
-            # TODO: If it is locked not resolve range
-            # TODO: This range-resolve might resolve in a given remote or cache
-            # Make sure next _resolve_recipe use it
-            self._resolver.resolve(require, str(node.ref), self._remotes, self._update)
-            resolved = self._resolve_recipe(require.ref, graph_lock)
-        except ConanException as e:
-            raise GraphError.missing(node, require, str(e))
+        resolved = self._resolved_system_tool(node, require, profile_build, profile_host)
+
+        if resolved is None:
+            try:
+                # TODO: If it is locked not resolve range
+                # TODO: This range-resolve might resolve in a given remote or cache
+                # Make sure next _resolve_recipe use it
+                self._resolver.resolve(require, str(node.ref), self._remotes, self._update)
+                resolved = self._resolve_recipe(require.ref, graph_lock)
+            except ConanException as e:
+                raise GraphError.missing(node, require, str(e))
 
         new_ref, dep_conanfile, recipe_status, remote = resolved
         # If the node is virtual or a test package, the require is also "root"

conans/client/installer.py

@@ -8,7 +8,7 @@
 from conans.client.conanfile.package import run_package_method
 from conans.client.generators import write_generators
 from conans.client.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_EDITABLE, \
-    BINARY_UPDATE, BINARY_EDITABLE_BUILD, BINARY_SKIP
+    BINARY_SYSTEM_TOOL, BINARY_UPDATE, BINARY_EDITABLE_BUILD, BINARY_SKIP
 from conans.client.graph.install_graph import InstallGraph
 from conans.client.source import retrieve_exports_sources, config_source
 from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod,
@@ -280,6 +280,8 @@ def _download_pkg(self, package):
         self._remote_manager.get_package(node.conanfile, node.pref, node.binary_remote)
 
     def _handle_package(self, package, install_reference):
+        if package.binary == BINARY_SYSTEM_TOOL:
+            return
         if package.binary in (BINARY_EDITABLE, BINARY_EDITABLE_BUILD):
             self._handle_node_editable(package)
             return

conans/client/profile_loader.py

@@ -252,7 +252,7 @@ class _ProfileValueParser(object):
     @staticmethod
     def get_profile(profile_text, base_profile=None):
         # Trying to strip comments might be problematic if things contain #
-        doc = ConfigParser(profile_text, allowed_fields=["tool_requires",
+        doc = ConfigParser(profile_text, allowed_fields=["tool_requires", "system_tool_requires",
                                                          "settings",
                                                          "options", "conf", "buildenv", "runenv"])
 
@@ -261,6 +261,12 @@ def get_profile(profile_text, base_profile=None):
         options = Options.loads(doc.options) if doc.options else None
         tool_requires = _ProfileValueParser._parse_tool_requires(doc)
 
+        if doc.system_tool_requires:
+            system_tool_requires = [RecipeReference.loads(r.strip())
+                                    for r in doc.system_tool_requires.splitlines() if r.strip()]
+        else:
+            system_tool_requires = []
+
         if doc.conf:
             conf = ConfDefinition()
             conf.loads(doc.conf, profile=True)
@@ -271,6 +277,7 @@ def get_profile(profile_text, base_profile=None):
 
         # Create or update the profile
         base_profile = base_profile or Profile()
+        base_profile.system_tool_requires = system_tool_requires
         base_profile.settings.update(settings)
         for pkg_name, values_dict in package_settings.items():
             base_profile.package_settings[pkg_name].update(values_dict)

conans/model/profile.py

@@ -17,6 +17,7 @@ def __init__(self):
         self.package_settings = defaultdict(OrderedDict)
         self.options = Options()
         self.tool_requires = OrderedDict()  # ref pattern: list of ref
+        self.system_tool_requires = []
         self.conf = ConfDefinition()
         self.buildenv = ProfileEnvironment()
         self.runenv = ProfileEnvironment()
@@ -34,6 +35,7 @@ def serialize(self):
             "package_settings": self.package_settings,
             "options": self.options.serialize(),
             "tool_requires": self.tool_requires,
+            "system_tool_requires": self.system_tool_requires,
             "conf": self.conf.serialize(),
             # FIXME: Perform a serialize method for ProfileEnvironment
             "build_env": self.buildenv.dumps()
@@ -70,6 +72,10 @@ def dumps(self):
             for pattern, req_list in self.tool_requires.items():
                 result.append("%s: %s" % (pattern, ", ".join(str(r) for r in req_list)))
 
+        if self.system_tool_requires:
+            result.append("[system_tool_requires]")
+            result.extend(str(t) for t in self.system_tool_requires)
+
         if self.conf:
             result.append("[conf]")
             result.append(self.conf.dumps())
@@ -107,6 +113,7 @@ def compose_profile(self, other):
                 existing[r.name] = req
             self.tool_requires[pattern] = list(existing.values())
 
+        self.system_tool_requires = self.system_tool_requires + other.system_tool_requires # FIXME repetitions
         self.conf.update_conf_definition(other.conf)
         self.buildenv.update_profile_env(other.buildenv)  # Profile composition, last has priority
         self.runenv.update_profile_env(other.runenv)

conans/test/integration/graph/test_deferred.py

@@ -0,0 +1,95 @@
+import pytest
+
+from conans.test.assets.genconanfile import GenConanfile
+from conans.test.utils.tools import TestClient
+from conans.util.files import save
+
+
+class TestToolRequires:
+    def test_deferred(self):
+        client = TestClient()
+        client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/1.0"),
+                     "profile": "[system_tool_requires]\ntool/1.0"})
+        client.run("create . -pr=profile")
+        print(client.out)
+        assert "tool/1.0 - System tool" in client.out
+
+    def test_deferred_non_matching(self):
+        """ if what is specified in [deferred] doesn't match what the recipe requires, then
+        the deferred will not be used, and the recipe will use its declared version
+        """
+        client = TestClient()
+        client.save({"tool/conanfile.py": GenConanfile("tool", "1.0"),
+                     "conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/1.0"),
+                     "profile": "[system_tool_requires]\ntool/1.1"})
+        client.run("create tool")
+        client.run("create . -pr=profile")
+        assert "tool/1.0#60ed6e65eae112df86da7f6d790887fd - Cache" in client.out
+
+    def test_deferred_range(self):
+        client = TestClient()
+        client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/[>=1.0]"),
+                     "profile": "[system_tool_requires]\ntool/1.1"})
+        client.run("create . -pr=profile")
+        assert "tool/1.1 - System tool" in client.out
+
+    def test_deferred_range_non_matching(self):
+        """ if what is specified in [deferred] doesn't match what the recipe requires, then
+        the deferred will not be used, and the recipe will use its declared version
+        """
+        client = TestClient()
+        client.save({"tool/conanfile.py": GenConanfile("tool", "1.1"),
+                     "conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("tool/[>=1.0]"),
+                     "profile": "[system_tool_requires]\ntool/0.1"})
+        client.run("create tool")
+        client.run("create . -pr=profile")
+        assert "tool/1.1#888bda2348dd2ddcf5960d0af63b08f7 - Cache" in client.out
+
+
+class TestRequires:
+    """ it works exactly the same for require requires
+    """
+    def test_deferred(self):
+        client = TestClient()
+        client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_requires("tool/1.0"),
+                     "profile": "[system_tool_requires]\ntool/1.0"})
+        client.run("create . -pr=profile", assert_error=True)
+        assert "ERROR: Package 'tool/1.0' not resolved: No remote defined" in client.out
+
+
+class TestPackageID:
+    """ if a consumer depends on recipe revision or package_id what happens
+    """
+
+    @pytest.mark.parametrize("package_id_mode", ["recipe_revision_mode", "full_package_mode"])
+    def test_package_id_modes(self, package_id_mode):
+        """ this test validates that the computation of the downstream consumers package_id
+        doesnt break even if it depends on fields not existing in upstream deferred, like revision
+        or package_id
+        """
+        client = TestClient()
+        save(client.cache.new_config_path, f"core.package_id:default_build_mode={package_id_mode}")
+        client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("dep/1.0"),
+                     "profile": "[system_tool_requires]\ndep/1.0"})
+        client.run("create . -pr=profile")
+        assert "dep/1.0:da39a3ee5e6b4b0d3255bfef95601890afd80709 - System tool" in client.out
+
+    def test_package_id_explicit_revision(self):
+        """
+        Changing the deferred revision affects consumers if package_revision_mode=recipe_revision
+        """
+        client = TestClient()
+        save(client.cache.new_config_path, "core.package_id:default_build_mode=recipe_revision_mode")
+        client.save({"conanfile.py": GenConanfile("pkg", "1.0").with_tool_requires("dep/1.0"),
+                     "profile": "[system_tool_requires]\ndep/1.0#r1",
+                     "profile2": "[system_tool_requires]\ndep/1.0#r2"})
+        client.run("create . -pr=profile")
+        assert "dep/1.0#r1:da39a3ee5e6b4b0d3255bfef95601890afd80709 - System tool" in client.out
+        assert "pkg/1.0#27a56f09310cf1237629bae4104fe5bd:" \
+               "ea0e320d94b4b70fcb3efbabf9ab871542f8f696 - Build" in client.out
+
+        client.run("create . -pr=profile2")
+        # pkg gets a new package_id because it is a different revision
+        assert "dep/1.0#r2:da39a3ee5e6b4b0d3255bfef95601890afd80709 - System tool" in client.out
+        assert "pkg/1.0#27a56f09310cf1237629bae4104fe5bd:" \
+               "334882884da082740e5a002a0b6fdb509a280159 - Build" in client.out