New local-recipes-index feature (#13930)

* proposal for adding a conan-center-index clone as remote

* urls

* wip

* raise on uploads

* remove file:/// protocol, use explicit --type=local

* wip

* review

* fix boost

* wip

* add hook

* review

* fixes

* wip

* wip

* Update conan/internal/api/new/oss_recipe.py

Co-authored-by: Carlos Zoido <mrgalleta@gmail.com>

* Update conan/internal/api/new/oss_recipe.py

Co-authored-by: Carlos Zoido <mrgalleta@gmail.com>

* fix

* strip-root

* not warn remote-add

* test_package in template and remove clone in conan remote remove

* fix export patches

* cover patches export too

* fix remove

* refactor add/remove remote

* fix

* removed git type

* unused imports

* review and automatic deduction

* renaming all

---------

Co-authored-by: Carlos Zoido <mrgalleta@gmail.com>

conan/api/model.py

@@ -9,15 +9,19 @@
 from conans.util.files import load
 from conans.model.version_range import VersionRange
 
+LOCAL_RECIPES_INDEX = "local-recipes-index"
+
 
 class Remote:
 
-    def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None):
+    def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None,
+                 remote_type=None):
         self.name = name  # Read only, is the key
         self.url = url
         self.verify_ssl = verify_ssl
         self.disabled = disabled
         self.allowed_packages = allowed_packages
+        self.remote_type = remote_type
 
     def __eq__(self, other):
         if other is None:
@@ -29,6 +33,9 @@ def __str__(self):
         allowed_msg = ""
         if self.allowed_packages:
             allowed_msg = ", Allowed packages: {}".format(", ".join(self.allowed_packages))
+        if self.remote_type == LOCAL_RECIPES_INDEX:
+            return "{}: {} [{}, Enabled: {}{}]".format(self.name, self.url, LOCAL_RECIPES_INDEX,
+                                                       not self.disabled, allowed_msg)
         return "{}: {} [Verify SSL: {}, Enabled: {}{}]".format(self.name, self.url, self.verify_ssl,
                                                                not self.disabled, allowed_msg)
 

conan/api/subapi/new.py

@@ -28,6 +28,7 @@ def get_builtin_template(template_name):
         from conan.internal.api.new.bazel_exe import bazel_exe_files
         from conan.internal.api.new.autotools_lib import autotools_lib_files
         from conan.internal.api.new.autoools_exe import autotools_exe_files
+        from conan.internal.api.new.local_recipes_index import local_recipes_index_files
         new_templates = {"basic": basic_file,
                          "cmake_lib": cmake_lib_files,
                          "cmake_exe": cmake_exe_files,
@@ -39,7 +40,8 @@ def get_builtin_template(template_name):
                          "bazel_exe": bazel_exe_files,
                          "autotools_lib": autotools_lib_files,
                          "autotools_exe": autotools_exe_files,
-                         "alias": alias_file}
+                         "alias": alias_file,
+                         "local_recipes_index": local_recipes_index_files}
         template_files = new_templates.get(template_name)
         return template_files
 

conan/api/subapi/remotes.py

@@ -3,11 +3,12 @@
 import os
 from urllib.parse import urlparse
 
-from conan.api.model import Remote
+from conan.api.model import Remote, LOCAL_RECIPES_INDEX
 from conan.api.output import ConanOutput
 from conan.internal.cache.home_paths import HomePaths
 from conan.internal.conan_app import ConanApp
-
+from conans.client.rest_client_local_recipe_index import add_local_recipes_index_remote, \
+    remove_local_recipes_index_remote
 from conans.errors import ConanException
 from conans.util.files import save, load
 
@@ -99,13 +100,16 @@ def add(self, remote: Remote, force=False, index=None):
         """
         Add a new ``Remote`` object to the existing ones
 
+
         :param remote: a ``Remote`` object to be added
         :param force: do not fail if the remote already exist (but default it failes)
         :param index: if not defined, the new remote will be last one. Pass an integer to insert
           the remote in that position instead of the last one
         """
+        add_local_recipes_index_remote(self.conan_api, remote)
         remotes = _load(self._remotes_file)
-        _validate_url(remote.url)
+        if remote.remote_type != LOCAL_RECIPES_INDEX:
+            _validate_url(remote.url)
         current = {r.name: r for r in remotes}.get(remote.name)
         if current:  # same name remote existing!
             if not force:
@@ -139,6 +143,7 @@ def remove(self, pattern):
         _save(self._remotes_file, remotes)
         app = ConanApp(self.conan_api)
         for remote in removed:
+            remove_local_recipes_index_remote(self.conan_api, remote)
             app.cache.localdb.clean(remote_url=remote.url)
         return removed
 
@@ -160,7 +165,8 @@ def update(self, remote_name: str, url=None, secure=None, disabled=None, index=N
         except KeyError:
             raise ConanException(f"Remote '{remote_name}' doesn't exist")
         if url is not None:
-            _validate_url(url)
+            if remote.remote_type != LOCAL_RECIPES_INDEX:
+                _validate_url(url)
             _check_urls(remotes, url, force=False, current=remote)
             remote.url = url
         if secure is not None:
@@ -255,7 +261,7 @@ def _load(remotes_file):
     result = []
     for r in data.get("remotes", []):
         remote = Remote(r["name"], r["url"], r["verify_ssl"], r.get("disabled", False),
-                        r.get("allowed_packages"))
+                        r.get("allowed_packages"), r.get("remote_type"))
         result.append(remote)
     return result
 
@@ -268,6 +274,8 @@ def _save(remotes_file, remotes):
             remote["disabled"] = True
         if r.allowed_packages:
             remote["allowed_packages"] = r.allowed_packages
+        if r.remote_type:
+            remote["remote_type"] = r.remote_type
         remote_list.append(remote)
     save(remotes_file, json.dumps({"remotes": remote_list}, indent=True))
 

conan/cli/commands/new.py

@@ -18,7 +18,7 @@ def new(conan_api, parser, *args):
                         "either a predefined built-in or a user-provided one. "
                         "Available built-in templates: basic, cmake_lib, cmake_exe, "
                         "meson_lib, meson_exe, msbuild_lib, msbuild_exe, bazel_lib, bazel_exe, "
-                        "autotools_lib, autotools_exe. "
+                        "autotools_lib, autotools_exe, local_recipes_index"
                         "E.g. 'conan new cmake_lib -d name=hello -d version=0.1'. "
                         "You can define your own templates too by inputting an absolute path "
                         "as your template, or a path relative to your conan home folder."

conan/cli/commands/remote.py

@@ -1,14 +1,16 @@
 import json
+import os
 from collections import OrderedDict
 
-from conan.api.output import cli_out_write, Color
 from conan.api.conan_api import ConanAPI
-from conan.api.model import Remote
+from conan.api.model import Remote, LOCAL_RECIPES_INDEX
+from conan.api.output import cli_out_write, Color
+from conan.cli import make_abs_path
 from conan.cli.command import conan_command, conan_subcommand, OnceArgument
 from conan.cli.commands.list import remote_color, error_color, recipe_color, \
     reference_color
-from conans.client.rest.remote_credentials import RemoteCredentials
 from conan.errors import ConanException
+from conans.client.rest.remote_credentials import RemoteCredentials
 
 
 def formatter_remote_list_json(remotes):
@@ -74,10 +76,19 @@ def remote_add(conan_api, parser, subparser, *args):
     subparser.add_argument("-f", "--force", action='store_true',
                            help="Force the definition of the remote even if duplicated")
     subparser.add_argument("-ap", "--allowed-packages", action="append", default=None,
-                           help="Add recipe reference pattern to the list of allowed packages for this remote")
+                           help="Add recipe reference pattern to list of allowed packages for "
+                                "this remote")
+    subparser.add_argument("-t", "--type", choices=[LOCAL_RECIPES_INDEX],
+                           help="Define the remote type")
+
     subparser.set_defaults(secure=True)
     args = parser.parse_args(*args)
-    r = Remote(args.name, args.url, args.secure, disabled=False, allowed_packages=args.allowed_packages)
+
+    url_folder = make_abs_path(args.url)
+    remote_type = args.type or (LOCAL_RECIPES_INDEX if os.path.isdir(url_folder) else None)
+    url = url_folder if remote_type == LOCAL_RECIPES_INDEX else args.url
+    r = Remote(args.name, url, args.secure, disabled=False, remote_type=remote_type,
+               allowed_packages=args.allowed_packages)
     conan_api.remotes.add(r, force=args.force, index=args.index)
 
 

conan/internal/api/new/local_recipes_index.py

@@ -0,0 +1,113 @@
+from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2
+
+config_yml = """\
+versions:
+  "{{version}}":
+    folder: all
+"""
+
+conandata_yml = """\
+sources:
+  "{{version}}":
+    url:
+      {% if url is defined -%}
+      - "{{url}}"
+      {% else -%}
+      - "http://put/here/the/url/to/release.1.2.3.zip"
+      {% endif %}
+    {% if sha256 is defined -%}
+    sha256: "{{sha256}}"
+    {%- else -%}
+    sha256: "Put here your tarball sha256"
+    {% endif -%}
+"""
+
+
+conanfile = """\
+from conan import ConanFile
+from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps
+from conan.tools.files import apply_conandata_patches, export_conandata_patches, get
+
+
+class {{package_name}}Recipe(ConanFile):
+    name = "{{name}}"
+    package_type = "library"
+
+    # Optional metadata
+    license = "<Put the package license here>"
+    author = "<Put your name here> <And your email here>"
+    url = "<Package recipe repository url here, for issues about the package>"
+    description = "<Description of {{ name }} package here>"
+    topics = ("<Put some tag here>", "<here>", "<and here>")
+
+    # Binary configuration
+    settings = "os", "compiler", "build_type", "arch"
+    options = {"shared": [True, False], "fPIC": [True, False]}
+    default_options = {"shared": False, "fPIC": True}
+
+    def config_options(self):
+        if self.settings.os == "Windows":
+            self.options.rm_safe("fPIC")
+
+    def configure(self):
+        if self.options.shared:
+            self.options.rm_safe("fPIC")
+
+    def export_sources(self):
+        export_conandata_patches(self)
+
+    def source(self):
+        get(self, **self.conan_data["sources"][self.version], destination=self.source_folder,
+            strip_root=True)
+        apply_conandata_patches(self)
+
+    def layout(self):
+        cmake_layout(self, src_folder="src")
+
+    def generate(self):
+        deps = CMakeDeps(self)
+        deps.generate()
+        tc = CMakeToolchain(self)
+        tc.generate()
+
+    def build(self):
+        cmake = CMake(self)
+        cmake.configure()
+        cmake.build()
+
+    def package(self):
+        cmake = CMake(self)
+        cmake.install()
+
+    def package_info(self):
+        self.cpp_info.libs = ["{{name}}"]
+
+    {% if requires is defined -%}
+    def requirements(self):
+        {% for require in requires -%}
+        self.requires("{{ require }}")
+        {% endfor %}
+    {%- endif %}
+
+    {% if tool_requires is defined -%}
+    def build_requirements(self):
+        {% for require in tool_requires -%}
+        self.tool_requires("{{ require }}")
+        {% endfor %}
+    {%- endif %}
+"""
+
+
+test_main = """#include "{{name}}.h"
+
+int main() {
+    {{package_name}}();
+}
+"""
+
+local_recipes_index_files = {"recipes/{{name}}/config.yml": config_yml,
+                             "recipes/{{name}}/all/conandata.yml": conandata_yml,
+                             "recipes/{{name}}/all/conanfile.py": conanfile,
+                             "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2,
+                             "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2,
+                             "recipes/{{name}}/all/test_package/src/example.cpp": test_main}

conan/internal/cache/home_paths.py

@@ -12,6 +12,10 @@ class HomePaths:
     def __init__(self, home_folder):
         self._home = home_folder
 
+    @property
+    def local_recipes_index_path(self):
+        return os.path.join(self._home, ".local_recipes_index")
+
     @property
     def global_conf_path(self):
         return os.path.join(self._home, "global.conf")

conans/client/remote_manager.py

@@ -4,6 +4,8 @@
 
 from requests.exceptions import ConnectionError
 
+from conan.api.model import LOCAL_RECIPES_INDEX
+from conans.client.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex
 from conan.api.model import Remote
 from conan.api.output import ConanOutput
 from conan.internal.cache.conan_reference_layout import METADATA
@@ -18,14 +20,17 @@
 from conans.util.files import mkdir, tar_extract
 
 
-class RemoteManager(object):
+class RemoteManager:
     """ Will handle the remotes to get recipes, packages etc """
-
     def __init__(self, cache, auth_manager):
         self._cache = cache
         self._auth_manager = auth_manager
         self._signer = PkgSignaturesPlugin(cache)
 
+    def _local_folder_remote(self, remote):
+        if remote.remote_type == LOCAL_RECIPES_INDEX:
+            return RestApiClientLocalRecipesIndex(remote, self._cache)
+
     def check_credentials(self, remote):
         self._call_remote(remote, "check_credentials")
 
@@ -46,6 +51,12 @@ def get_recipe(self, ref, remote, metadata=None):
         layout = self._cache.get_or_create_ref_layout(ref)
         layout.export_remove()
 
+        export_folder = layout.export()
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            local_folder_remote.get_recipe(ref, export_folder)
+            return layout
+
         download_export = layout.download_export()
         try:
             zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata,
@@ -100,6 +111,11 @@ def get_recipe_sources(self, ref, layout, remote):
 
         download_folder = layout.download_export()
         export_sources_folder = layout.export_sources()
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            local_folder_remote.get_recipe_sources(ref, export_sources_folder)
+            return
+
         zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder)
         if not zipped_files:
             mkdir(export_sources_folder)  # create the folder even if no source files
@@ -234,6 +250,9 @@ def _call_remote(self, remote, method, *args, **kwargs):
         enforce_disabled = kwargs.pop("enforce_disabled", True)
         if remote.disabled and enforce_disabled:
             raise ConanException("Remote '%s' is disabled" % remote.name)
+        local_folder_remote = self._local_folder_remote(remote)
+        if local_folder_remote is not None:
+            return local_folder_remote.call_method(method, *args, **kwargs)
         try:
             return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs)
         except ConnectionError as exc: