diff --git a/conan/api/model.py b/conan/api/model.py index 70416a2c57b..f45a6f92221 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -9,15 +9,19 @@ from conans.util.files import load from conans.model.version_range import VersionRange +LOCAL_RECIPES_INDEX = "local-recipes-index" + class Remote: - def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None): + def __init__(self, name, url, verify_ssl=True, disabled=False, allowed_packages=None, + remote_type=None): self.name = name # Read only, is the key self.url = url self.verify_ssl = verify_ssl self.disabled = disabled self.allowed_packages = allowed_packages + self.remote_type = remote_type def __eq__(self, other): if other is None: @@ -29,6 +33,9 @@ def __str__(self): allowed_msg = "" if self.allowed_packages: allowed_msg = ", Allowed packages: {}".format(", ".join(self.allowed_packages)) + if self.remote_type == LOCAL_RECIPES_INDEX: + return "{}: {} [{}, Enabled: {}{}]".format(self.name, self.url, LOCAL_RECIPES_INDEX, + not self.disabled, allowed_msg) return "{}: {} [Verify SSL: {}, Enabled: {}{}]".format(self.name, self.url, self.verify_ssl, not self.disabled, allowed_msg) diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index d89280eec2f..573ce75d49d 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -28,6 +28,7 @@ def get_builtin_template(template_name): from conan.internal.api.new.bazel_exe import bazel_exe_files from conan.internal.api.new.autotools_lib import autotools_lib_files from conan.internal.api.new.autoools_exe import autotools_exe_files + from conan.internal.api.new.local_recipes_index import local_recipes_index_files new_templates = {"basic": basic_file, "cmake_lib": cmake_lib_files, "cmake_exe": cmake_exe_files, @@ -39,7 +40,8 @@ def get_builtin_template(template_name): "bazel_exe": bazel_exe_files, "autotools_lib": autotools_lib_files, "autotools_exe": autotools_exe_files, - "alias": alias_file} + "alias": alias_file, + "local_recipes_index": local_recipes_index_files} template_files = new_templates.get(template_name) return template_files diff --git a/conan/api/subapi/remotes.py b/conan/api/subapi/remotes.py index 83c25cc128b..74162fd914b 100644 --- a/conan/api/subapi/remotes.py +++ b/conan/api/subapi/remotes.py @@ -3,11 +3,12 @@ import os from urllib.parse import urlparse -from conan.api.model import Remote +from conan.api.model import Remote, LOCAL_RECIPES_INDEX from conan.api.output import ConanOutput from conan.internal.cache.home_paths import HomePaths from conan.internal.conan_app import ConanApp - +from conans.client.rest_client_local_recipe_index import add_local_recipes_index_remote, \ + remove_local_recipes_index_remote from conans.errors import ConanException from conans.util.files import save, load @@ -99,13 +100,16 @@ def add(self, remote: Remote, force=False, index=None): """ Add a new ``Remote`` object to the existing ones + :param remote: a ``Remote`` object to be added :param force: do not fail if the remote already exist (but default it failes) :param index: if not defined, the new remote will be last one. Pass an integer to insert the remote in that position instead of the last one """ + add_local_recipes_index_remote(self.conan_api, remote) remotes = _load(self._remotes_file) - _validate_url(remote.url) + if remote.remote_type != LOCAL_RECIPES_INDEX: + _validate_url(remote.url) current = {r.name: r for r in remotes}.get(remote.name) if current: # same name remote existing! if not force: @@ -139,6 +143,7 @@ def remove(self, pattern): _save(self._remotes_file, remotes) app = ConanApp(self.conan_api) for remote in removed: + remove_local_recipes_index_remote(self.conan_api, remote) app.cache.localdb.clean(remote_url=remote.url) return removed @@ -160,7 +165,8 @@ def update(self, remote_name: str, url=None, secure=None, disabled=None, index=N except KeyError: raise ConanException(f"Remote '{remote_name}' doesn't exist") if url is not None: - _validate_url(url) + if remote.remote_type != LOCAL_RECIPES_INDEX: + _validate_url(url) _check_urls(remotes, url, force=False, current=remote) remote.url = url if secure is not None: @@ -255,7 +261,7 @@ def _load(remotes_file): result = [] for r in data.get("remotes", []): remote = Remote(r["name"], r["url"], r["verify_ssl"], r.get("disabled", False), - r.get("allowed_packages")) + r.get("allowed_packages"), r.get("remote_type")) result.append(remote) return result @@ -268,6 +274,8 @@ def _save(remotes_file, remotes): remote["disabled"] = True if r.allowed_packages: remote["allowed_packages"] = r.allowed_packages + if r.remote_type: + remote["remote_type"] = r.remote_type remote_list.append(remote) save(remotes_file, json.dumps({"remotes": remote_list}, indent=True)) diff --git a/conan/cli/commands/new.py b/conan/cli/commands/new.py index f4274d1cc02..1e93fe7feb7 100644 --- a/conan/cli/commands/new.py +++ b/conan/cli/commands/new.py @@ -18,7 +18,7 @@ def new(conan_api, parser, *args): "either a predefined built-in or a user-provided one. " "Available built-in templates: basic, cmake_lib, cmake_exe, " "meson_lib, meson_exe, msbuild_lib, msbuild_exe, bazel_lib, bazel_exe, " - "autotools_lib, autotools_exe. " + "autotools_lib, autotools_exe, local_recipes_index" "E.g. 'conan new cmake_lib -d name=hello -d version=0.1'. " "You can define your own templates too by inputting an absolute path " "as your template, or a path relative to your conan home folder." diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index dee401ee0b2..8cd43711003 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -1,14 +1,16 @@ import json +import os from collections import OrderedDict -from conan.api.output import cli_out_write, Color from conan.api.conan_api import ConanAPI -from conan.api.model import Remote +from conan.api.model import Remote, LOCAL_RECIPES_INDEX +from conan.api.output import cli_out_write, Color +from conan.cli import make_abs_path from conan.cli.command import conan_command, conan_subcommand, OnceArgument from conan.cli.commands.list import remote_color, error_color, recipe_color, \ reference_color -from conans.client.rest.remote_credentials import RemoteCredentials from conan.errors import ConanException +from conans.client.rest.remote_credentials import RemoteCredentials def formatter_remote_list_json(remotes): @@ -74,10 +76,19 @@ def remote_add(conan_api, parser, subparser, *args): subparser.add_argument("-f", "--force", action='store_true', help="Force the definition of the remote even if duplicated") subparser.add_argument("-ap", "--allowed-packages", action="append", default=None, - help="Add recipe reference pattern to the list of allowed packages for this remote") + help="Add recipe reference pattern to list of allowed packages for " + "this remote") + subparser.add_argument("-t", "--type", choices=[LOCAL_RECIPES_INDEX], + help="Define the remote type") + subparser.set_defaults(secure=True) args = parser.parse_args(*args) - r = Remote(args.name, args.url, args.secure, disabled=False, allowed_packages=args.allowed_packages) + + url_folder = make_abs_path(args.url) + remote_type = args.type or (LOCAL_RECIPES_INDEX if os.path.isdir(url_folder) else None) + url = url_folder if remote_type == LOCAL_RECIPES_INDEX else args.url + r = Remote(args.name, url, args.secure, disabled=False, remote_type=remote_type, + allowed_packages=args.allowed_packages) conan_api.remotes.add(r, force=args.force, index=args.index) diff --git a/conan/internal/api/new/local_recipes_index.py b/conan/internal/api/new/local_recipes_index.py new file mode 100644 index 00000000000..cea8a9c8491 --- /dev/null +++ b/conan/internal/api/new/local_recipes_index.py @@ -0,0 +1,113 @@ +from conan.internal.api.new.cmake_lib import test_conanfile_v2, test_cmake_v2 + +config_yml = """\ +versions: + "{{version}}": + folder: all +""" + +conandata_yml = """\ +sources: + "{{version}}": + url: + {% if url is defined -%} + - "{{url}}" + {% else -%} + - "http://put/here/the/url/to/release.1.2.3.zip" + {% endif %} + {% if sha256 is defined -%} + sha256: "{{sha256}}" + {%- else -%} + sha256: "Put here your tarball sha256" + {% endif -%} +""" + + +conanfile = """\ +from conan import ConanFile +from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps +from conan.tools.files import apply_conandata_patches, export_conandata_patches, get + + +class {{package_name}}Recipe(ConanFile): + name = "{{name}}" + package_type = "library" + + # Optional metadata + license = "" + author = " " + url = "" + description = "" + topics = ("", "", "") + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def export_sources(self): + export_conandata_patches(self) + + def source(self): + get(self, **self.conan_data["sources"][self.version], destination=self.source_folder, + strip_root=True) + apply_conandata_patches(self) + + def layout(self): + cmake_layout(self, src_folder="src") + + def generate(self): + deps = CMakeDeps(self) + deps.generate() + tc = CMakeToolchain(self) + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = ["{{name}}"] + + {% if requires is defined -%} + def requirements(self): + {% for require in requires -%} + self.requires("{{ require }}") + {% endfor %} + {%- endif %} + + {% if tool_requires is defined -%} + def build_requirements(self): + {% for require in tool_requires -%} + self.tool_requires("{{ require }}") + {% endfor %} + {%- endif %} +""" + + +test_main = """#include "{{name}}.h" + +int main() { + {{package_name}}(); +} +""" + +local_recipes_index_files = {"recipes/{{name}}/config.yml": config_yml, + "recipes/{{name}}/all/conandata.yml": conandata_yml, + "recipes/{{name}}/all/conanfile.py": conanfile, + "recipes/{{name}}/all/test_package/conanfile.py": test_conanfile_v2, + "recipes/{{name}}/all/test_package/CMakeLists.txt": test_cmake_v2, + "recipes/{{name}}/all/test_package/src/example.cpp": test_main} diff --git a/conan/internal/cache/home_paths.py b/conan/internal/cache/home_paths.py index 9b907c7ac90..ba2402ded2a 100644 --- a/conan/internal/cache/home_paths.py +++ b/conan/internal/cache/home_paths.py @@ -12,6 +12,10 @@ class HomePaths: def __init__(self, home_folder): self._home = home_folder + @property + def local_recipes_index_path(self): + return os.path.join(self._home, ".local_recipes_index") + @property def global_conf_path(self): return os.path.join(self._home, "global.conf") diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 226df269537..4feb4a094ad 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -4,6 +4,8 @@ from requests.exceptions import ConnectionError +from conan.api.model import LOCAL_RECIPES_INDEX +from conans.client.rest_client_local_recipe_index import RestApiClientLocalRecipesIndex from conan.api.model import Remote from conan.api.output import ConanOutput from conan.internal.cache.conan_reference_layout import METADATA @@ -18,14 +20,17 @@ from conans.util.files import mkdir, tar_extract -class RemoteManager(object): +class RemoteManager: """ Will handle the remotes to get recipes, packages etc """ - def __init__(self, cache, auth_manager): self._cache = cache self._auth_manager = auth_manager self._signer = PkgSignaturesPlugin(cache) + def _local_folder_remote(self, remote): + if remote.remote_type == LOCAL_RECIPES_INDEX: + return RestApiClientLocalRecipesIndex(remote, self._cache) + def check_credentials(self, remote): self._call_remote(remote, "check_credentials") @@ -46,6 +51,12 @@ def get_recipe(self, ref, remote, metadata=None): layout = self._cache.get_or_create_ref_layout(ref) layout.export_remove() + export_folder = layout.export() + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + local_folder_remote.get_recipe(ref, export_folder) + return layout + download_export = layout.download_export() try: zipped_files = self._call_remote(remote, "get_recipe", ref, download_export, metadata, @@ -100,6 +111,11 @@ def get_recipe_sources(self, ref, layout, remote): download_folder = layout.download_export() export_sources_folder = layout.export_sources() + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + local_folder_remote.get_recipe_sources(ref, export_sources_folder) + return + zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder) if not zipped_files: mkdir(export_sources_folder) # create the folder even if no source files @@ -234,6 +250,9 @@ def _call_remote(self, remote, method, *args, **kwargs): enforce_disabled = kwargs.pop("enforce_disabled", True) if remote.disabled and enforce_disabled: raise ConanException("Remote '%s' is disabled" % remote.name) + local_folder_remote = self._local_folder_remote(remote) + if local_folder_remote is not None: + return local_folder_remote.call_method(method, *args, **kwargs) try: return self._auth_manager.call_rest_api_method(remote, method, *args, **kwargs) except ConnectionError as exc: diff --git a/conans/client/rest_client_local_recipe_index.py b/conans/client/rest_client_local_recipe_index.py new file mode 100644 index 00000000000..a8a2ecb0cc8 --- /dev/null +++ b/conans/client/rest_client_local_recipe_index.py @@ -0,0 +1,228 @@ +import os +import sys +import textwrap +from distutils.dir_util import copy_tree +from fnmatch import fnmatch +from io import StringIO + +import yaml + +from conan.api.model import LOCAL_RECIPES_INDEX +from conan.api.output import ConanOutput +from conan.internal.cache.home_paths import HomePaths +from conans.client.cmd.export import cmd_export +from conans.errors import ConanException, PackageNotFoundException, RecipeNotFoundException +from conans.model.conf import ConfDefinition +from conans.model.recipe_ref import RecipeReference +from conans.util.files import load, save, rmdir + + +def add_local_recipes_index_remote(conan_api, remote): + if remote.remote_type != LOCAL_RECIPES_INDEX: + return + local_recipes_index_path = HomePaths(conan_api.cache_folder).local_recipes_index_path + repo_folder = os.path.join(local_recipes_index_path, remote.name) + + output = ConanOutput() + if os.path.exists(repo_folder): + output.warning(f"The cache folder for remote {remote.name} existed, removing it") + rmdir(repo_folder) + + cache_path = os.path.join(repo_folder, ".conan") + hook_folder = HomePaths(cache_path).hooks_path + trim_hook = os.path.join(hook_folder, "hook_trim_conandata.py") + hook_content = textwrap.dedent("""\ + from conan.tools.files import trim_conandata + def post_export(conanfile): + if conanfile.conan_data: + trim_conandata(conanfile) + """) + save(trim_hook, hook_content) + + +def remove_local_recipes_index_remote(conan_api, remote): + if remote.remote_type == LOCAL_RECIPES_INDEX: + local_recipes_index_path = HomePaths(conan_api.cache_folder).local_recipes_index_path + local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name) + ConanOutput().info(f"Removing temporary files for '{remote.name}' " + f"local-recipes-index remote") + rmdir(local_recipes_index_path) + + +class RestApiClientLocalRecipesIndex: + """ + Implements the RestAPI but instead of over HTTP for a remote server, using just + a local folder assuming the conan-center-index repo layout + """ + + def __init__(self, remote, cache): + self._remote = remote + local_recipes_index_path = HomePaths(cache.cache_folder).local_recipes_index_path + local_recipes_index_path = os.path.join(local_recipes_index_path, remote.name) + local_recipes_index_path = os.path.join(local_recipes_index_path, ".conan") + repo_folder = self._remote.url + + from conan.internal.conan_app import ConanApp + from conan.api.conan_api import ConanAPI + conan_api = ConanAPI(local_recipes_index_path) + self._app = ConanApp(conan_api) + self._layout = _LocalRecipesIndexLayout(repo_folder) + + def call_method(self, method_name, *args, **kwargs): + return getattr(self, method_name)(*args, **kwargs) + + def get_recipe(self, ref, dest_folder): + export_folder = self._app.cache.recipe_layout(ref).export() + return self._copy_files(export_folder, dest_folder) + + def get_recipe_sources(self, ref, dest_folder): + export_sources = self._app.cache.recipe_layout(ref).export_sources() + return self._copy_files(export_sources, dest_folder) + + def get_package(self, pref, dest_folder, metadata, only_metadata): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support " + "binary packages") + + def upload_recipe(self, ref, files_to_upload): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") + + def upload_package(self, pref, files_to_upload): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") + + def authenticate(self, user, password): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support " + "authentication") + + def check_credentials(self): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support upload") + + def search(self, pattern=None): + return self._layout.get_recipes_references(pattern) + + def search_packages(self, reference): + assert self and reference + return {} + + def remove_recipe(self, ref): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") + + def remove_all_packages(self, ref): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") + + def remove_packages(self, prefs): + raise ConanException(f"Remote local-recipes-index '{self._remote.name}' doesn't support remove") + + def get_recipe_revisions_references(self, ref): + ref = self._export_recipe(ref) + return [ref] + + def get_package_revisions_references(self, pref, headers=None): + raise PackageNotFoundException(pref) + + def get_latest_recipe_reference(self, ref): + ref = self._export_recipe(ref) + return ref + + def get_latest_package_reference(self, pref, headers): + raise PackageNotFoundException(pref) + + def get_recipe_revision_reference(self, ref): + new_ref = self._export_recipe(ref) + if new_ref != ref: + raise RecipeNotFoundException(ref) + return new_ref + + def get_package_revision_reference(self, pref): + raise PackageNotFoundException(pref) + + # Helper methods to implement the interface + def _export_recipe(self, ref): + folder = self._layout.get_recipe_folder(ref) + conanfile_path = os.path.join(folder, "conanfile.py") + original_stderr = sys.stderr + sys.stderr = StringIO() + try: + global_conf = ConfDefinition() + new_ref, _ = cmd_export(self._app, global_conf, conanfile_path, + ref.name, str(ref.version), None, None) + except Exception as e: + raise ConanException(f"Error while exporting recipe from remote: {self._remote.name}\n" + f"{str(e)}") + finally: + export_stderr = sys.stderr.getvalue() + sys.stderr = original_stderr + ConanOutput().debug(f"Internal export for {ref}:\n" + f"{textwrap.indent(export_stderr, ' ')}") + + return new_ref + + @staticmethod + def _copy_files(source_folder, dest_folder): + if not os.path.exists(source_folder): + return {} + copy_tree(source_folder, dest_folder) + ret = {} + for root, _, _files in os.walk(dest_folder): + for _f in _files: + rel = os.path.relpath(os.path.join(root, _f), dest_folder) + ret[rel] = os.path.join(dest_folder, root, _f) + return ret + + +class _LocalRecipesIndexLayout: + + def __init__(self, base_folder): + self._base_folder = base_folder + + def _get_base_folder(self, recipe_name): + return os.path.join(self._base_folder, "recipes", recipe_name) + + @staticmethod + def _load_config_yml(folder): + config = os.path.join(folder, "config.yml") + if not os.path.isfile(config): + return None + return yaml.safe_load(load(config)) + + def get_recipes_references(self, pattern): + name_pattern = pattern.split("/", 1)[0] + recipes_dir = os.path.join(self._base_folder, "recipes") + recipes = os.listdir(recipes_dir) + recipes.sort() + ret = [] + excluded = set() + for r in recipes: + if not fnmatch(r, name_pattern): + continue + folder = self._get_base_folder(r) + config_yml = self._load_config_yml(folder) + if config_yml is None: + raise ConanException(f"Corrupted repo, folder {r} without 'config.yml'") + versions = config_yml["versions"] + for v in versions: + # TODO: Check the search pattern is the same as remotes and cache + ref = f"{r}/{v}" + if not fnmatch(ref, pattern): + continue + subfolder = versions[v]["folder"] + # This check can be removed after compatibility with 2.0 + conanfile = os.path.join(recipes_dir, r, subfolder, "conanfile.py") + conanfile_content = load(conanfile) + if "from conans" in conanfile_content or "import conans" in conanfile_content: + excluded.add(r) + continue + ret.append(RecipeReference.loads(ref)) + if excluded: + ConanOutput().warning(f"Excluding recipes not Conan 2.0 ready: {', '.join(excluded)}") + return ret + + def get_recipe_folder(self, ref): + folder = self._get_base_folder(ref.name) + data = self._load_config_yml(folder) + if data is None: + raise RecipeNotFoundException(ref) + versions = data["versions"] + if str(ref.version) not in versions: + raise RecipeNotFoundException(ref) + subfolder = versions[str(ref.version)]["folder"] + return os.path.join(folder, subfolder) diff --git a/conans/test/functional/test_local_recipes_index.py b/conans/test/functional/test_local_recipes_index.py new file mode 100644 index 00000000000..5c17dac6ed6 --- /dev/null +++ b/conans/test/functional/test_local_recipes_index.py @@ -0,0 +1,120 @@ +import os +import textwrap + +from conans.test.assets.cmake import gen_cmakelists +from conans.test.assets.sources import gen_function_cpp, gen_function_h +from conans.test.utils.file_server import TestFileServer +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient, zipdir +from conans.util.files import save_files, sha256sum + + +class TestLocalRecipeIndexNew: + def test_conan_new_local_recipes_index(self): + # Setup the release pkg0.1.zip http server + file_server = TestFileServer() + zippath = os.path.join(file_server.store, "pkg0.1.zip") + repo_folder = temp_folder() + cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, + public_header="pkg.h") + save_files(repo_folder, {"pkg/CMakeLists.txt": cmake, + "pkg/pkg.h": gen_function_h(name="pkg"), + "pkg/pkg.cpp": gen_function_cpp(name="pkg")}) + zipdir(repo_folder, zippath) + sha256 = sha256sum(zippath) + url = f"{file_server.fake_url}/pkg0.1.zip" + + c0 = TestClient() + c0.servers["file_server"] = file_server + c0.run(f"new local_recipes_index -d name=pkg -d version=0.1 -d url={url} -d sha256={sha256}") + # A local create is possible, and it includes a test_package + c0.run("create recipes/pkg/all --version=0.1") + assert "pkg: Release!" in c0.out + remote_folder = c0.current_folder + + c = TestClient() + c.servers["file_server"] = file_server + c.run(f"remote add local '{remote_folder}'") + c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") + c.run("create . --version=0.1 --build=missing") + assert "pkg: Release!" in c.out + + +class TestInRepo: + def test_in_repo(self): + """testing that it is possible to put a "recipes" folder inside a source repo, and + use it as local-recipes-index repository, exporting the source from itself + """ + repo_folder = temp_folder() + cmake = gen_cmakelists(libname="pkg", libsources=["pkg.cpp"], install=True, + public_header="pkg.h") + config_yml = textwrap.dedent("""\ + versions: + "0.1": + folder: all + """) + conanfile = textwrap.dedent("""\ + import os + from conan import ConanFile + from conan.tools.cmake import CMake, cmake_layout + from conan.tools.files import copy + + class PkgRecipe(ConanFile): + name = "pkg" + package_type = "library" + + # Binary configuration + settings = "os", "compiler", "build_type", "arch" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + + generators = "CMakeToolchain" + + def export_sources(self): + src = os.path.dirname(os.path.dirname(os.path.dirname(self.recipe_folder))) + copy(self, "*", src=src, dst=self.export_sources_folder, excludes=["recipes*"]) + + def config_options(self): + if self.settings.os == "Windows": + self.options.rm_safe("fPIC") + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self) + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = [self.name] + """) + + save_files(repo_folder, {"recipes/pkg/config.yml": config_yml, + "recipes/pkg/all/conanfile.py": conanfile, + "CMakeLists.txt": cmake, + "pkg.h": gen_function_h(name="pkg"), + "pkg.cpp": gen_function_cpp(name="pkg")}) + + c = TestClient() + c.run(f"remote add local '{repo_folder}'") + c.run("new cmake_exe -d name=app -d version=0.1 -d requires=pkg/0.1") + c.run("create . --build=missing") + assert "pkg: Release!" in c.out + + # Of course the recipe can also be created locally + path = os.path.join(repo_folder, "recipes/pkg/all") + c.run(f'create "{path}" --version=0.1') + assert "pkg/0.1: Created package" in c.out + + # Finally lets remove the remote, check that the clone is cleared + c.run('remote remove local') + assert "Removing temporary files for 'local' local-recipes-index remote" in c.out diff --git a/conans/test/integration/remote/test_local_recipes_index.py b/conans/test/integration/remote/test_local_recipes_index.py new file mode 100644 index 00000000000..b53465f7028 --- /dev/null +++ b/conans/test/integration/remote/test_local_recipes_index.py @@ -0,0 +1,306 @@ +import json +import os +import textwrap + +import pytest + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient +from conans.util.files import mkdir, save, save_files + + +@pytest.fixture(scope="module") +def c3i_folder(): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "1.2.8": + folder: all + "1.2.11": + folder: all + """) + zlib = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import load + class Zlib(ConanFile): + name = "zlib" + exports_sources = "*" + def build(self): + self.output.info(f"CONANDATA: {self.conan_data}") + self.output.info(f"BUILDING: {load(self, 'file.h')}") + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib, + "zlib/all/conandata.yml": "", + "zlib/all/file.h": "//myheader"}) + mkdir(os.path.join(recipes_folder, "openssl", "1.X")) + mkdir(os.path.join(recipes_folder, "openssl", "2.X")) + save(os.path.join(recipes_folder, "openssl", "config.yml"), textwrap.dedent(""" + versions: + "1.0": + folder: "1.X" + "1.1": + folder: "1.X" + "2.0": + folder: "2.X" + """)) + save(os.path.join(recipes_folder, "openssl", "1.X", "conanfile.py"), + str(GenConanfile().with_require("zlib/1.2.8"))) + save(os.path.join(recipes_folder, "openssl", "2.X", "conanfile.py"), + str(GenConanfile().with_require("zlib/1.2.11"))) + mkdir(os.path.join(recipes_folder, "libcurl", "all")) + save(os.path.join(recipes_folder, "libcurl", "config.yml"), textwrap.dedent(""" + versions: + "1.0": + folder: "all" + """)) + save(os.path.join(recipes_folder, "libcurl", "all", "conanfile.py"), + str(GenConanfile().with_require("openssl/2.0"))) + return folder + + +class TestSearchList: + def test_basic_search(self, c3i_folder): + client = TestClient() + client.run(f"remote add local '{c3i_folder}' --type=local-recipes-index") # Keep --type test + assert "WARN" not in client.out # Make sure it does not complain about url + client.run("search *") + assert textwrap.dedent("""\ + local + libcurl + libcurl/1.0 + openssl + openssl/1.0 + openssl/1.1 + openssl/2.0 + zlib + zlib/1.2.8 + zlib/1.2.11 + """) in client.out + + def test_list_refs(self, c3i_folder): + client = TestClient() + client.run(f"remote add local '{c3i_folder}'") + client.run("list *#* -r=local --format=json") + listjson = json.loads(client.stdout) + revs = listjson["local"]["libcurl/1.0"]["revisions"] + assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs + revs = listjson["local"]["openssl/1.0"]["revisions"] + assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs + revs = listjson["local"]["openssl/1.1"]["revisions"] + assert len(revs) == 1 and "b35ffb31b6d5a9d8af39f5de3cf4fd63" in revs + revs = listjson["local"]["openssl/2.0"]["revisions"] + assert len(revs) == 1 and "e50e871efca149f160fa6354c8534449" in revs + revs = listjson["local"]["zlib/1.2.8"]["revisions"] + assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs + revs = listjson["local"]["zlib/1.2.11"]["revisions"] + assert len(revs) == 1 and "6f5c31bb1219e9393743d1fbf2ee1b52" in revs + + def test_list_rrevs(self, c3i_folder): + client = TestClient() + client.run(f"remote add local '{c3i_folder}'") + client.run("list libcurl/1.0#* -r=local --format=json") + listjson = json.loads(client.stdout) + revs = listjson["local"]["libcurl/1.0"]["revisions"] + assert len(revs) == 1 and "e468388f0e4e098d5b62ad68979aebd5" in revs + + def test_list_binaries(self, c3i_folder): + client = TestClient() + client.run(f"remote add local '{c3i_folder}'") + client.run("list libcurl/1.0:* -r=local --format=json") + listjson = json.loads(client.stdout) + rev = listjson["local"]["libcurl/1.0"]["revisions"]["e468388f0e4e098d5b62ad68979aebd5"] + assert rev["packages"] == {} + + +class TestInstall: + def test_install(self, c3i_folder): + c = TestClient() + c.run(f"remote add local '{c3i_folder}'") + c.run("install --requires=libcurl/1.0 --build missing") + assert "zlib/1.2.11: CONANDATA: {}" in c.out + assert "zlib/1.2.11: BUILDING: //myheader" in c.out + bins = {"libcurl/1.0": ("aa69c1e1e39a18fe70001688213dbb7ada95f890", "Build"), + "openssl/2.0": ("594ed0eb2e9dfcc60607438924c35871514e6c2a", "Build"), + "zlib/1.2.11": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")} + c.assert_listed_binary(bins) + + # Already installed in the cache + c.run("install --requires=libcurl/1.0") + assert "zlib/1.2.11: Already installed!" in c.out + assert "openssl/2.0: Already installed!" in c.out + assert "libcurl/1.0: Already installed!" in c.out + + # Update doesn't fail, but doesn't update revision time + c.run("install --requires libcurl/1.0 --update") + bins = {"libcurl/1.0": "Cache (Updated date) (local)", + "openssl/2.0": "Cache (Updated date) (local)", + "zlib/1.2.11": "Cache (Updated date) (local)"} + + c.assert_listed_require(bins) + assert "zlib/1.2.11: Already installed!" in c.out + assert "openssl/2.0: Already installed!" in c.out + assert "libcurl/1.0: Already installed!" in c.out + + # Doing local changes creates a new revision + # New recipe revision for the zlib library + save(os.path.join(c3i_folder, "recipes", "zlib", "all", "conanfile.py"), + str(GenConanfile()) + "\n") + c.run("install --requires=libcurl/1.0 --build missing --update") + # it is updated + assert "zlib/1.2.11#dd82451a95902c89bb66a2b980c72de5 - Updated (local)" in c.out + + def test_install_with_exported_files(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + boost_config = textwrap.dedent(""" + versions: + "1.0": + folder: all + """) + boost = textwrap.dedent(""" + import os + from conan.tools.files import load + from conan import ConanFile + class Boost(ConanFile): + name = "boost" + version = "1.0" + exports = "*" + def source(self): + myfile = os.path.join(self.recipe_folder, "dependencies", "myfile.json") + self.output.info(load(self, myfile)) + """) + deps_json = '{"potato": 42}' + save_files(recipes_folder, + {"boost/config.yml": boost_config, + "boost/all/conanfile.py": boost, + "boost/all/dependencies/myfile.json": deps_json}) + c = TestClient() + c.run(f"remote add local '{folder}'") + c.run("install --requires=boost/[*] --build missing") + assert 'boost/1.0: {"potato": 42}' in c.out + + def test_trim_conandata_yaml(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + config = textwrap.dedent(""" + versions: + "1.0": + folder: all + """) + conandata = textwrap.dedent("""\ + sources: + "1.0": + url: + sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + patches: + "1.0": + - patch_file: "patches/1.3/0001-fix-cmake.patch" + """) + save_files(recipes_folder, + {"pkg/config.yml": config, + "pkg/all/conanfile.py": str(GenConanfile("pkg")), + "pkg/all/conandata.yml": conandata}) + c = TestClient() + c.run(f"remote add local '{folder}'") + c.run("install --requires=pkg/1.0 --build missing -vvv") + assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out + + # User modifies conandata.yml to add new version + new_conandata = textwrap.dedent("""\ + sources: + "1.0": + url: + sha256: "ff0ba4c292013dbc27530b3a81e1f9a813cd39de01ca5e0f8bf355702efa593e" + "1.1": + url: + patches: + "1.0": + - patch_file: "patches/1.3/0001-fix-cmake.patch" + """) + save_files(recipes_folder, {"pkg/all/conandata.yml": new_conandata}) + c.run("install --requires=pkg/1.0 --build missing --update -vvv") + assert "pkg/1.0#86b609916bbdfe63c579f034ad0edfe7" in c.out + + def test_export_patches(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "0.1": + folder: all + """) + zlib = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import export_conandata_patches, apply_conandata_patches + class Zlib(ConanFile): + name = "zlib" + exports_sources = "*.cpp" + def export_sources(self): + export_conandata_patches(self) + + def source(self): + apply_conandata_patches(self) + """) + conandata_yml = textwrap.dedent("""\ + versions: + "0.1": + patches: + "0.1": + - patch_file: "patches/patch1" + """) + patch = textwrap.dedent("""\ + --- a/main.cpp + +++ b/main.cpp + @@ -0,0 +1 @@ + +hello + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib, + "zlib/all/conandata.yml": conandata_yml, + "zlib/all/patches/patch1": patch, + "zlib/all/main.cpp": "\n"}) + client = TestClient() + client.run(f"remote add local '{folder}'") + client.run("install --requires=zlib/0.1 --build=missing -vv") + assert "zlib/0.1: Copied 1 file: patch1" in client.out + assert "zlib/0.1: Apply patch (file): patches/patch1" in client.out + + +class TestRestrictedOperations: + def test_upload(self): + folder = temp_folder() + c3i_folder = os.path.join(folder, "recipes") + mkdir(c3i_folder) + c = TestClient() + c.run(f"remote add local '{c3i_folder}'") + c.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + c.run("create .") + c.run("upload pkg/0.1 -r=local", assert_error=True) + assert "ERROR: Remote local-recipes-index 'local' doesn't support upload" in c.out + + +class TestErrorsUx: + def test_errors(self): + folder = temp_folder() + recipes_folder = os.path.join(folder, "recipes") + zlib_config = textwrap.dedent(""" + versions: + "1.2.11": + folder: all + """) + zlib = textwrap.dedent(""" + class Zlib(ConanFile): + name = "zlib" + """) + save_files(recipes_folder, + {"zlib/config.yml": zlib_config, + "zlib/all/conanfile.py": zlib}) + c = TestClient() + c.run(f"remote add local '{folder}'") + c.run("install --requires=zlib/[*] --build missing", assert_error=True) + assert "NameError: name 'ConanFile' is not defined" in c.out