Feature: link editable packages (conan-io#4181)

* first try

* fix includes

* minor changes

* more import fixes

* fix more imports

* short path as decorator

* move SimplePaths to conans/paths/simple_paths.py

* move back constants to conans.paths

* moving folders to conans.paths

* remove diffs caused just by ordering of imports

* remove diffs caused just by ordering of imports

* no diffs

* no diff

* move is_case_insensitive_os to conans.paths

* add rm_conandir to windows imports

* move package_layouts to paths

* fix imports

* move LINKED_FOLDER_SENTINEL to conans.paths.__init__.py

* move 'get_package_layout' to 'SimplePaths'

* get rid of PackageBaseLayout, go for duck typing

* add parser for the file that will contain paths to includes, res, bins, libs,...

* rename PackageUserLayout > PackageEditableLayout

* interface for installed_as_editable info

* test with a header only editable package

* working example for header only lib

* add test for a library with settings and options (still header only)

* fix py2 test (declare string in source as unicode)

* change import to avoid conans.tools one, so we need to mock a different function

* make it compatible for win/linux

* create the editable file sentinel using --editable argument in install command

* add needed functions to simple_paths

* add helper functions to simple paths

* add package_metadata to new layouts

* add missing named var (something went wrong merging from develop)

* remove lines, not anymore there

* add package metadata to users one

* handle error: package folders needs a ConanFileReference

* reorder imports

* add missing import

* reorder imports

* if an editable package is dirty, log an error

* moved parser to conan_file

* move parsing for conan_package_layout closer to the ConanFile, we will be able to change it in the future

* placeholders should be preprended with settings|options

* fix typos, remove unused code

* fix bug/typo

* order imports

* remove unreachable code

* remove unreachable code

* remove comments

* move unittests to its folder

* install as editable (requires the package to be already in the cache)

* declare tests

* refactor the editable hack: rely on member attribute

* move imports

* rename file to match class name

* notify user about editable condition

* gather logic together

* make it editable from the very beginning, we don't care if it is editable or not

* any error will require us to delete the editable flag

* add tests

* add tests related to editable package removal

* rename tests

* add test for other commands

* add tests for conan commands

* Update conans/test/command/editable_reference/create_editable_package_test.py

Co-Authored-By: jgsogo <jgsogo@gmail.com>

* add changes from review

* move tests to final location

* remove unused var

* change variable name (it was a py conanfile)

* don0t care about the output, but check something

* remove comments

* add tests installing an editable package (and installing its dependencies)

* remove unused import

* move cpp_info overrides to an standalone model

* minor changes related to reviews

* mode create/remove editables to client_cache class

* renamed test folder

* remove code related to 'install --editable'

* link/unlink editable packages

* consider these tests

* handle remove command on editable

* adapt message

* removal is on its own test

* check after unlink

* add tests using linked packages

* add tests for several commands

* fix test related to utf8 and ordering

* apply ordering

* remove prints

* putting path between quotes seems to solve windows issue

* review (add more checks in tests)

* function changed its name

* no import in __init__.py

* remove installed as editable

* order is not guaranteed on info command output

* no need for os.path.join

* move EDITABLE before locking package layout

* no different behaviour for update

* new model for editable_cpp_info

* check that cache file has more priority than the repo one

* review

* remove test, it is accepted behaviour now

* review

* revert change, this line is needed to pass tests in Windows

* weird substitution is not needed, but path normalization is

* inform about link removal

* use conditional 'strict' arg when reusing cache for several TestClients

* pass string to conan_api, build ConanFileReference inside it

* add tests case for editable package without files, it will use package_info data

* remove unused import

* remove uneeded command

* get the path to the conanfile.py from the argument to 'link' command inside the 'conan_api'

* remove 'EditableCppInfo.create', rename loader ones to 'load(file...)' and 'loads(teext...)'

* rename func argument

* move CONAN_PACKAGE_LAYOUT_FILE to conans/paths

* reorder variables inside file

* renamed variables

* pep8

* not using revisions in editable nodes, fix ordering issue

* rename variables

conans/client/client_cache.py

@@ -16,7 +16,8 @@
 from conans.model.profile import Profile
 from conans.model.ref import ConanFileReference
 from conans.model.settings import Settings
-from conans.paths import CONAN_MANIFEST, PUT_HEADERS, SimplePaths, check_ref_case
+from conans.paths import CONAN_MANIFEST, PUT_HEADERS
+from conans.paths.simple_paths import SimplePaths
 from conans.unicode import get_cwd
 from conans.util.files import list_folder_subdirs, load, normalize, save
 from conans.util.locks import Lock, NoLock, ReadLock, SimpleLock, WriteLock
@@ -28,6 +29,9 @@
 REGISTRY_JSON = "registry.json"
 PROFILES_FOLDER = "profiles"
 HOOKS_FOLDER = "hooks"
+LAYOUTS_FOLDER = 'layouts'
+
+DEFAULT_LAYOUT_FILE = "default"
 
 # Client certificates
 CLIENT_CERT = "client.crt"
@@ -161,6 +165,10 @@ def default_profile_path(self):
             return join(self.conan_folder, PROFILES_FOLDER,
                         self.conan_config.default_profile)
 
+    @property
+    def default_editable_path(self):
+        return os.path.join(self.conan_folder, LAYOUTS_FOLDER, DEFAULT_LAYOUT_FILE)
+
     @property
     def hooks_path(self):
         """
@@ -246,7 +254,6 @@ def load_manifest(self, conan_reference):
         """conan_id = sha(zip file)"""
         assert isinstance(conan_reference, ConanFileReference)
         export_folder = self.export(conan_reference)
-        check_ref_case(conan_reference, export_folder, self.store)
         return FileTreeManifest.load(export_folder)
 
     def load_package_manifest(self, package_reference):
@@ -313,6 +320,17 @@ def package_summary_hash(self, package_ref):
         readed_digest = FileTreeManifest.load(package_folder)
         return readed_digest.summary_hash
 
+    def install_as_editable(self, conan_reference, target_path):
+        linked_folder_sentinel = self._build_path_to_linked_folder_sentinel(conan_reference)
+        save(linked_folder_sentinel, content=target_path)
+
+    def remove_editable(self, conan_reference):
+        if self.installed_as_editable(conan_reference):
+            linked_folder_sentinel = self._build_path_to_linked_folder_sentinel(conan_reference)
+            os.remove(linked_folder_sentinel)
+            return True
+        return False
+
 
 def _mix_settings_with_env(settings):
     """Reads CONAN_ENV_XXXX variables from environment

conans/client/command.py

@@ -11,15 +11,16 @@
 from conans.client.cmd.uploader import UPLOAD_POLICY_FORCE, \
     UPLOAD_POLICY_NO_OVERWRITE, UPLOAD_POLICY_NO_OVERWRITE_RECIPE, UPLOAD_POLICY_SKIP
 from conans.client.conan_api import (Conan, default_manifest_folder)
+from conans.client.conan_api import _get_conanfile_path
 from conans.client.conan_command_output import CommandOutputer
 from conans.client.output import Color
 from conans.client.printer import Printer
-from conans.client.tools.files import save
 from conans.errors import ConanException, ConanInvalidConfiguration, NoRemoteAvailable
 from conans.model.ref import ConanFileReference
 from conans.unicode import get_cwd
 from conans.util.config_parser import get_bool_from_text
 from conans.util.files import exception_message_safe
+from conans.util.files import save
 from conans.util.log import logger
 
 # Exit codes for conan command:
@@ -1332,12 +1333,46 @@ def alias(self, *args):
 
         self._conan.export_alias(args.reference, args.target)
 
+    def link(self, *args):
+        """ Links a conan reference (e.g lib/1.0@conan/stable) with a local folder path.
+
+        """
+        parser = argparse.ArgumentParser(description=self.link.__doc__,
+                                         prog="conan link")
+        parser.add_argument('target', help='Path to the package folder in the user workspace',
+                            nargs='?',)
+        parser.add_argument('reference', help='Reference to link. e.g.: mylib/1.X@user/channel')
+        parser.add_argument("--remove", action='store_true', default=False,
+                            help='Remove linked reference (target not required)')
+
+        args = parser.parse_args(*args)
+        self._warn_python2()
+
+        # Args sanity check
+        if args.remove and args.target:
+            raise ConanException("Do not provide the 'target' argument for removal")
+
+        if not args.remove and not args.target:
+            raise ConanException("Argument 'target' is required to link a reference")
+
+        if not args.remove:
+            self._conan.link(args.target, args.reference, cwd=os.getcwd())
+            self._outputer.writeln("Reference '{}' linked to directory "
+                                   "'{}'".format(args.reference, os.path.dirname(args.target)))
+        else:
+            ret = self._conan.unlink(args.reference)
+            if ret:
+                self._outputer.writeln("Removed linkage for reference '{}'".format(args.reference))
+            else:
+                self._user_io.out.warn("Reference '{}' was not installed "
+                                       "as editable".format(args.reference))
+
     def _show_help(self):
         """Prints a summary of all commands
         """
         grps = [("Consumer commands", ("install", "config", "get", "info", "search")),
                 ("Creator commands", ("new", "create", "upload", "export", "export-pkg", "test")),
-                ("Package development commands", ("source", "build", "package")),
+                ("Package development commands", ("source", "build", "package", "link")),
                 ("Misc commands", ("profile", "remote", "user", "imports", "copy", "remove",
                                    "alias", "download", "inspect", "help"))]
 

conans/client/conan_api.py

@@ -49,7 +49,7 @@
 from conans.model.ref import ConanFileReference, PackageReference, check_valid_ref
 from conans.model.version import Version
 from conans.model.workspace import Workspace
-from conans.paths import BUILD_INFO, CONANINFO, get_conan_user_home
+from conans.paths import BUILD_INFO, CONANINFO, get_conan_user_home, CONAN_PACKAGE_LAYOUT_FILE
 from conans.tools import set_global_instances
 from conans.unicode import get_cwd
 from conans.util.env_reader import get_env
@@ -933,6 +933,26 @@ def get_default_remote(self):
     def get_remote_by_name(self, remote_name):
         return self._client_cache.registry.remotes.get(remote_name)
 
+    @api_method
+    def link(self, target_path, target_reference, cwd):
+        # Retrieve conanfile.py from target_path
+        target_path = _get_conanfile_path(path=target_path, cwd=cwd, py=True)
+
+        ref = ConanFileReference.loads(target_reference, validate=True)
+        target_conanfile = self._graph_manager._loader.load_class(target_path)
+        if (target_conanfile.name and target_conanfile.name != ref.name) or \
+                (target_conanfile.version and target_conanfile.version != ref.version):
+            raise ConanException("Name and version from reference ({}) and target "
+                                 "conanfile.py ({}/{}) must match".
+                                 format(ref, target_conanfile.name, target_conanfile.version))
+
+        self._client_cache.install_as_editable(ref, os.path.dirname(target_path))
+
+    @api_method
+    def unlink(self, reference):
+        ref = ConanFileReference.loads(reference, validate=True)
+        return self._client_cache.remove_editable(ref)
+
 
 Conan = ConanAPIV1
 

conans/client/conan_command_output.py

@@ -7,6 +7,7 @@
 from conans.search.binary_html_table import html_binary_graph
 from conans.unicode import get_cwd
 from conans.util.files import save
+from conans.client.graph.graph import RECIPE_EDITABLE
 
 
 class CommandOutputer(object):
@@ -73,7 +74,7 @@ def _read_dates(self, deps_graph):
         ret = {}
         for node in sorted(deps_graph.nodes):
             ref = node.conan_ref
-            if node.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
+            if node.recipe not in (RECIPE_CONSUMER, RECIPE_VIRTUAL, RECIPE_EDITABLE):
                 manifest = self.client_cache.load_manifest(ref)
                 ret[ref] = manifest.time_str
         return ret

conans/client/graph/graph.py

@@ -12,6 +12,7 @@
 RECIPE_UPDATEABLE = "Update available"  # The update of the recipe is available (only in conan info)
 RECIPE_NO_REMOTE = "No remote"
 RECIPE_WORKSPACE = "Workspace"
+RECIPE_EDITABLE = "Editable"
 RECIPE_CONSUMER = "Consumer"  # A conanfile from the user
 RECIPE_VIRTUAL = "Virtual"  # A virtual conanfile (dynamic in memory conanfile)
 
@@ -22,6 +23,7 @@
 BINARY_MISSING = "Missing"
 BINARY_SKIP = "Skip"
 BINARY_WORKSPACE = "Workspace"
+BINARY_EDITABLE = "Editable"
 
 
 class Node(object):
@@ -99,6 +101,10 @@ def __cmp__(self, other):
         if self.conan_ref.revision is not None and other.conan_ref.revision is None:
             return -1
 
+        if self.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
+            return 1
+        if other.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
+            return -1
         if self.conan_ref < other.conan_ref:
             return -1
 

conans/client/graph/graph_binaries.py

@@ -2,7 +2,9 @@
 
 from conans.client.graph.graph import (BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_MISSING,
                                        BINARY_SKIP, BINARY_UPDATE, BINARY_WORKSPACE,
+                                       RECIPE_EDITABLE, BINARY_EDITABLE,
                                        RECIPE_CONSUMER, RECIPE_VIRTUAL)
+from conans.client.output import ScopedOutput
 from conans.errors import NoRemoteAvailable, NotFoundException
 from conans.model.info import ConanInfo
 from conans.model.manifest import FileTreeManifest
@@ -74,6 +76,11 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_
         evaluated_references[package_ref] = node
 
         output = conanfile.output
+
+        if node.recipe == RECIPE_EDITABLE:
+            node.binary = BINARY_EDITABLE
+            return
+
         if build_mode.forced(conanfile, conan_ref):
             output.warn('Forced build from source')
             node.binary = BINARY_BUILD
@@ -91,7 +98,8 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_
         with self._client_cache.package_lock(package_ref):
             if is_dirty(package_folder):
                 output.warn("Package is corrupted, removing folder: %s" % package_folder)
-                rmdir(package_folder)
+                assert node.recipe != RECIPE_EDITABLE, "Editable package cannot be dirty"
+                rmdir(package_folder)  # Do not remove if it is EDITABLE
 
         if remote_name:
             remote = self._registry.remotes.get(remote_name)
@@ -116,6 +124,7 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_
             if not node.binary:
                 node.binary = BINARY_CACHE
                 package_hash = ConanInfo.load_from_package(package_folder).recipe_hash
+
         else:  # Binary does NOT exist locally
             if not revisions_enabled and not node.revision_pinned:
                 # Do not search for packages for the specific resolved recipe revision but all

conans/client/graph/proxy.py

@@ -4,7 +4,7 @@
 
 from conans.client.graph.graph import (RECIPE_DOWNLOADED, RECIPE_INCACHE, RECIPE_NEWER,
                                        RECIPE_NOT_IN_REMOTE, RECIPE_NO_REMOTE, RECIPE_UPDATEABLE,
-                                       RECIPE_UPDATED)
+                                       RECIPE_UPDATED, RECIPE_EDITABLE)
 from conans.client.output import ScopedOutput
 from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING, INSTALL_ERROR_NETWORK
 from conans.client.remover import DiskRemover
@@ -23,6 +23,13 @@ def __init__(self, client_cache, output, remote_manager):
         self._registry = client_cache.registry
 
     def get_recipe(self, conan_reference, check_updates, update, remote_name, recorder):
+        if self._client_cache.installed_as_editable(conan_reference):
+            conanfile_path = self._client_cache.conanfile(conan_reference)
+            status = RECIPE_EDITABLE
+            # TODO: log_recipe_got_from_editable(reference)
+            # TODO: recorder.recipe_fetched_as_editable(reference)
+            return conanfile_path, status, None, conan_reference
+
         with self._client_cache.conanfile_write_lock(conan_reference):
             result = self._get_recipe(conan_reference, check_updates, update, remote_name, recorder)
             conanfile_path, status, remote, reference = result

conans/client/installer.py

@@ -7,7 +7,7 @@
 from conans.client.file_copier import report_copied_files
 from conans.client.generators import TXTGenerator, write_generators
 from conans.client.graph.graph import BINARY_BUILD, BINARY_CACHE, BINARY_DOWNLOAD, BINARY_MISSING, \
-    BINARY_SKIP, BINARY_UPDATE
+    BINARY_SKIP, BINARY_UPDATE, BINARY_EDITABLE
 from conans.client.importer import remove_imports
 from conans.client.output import ScopedOutput
 from conans.client.packager import create_package
@@ -19,6 +19,7 @@
                            conanfile_exception_formatter)
 from conans.model.build_info import CppInfo
 from conans.model.conan_file import get_env_context_manager
+from conans.model.editable_cpp_info import EditableCppInfo
 from conans.model.env_info import EnvInfo
 from conans.model.manifest import FileTreeManifest
 from conans.model.ref import PackageReference
@@ -265,6 +266,7 @@ def __init__(self, client_cache, output, remote_manager, recorder, workspace, ho
         self._recorder = recorder
         self._workspace = workspace
         self._hook_manager = hook_manager
+        self._editable_cpp_info = self._load_editables_cpp_info()
 
     def install(self, deps_graph, keep_build=False, graph_info=None):
         # order by levels and separate the root node (conan_ref=None) from the rest
@@ -288,6 +290,10 @@ def _build(self, nodes_by_level, deps_graph, keep_build, root_node, graph_info):
                     raise_package_not_found_error(conan_file, conan_ref, package_id, dependencies,
                                                   out=output, recorder=self._recorder)
 
+                if node.binary == BINARY_EDITABLE:
+                    self._handle_node_editable(node)
+                    continue
+
                 workspace_package = self._workspace[node.conan_ref] if self._workspace else None
                 if workspace_package:
                     self._handle_node_workspace(node, workspace_package, inverse_levels, deps_graph,
@@ -311,6 +317,41 @@ def _node_concurrently_installed(self, node, package_folder):
             if node.update_manifest == read_manifest:
                 return True
 
+    def _load_editables_cpp_info(self):
+        editables_path = self._client_cache.default_editable_path
+        if os.path.exists(editables_path):
+            return EditableCppInfo.load(editables_path, require_namespace=True)
+        return None
+
+    def _handle_node_editable(self, node):
+        # Get source of information
+        package_layout = self._client_cache.package_layout(node.conan_ref)
+        base_path = package_layout.conan()
+        self._call_package_info(node.conanfile, package_folder=base_path)
+
+        # Try with package-provided file
+        package_layout_file = package_layout.editable_package_layout_file()
+        if os.path.exists(package_layout_file):
+            editable_cpp_info = EditableCppInfo.load(package_layout_file,
+                                                     require_namespace=False)
+            editable_cpp_info.apply_to(node.conanfile.name,
+                                       node.conanfile.cpp_info,
+                                       base_path=base_path,
+                                       settings=node.conanfile.settings,
+                                       options=node.conanfile.options)
+
+        # Try with the profile-like file
+        elif self._editable_cpp_info and self._editable_cpp_info.has_info_for(node.conanfile.name):
+            self._editable_cpp_info.apply_to(node.conanfile.name,
+                                             node.conanfile.cpp_info,
+                                             base_path=base_path,
+                                             settings=node.conanfile.settings,
+                                             options=node.conanfile.options)
+
+        # Use `package_info()` data
+        else:
+            pass  # It will use `package_info()` data relative to path used as 'package_folder'
+
     def _handle_node_cache(self, node, package_ref, keep_build, processed_package_references):
         conan_file = node.conanfile
         output = conan_file.output

conans/client/manager.py

@@ -2,6 +2,7 @@
 
 from conans.client.client_cache import ClientCache
 from conans.client.generators import write_generators
+from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL
 from conans.client.graph.printer import print_graph
 from conans.client.importer import run_deploy, run_imports
 from conans.client.installer import BinaryInstaller, call_system_requirements
@@ -14,7 +15,6 @@
 from conans.model.ref import ConanFileReference
 from conans.paths import CONANINFO
 from conans.util.files import normalize, save
-from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL
 
 
 class ConanManager(object):
@@ -52,7 +52,7 @@ def install(self, reference, install_folder, graph_info, remote_name=None, build
         """ Fetch and build all dependencies for the given reference
         @param reference: ConanFileReference or path to user space conanfile
         @param install_folder: where the output files will be saved
-        @param remote: install only from that remote
+        @param remote_name: install only from that remote
         @param profile: Profile object with both the -s introduced options and profile read values
         @param build_modes: List of build_modes specified
         @param update: Check for updated in the upstream remotes (and update)

conans/client/manifest_manager.py

@@ -1,11 +1,11 @@
 import os
 
+from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL
 from conans.client.remote_registry import Remote
 from conans.errors import ConanException
 from conans.model.manifest import FileTreeManifest
 from conans.model.ref import PackageReference
-from conans.paths import SimplePaths
-from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL
+from conans.paths.simple_paths import SimplePaths
 
 
 class ManifestManager(object):

conans/client/printer.py

@@ -5,7 +5,7 @@
 from conans.client.output import Color
 from conans.model.options import OptionsValues
 from conans.model.ref import ConanFileReference, PackageReference
-from conans.paths import SimplePaths
+from conans.paths.simple_paths import SimplePaths
 from conans.util.env_reader import get_env
 from conans.client.graph.graph import RECIPE_CONSUMER, RECIPE_VIRTUAL