fixing source bugs and issues (conan-io#4033)

* fixing source bugs and issues

* clean scm subfolder too

* fixing tests

conans/client/conan_api.py

@@ -682,7 +682,7 @@ def source(self, path, source_folder=None, info_folder=None, cwd=None):
             output.info("Executing exports to: %s" % source_folder)
             export_recipe(conanfile, conanfile_folder, source_folder, output)
             export_source(conanfile, conanfile_folder, source_folder, output)
-        config_source_local(source_folder, conanfile, conanfile_folder, output, conanfile_path,
+        config_source_local(source_folder, conanfile, output, conanfile_path,
                             self._hook_manager)
 
     @api_method

conans/client/installer.py

@@ -88,9 +88,7 @@ def prepare_build(self):
                                  "Close any app using it, and retry" % str(e))
 
         self._out.info('Building your package in %s' % self.build_folder)
-        sources_pointer = self._client_cache.scm_folder(self._conan_ref)
-        local_sources_path = load(sources_pointer) if os.path.exists(sources_pointer) else None
-        config_source(export_folder, export_source_folder, local_sources_path, self.source_folder,
+        config_source(export_folder, export_source_folder, self.source_folder,
                       self._conan_file, self._out, conanfile_path, self._conan_ref,
                       self._hook_manager, self._client_cache)
         self._out.info('Copying sources to build folder')

conans/client/remote_manager.py

@@ -237,6 +237,7 @@ def get_recipe_sources(self, conan_reference, export_folder, export_sources_fold
 
         unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME,
                             output=self._output)
+        # REMOVE in Conan 2.0
         c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
         if os.path.exists(c_src_path):
             merge_directories(c_src_path, export_sources_folder)

conans/client/source.py

@@ -3,16 +3,21 @@
 import shutil
 import six
 
-from conans.errors import ConanException, conanfile_exception_formatter, \
-    ConanExceptionInUserConanfileMethod
+from conans.client import tools
+from conans.errors import (ConanException, conanfile_exception_formatter,
+                           ConanExceptionInUserConanfileMethod)
 from conans.model.conan_file import get_env_context_manager
 from conans.model.scm import SCM, get_scm_data
 from conans.paths import EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE, CONAN_MANIFEST
-from conans.util.files import rmdir, set_dirty, is_dirty, clean_dirty, mkdir, walk
-from conans.client import tools
+from conans.util.files import (rmdir, set_dirty, is_dirty, clean_dirty, mkdir, walk,
+                               load)
 
 
 def complete_recipe_sources(remote_manager, client_cache, registry, conanfile, conan_reference):
+    """ the "exports_sources" sources are not retrieved unless necessary to build. In some
+    occassions, conan needs to get them too, like if uploading to a server, to keep the recipes
+    complete
+    """
     sources_folder = client_cache.export_sources(conan_reference, conanfile.short_paths)
     if os.path.exists(sources_folder):
         return None
@@ -67,20 +72,19 @@ def is_excluded(origin_path):
                 shutil.copy2(src_file, dst_file)
 
 
-def _clean_source_folder(folder):
-    for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c",
-              CONANFILE+"o", CONANFILE, CONAN_MANIFEST):
-        try:
-            os.remove(os.path.join(folder, f))
-        except OSError:
-            pass
+def config_source_local(src_folder, conanfile, output, conanfile_path, hook_manager):
+    """ Entry point for the "conan source" command.
+    """
+    conanfile_folder = os.path.dirname(conanfile_path)
+    _run_source(conanfile, conanfile_path, src_folder, hook_manager, output, reference=None,
+                client_cache=None, export_folder=None, export_source_folder=None,
+                local_sources_path=conanfile_folder)
 
 
-def config_source(export_folder, export_source_folder, local_sources_path, src_folder,
-                  conanfile, output, conanfile_path, reference, hook_manager,
-                  client_cache):
-    """ creates src folder and retrieve, calling source() from conanfile
-    the necessary source code
+def config_source(export_folder, export_source_folder, src_folder, conanfile, output,
+                  conanfile_path, reference, hook_manager, client_cache):
+    """ Implements the sources configuration when a package is going to be built in the
+    local cache.
     """
 
     def remove_source(raise_error=True):
@@ -97,6 +101,8 @@ def remove_source(raise_error=True):
             if raise_error or isinstance(e_rm, KeyboardInterrupt):
                 raise ConanException("Unable to remove source folder")
 
+    sources_pointer = client_cache.scm_folder(reference)
+    local_sources_path = load(sources_pointer) if os.path.exists(sources_pointer) else None
     if is_dirty(src_folder):
         output.warn("Trying to remove corrupted source folder")
         remove_source()
@@ -107,86 +113,93 @@ def remove_source(raise_error=True):
         output.warn("Detected 'scm' auto in conanfile, trying to remove source folder")
         remove_source()
 
-    if not os.path.exists(src_folder):
+    if not os.path.exists(src_folder):  # No source folder, need to get it
         set_dirty(src_folder)
         mkdir(src_folder)
-        os.chdir(src_folder)
-        conanfile.source_folder = src_folder
+        _run_source(conanfile, conanfile_path, src_folder, hook_manager, output, reference,
+                    client_cache, export_folder, export_source_folder, local_sources_path)
+        clean_dirty(src_folder)  # Everything went well, remove DIRTY flag
+
+
+def _run_source(conanfile, conanfile_path, src_folder, hook_manager, output, reference,
+                client_cache, export_folder, export_source_folder, local_sources_path):
+    """Execute the source core functionality, both for local cache and user space, in order:
+        - Calling pre_source hook
+        - Getting sources from SCM
+        - Getting sources from exported folders in the local cache
+        - Clean potential TGZ and other files in the local cache
+        - Executing the recipe source() method
+        - Calling post_source hook
+    """
+    conanfile.source_folder = src_folder
+    conanfile.build_folder = None
+    conanfile.package_folder = None
+    with tools.chdir(src_folder):
         try:
-            with conanfile_exception_formatter(str(conanfile), "source"):
-                with get_env_context_manager(conanfile):
-                    conanfile.build_folder = None
-                    conanfile.package_folder = None
-                    hook_manager.execute("pre_source", conanfile=conanfile,
-                                         conanfile_path=conanfile_path, reference=reference)
-                    output.info('Configuring sources in %s' % src_folder)
-                    scm_data = get_scm_data(conanfile)
-                    if scm_data:
-                        dest_dir = os.path.normpath(os.path.join(src_folder, scm_data.subfolder))
-                        captured = local_sources_path and os.path.exists(local_sources_path)
-                        local_sources_path = local_sources_path if captured else None
-                        _fetch_scm(scm_data, dest_dir, local_sources_path, output)
-
-                    # Files from python requires are obtained before the self files
-                    from conans.client.cmd.export import export_source
-                    for python_require in conanfile.python_requires:
-                        src = client_cache.export_sources(python_require.conan_ref)
-                        export_source(conanfile, src, src_folder, output)
-
-                    # so self exported files have precedence over python_requires ones
-                    merge_directories(export_folder, src_folder)
-                    # Now move the export-sources to the right location
-                    merge_directories(export_source_folder, src_folder)
+            with get_env_context_manager(conanfile):
+                hook_manager.execute("pre_source", conanfile=conanfile,
+                                     conanfile_path=conanfile_path,
+                                     reference=reference)
+                output.info('Configuring sources in %s' % src_folder)
+                _run_scm(conanfile, src_folder, local_sources_path, output, cache=client_cache)
+
+                if client_cache:
+                    _get_sources_from_exports(conanfile, src_folder, export_folder,
+                                              export_source_folder, output, client_cache)
                     _clean_source_folder(src_folder)
-                    try:
-                        shutil.rmtree(os.path.join(src_folder, "__pycache__"))
-                    except OSError:
-                        pass
-
+                with conanfile_exception_formatter(str(conanfile), "source"):
                     conanfile.source()
-                    hook_manager.execute("post_source", conanfile=conanfile,
-                                         conanfile_path=conanfile_path, reference=reference)
-            clean_dirty(src_folder)  # Everything went well, remove DIRTY flag
+
+                hook_manager.execute("post_source", conanfile=conanfile,
+                                     conanfile_path=conanfile_path,
+                                     reference=reference)
+        except ConanExceptionInUserConanfileMethod:
+            raise
         except Exception as e:
-            os.chdir(export_folder)
-            # in case source() fails (user error, typically), remove the src_folder
-            # and raise to interrupt any other processes (build, package)
-            output.warn("Trying to remove corrupted source folder")
-            remove_source(raise_error=False)
-            if isinstance(e, ConanExceptionInUserConanfileMethod):
-                raise e
             raise ConanException(e)
 
 
-def config_source_local(dest_dir, conanfile, conanfile_folder, output, conanfile_path,
-                        hook_manager):
-    conanfile.source_folder = dest_dir
-    conanfile.build_folder = None
-    conanfile.package_folder = None
-    with tools.chdir(dest_dir):
+def _get_sources_from_exports(conanfile, src_folder, export_folder, export_source_folder,
+                              output, client_cache):
+    # Files from python requires are obtained before the self files
+    from conans.client.cmd.export import export_source
+    for python_require in conanfile.python_requires:
+        src = client_cache.export_sources(python_require.conan_ref)
+        export_source(conanfile, src, src_folder, output)
+
+    # so self exported files have precedence over python_requires ones
+    merge_directories(export_folder, src_folder)
+    # Now move the export-sources to the right location
+    merge_directories(export_source_folder, src_folder)
+
+
+def _clean_source_folder(folder):
+    for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c",
+              CONANFILE+"o", CONANFILE, CONAN_MANIFEST):
         try:
-            with conanfile_exception_formatter(str(conanfile), "source"):
-                with get_env_context_manager(conanfile):
-                    hook_manager.execute("pre_source", conanfile=conanfile,
-                                         conanfile_path=conanfile_path)
-                    output.info('Configuring sources in %s' % dest_dir)
-                    scm_data = get_scm_data(conanfile)
-                    if scm_data:
-                        dest_dir = os.path.join(dest_dir, scm_data.subfolder)
-                        capture = scm_data.capture_origin or scm_data.capture_revision
-                        local_sources_path = conanfile_folder if capture else None
-                        _fetch_scm(scm_data, dest_dir, local_sources_path, output)
+            os.remove(os.path.join(folder, f))
+        except OSError:
+            pass
+    try:
+        shutil.rmtree(os.path.join(folder, "__pycache__"))
+    except OSError:
+        pass
 
-                    conanfile.source()
-                    hook_manager.execute("post_source", conanfile=conanfile,
-                                         conanfile_path=conanfile_path)
-        except ConanExceptionInUserConanfileMethod:
-            raise
-        except Exception as e:
-            raise ConanException(e)
 
+def _run_scm(conanfile, src_folder, local_sources_path, output, cache):
+    scm_data = get_scm_data(conanfile)
+    if not scm_data:
+        return
+
+    dest_dir = os.path.normpath(os.path.join(src_folder, scm_data.subfolder))
+    if cache:
+        # When in cache, capturing the sources from user space is done only if exists
+        captured = local_sources_path and os.path.exists(local_sources_path)
+    else:
+        # In user space, if revision="auto", then copy
+        captured = scm_data.capture_origin or scm_data.capture_revision
+    local_sources_path = local_sources_path if captured else None
 
-def _fetch_scm(scm_data, dest_dir, local_sources_path, output):
     if local_sources_path:
         excluded = SCM(scm_data, local_sources_path).excluded_files
         output.info("Getting sources from folder: %s" % local_sources_path)
@@ -195,4 +208,6 @@ def _fetch_scm(scm_data, dest_dir, local_sources_path, output):
         output.info("Getting sources from url: '%s'" % scm_data.url)
         scm = SCM(scm_data, dest_dir)
         scm.checkout()
-    _clean_source_folder(dest_dir)
+    if cache:
+        # This is a bit weird. Why after a SCM should we remove files. Maybe check conan 2.0
+        _clean_source_folder(dest_dir)

conans/test/command/export_dirty_test.py

@@ -5,6 +5,36 @@
 from conans.test.utils.cpp_test_files import cpp_hello_conan_files
 from conans.test.utils.tools import TestClient
 import platform
+from conans.util.files import load
+
+
+class SourceDirtyTest(unittest.TestCase):
+    def test_keep_failing_source_folder(self):
+        # https://github.com/conan-io/conan/issues/4025
+        client = TestClient()
+        conanfile = """from conans import ConanFile
+from conans.tools import save
+class Pkg(ConanFile):
+    def source(self):
+        save("somefile.txt", "hello world!!!")
+        raise Exception("boom")
+"""
+        client.save({"conanfile.py": conanfile})
+        client.run("create . pkg/1.0@user/channel", should_error=True)
+        self.assertIn("ERROR: pkg/1.0@user/channel: Error in source() method, line 6", client.out)
+        ref = ConanFileReference.loads("pkg/1.0@user/channel")
+        # Check that we can debug and see the folder
+        self.assertEqual(load(os.path.join(client.client_cache.source(ref), "somefile.txt")),
+                         "hello world!!!")
+        client.run("create . pkg/1.0@user/channel", should_error=True)
+        self.assertIn("pkg/1.0@user/channel: Source folder is corrupted, forcing removal",
+                      client.out)
+        client.save({"conanfile.py": conanfile.replace("source(", "source2(")})
+        client.run("create . pkg/1.0@user/channel")
+        self.assertIn("pkg/1.0@user/channel: Source folder is corrupted, forcing removal",
+                      client.out)
+        # Check that it is empty
+        self.assertEqual(os.listdir(os.path.join(client.client_cache.source(ref))), [])
 
 
 class ExportDirtyTest(unittest.TestCase):

conans/test/command/source_test.py

@@ -7,6 +7,21 @@
 
 class SourceTest(unittest.TestCase):
 
+    def test_conanfile_removed(self):
+        # https://github.com/conan-io/conan/issues/4013
+        conanfile = """from conans import ConanFile
+class ScmtestConan(ConanFile):
+    scm = {
+        "type": "git",
+        "url": "auto",
+        "revision": "auto"
+    }
+"""
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.run("source .")
+        self.assertEqual(["conanfile.py"], os.listdir(client.current_folder))
+
     def local_flow_patch_test(self):
         # https://github.com/conan-io/conan/issues/2327
         conanfile = """from conans import ConanFile, tools

conans/test/utils/tools.py

@@ -582,7 +582,7 @@ def init_dynamic_vars(self, user_io=None):
         # Maybe something have changed with migrations
         return self._init_collaborators(user_io)
 
-    def run(self, command_line, user_io=None, ignore_error=False):
+    def run(self, command_line, user_io=None, ignore_error=False, should_error=False):
         """ run a single command as in the command line.
             If user or password is filled, user_io will be mocked to return this
             tuple if required
@@ -614,7 +614,7 @@ def run(self, command_line, user_io=None, ignore_error=False):
             for added in added_modules:
                 sys.modules.pop(added, None)
 
-        if not ignore_error and error:
+        if not ignore_error and error and not should_error:
             exc_message = "\n{command_header}\n{command}\n{output_header}\n{output}\n{output_footer}\n".format(
                 command_header='{:-^80}'.format(" Command failed: "),
                 output_header='{:-^80}'.format(" Output: "),
@@ -624,6 +624,9 @@ def run(self, command_line, user_io=None, ignore_error=False):
             )
             raise Exception(exc_message)
 
+        if should_error and not error:
+            raise Exception("This command should have failed: %s" % command_line)
+
         self.all_output += str(self.user_io.out)
         return error