Fix: handle corner cases related to SCM with local sources optimization

conans/client/cmd/export.py

@@ -74,10 +74,7 @@ def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, outpu
     captured_revision = scm_data.capture_revision
 
     scm = SCM(scm_data, conanfile_dir, output)
-    if scm_data.capture_origin or scm_data.capture_revision:
-        # Generate the scm_folder.txt file pointing to the src_path
-        src_path = scm.get_repo_root()
-        save(scm_src_file, src_path.replace("\\", "/"))
+    captured = scm_data.capture_origin or scm_data.capture_revision
 
     if scm_data.url == "auto":
         origin = scm.get_qualified_remote_url(remove_credentials=True)
@@ -96,6 +93,15 @@ def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, outpu
 
     _replace_scm_data_in_conanfile(os.path.join(destination_folder, "conanfile.py"), scm_data)
 
+    if captured:
+        # Generate the scm_folder.txt file pointing to the src_path
+        src_path = scm.get_repo_root()
+        scm_url = SCM.clean_url(scm_data.url)  # Remove peg_revision (SVN), ....
+        url_root = SCM(scm_data, src_path, output).get_remote_url(remove_credentials=True)
+        if url_root:
+            src_path = os.path.join(src_path, os.path.relpath(scm_url, url_root))
+            save(scm_src_file, os.path.normpath(src_path).replace("\\", "/"))
+
     return scm_data, captured_revision
 
 

conans/client/source.py

@@ -37,6 +37,7 @@ def complete_recipe_sources(remote_manager, cache, conanfile, ref):
 
 
 def merge_directories(src, dst, excluded=None, symlinks=True):
+    src = os.path.normpath(src)
     dst = os.path.normpath(dst)
     excluded = excluded or []
     excluded = [os.path.normpath(entry) for entry in excluded]
@@ -192,20 +193,32 @@ def _run_scm(conanfile, src_folder, local_sources_path, output, cache):
     dest_dir = os.path.normpath(os.path.join(src_folder, scm_data.subfolder))
     if cache:
         # When in cache, capturing the sources from user space is done only if exists
-        captured = local_sources_path and os.path.exists(local_sources_path)
+        if not local_sources_path or not os.path.exists(local_sources_path):
+            local_sources_path = None
     else:
         # In user space, if revision="auto", then copy
-        captured = scm_data.capture_origin or scm_data.capture_revision
-    local_sources_path = local_sources_path if captured and conanfile.develop else None
-
-    if local_sources_path:
+        if scm_data.capture_origin or scm_data.capture_revision:  # FIXME: or clause?
+            scm = SCM(scm_data, local_sources_path, output)
+            scm_url = scm_data.url if scm_data.url != "auto" else \
+                scm.get_qualified_remote_url(remove_credentials=True)
+
+            if scm_url:  # If there is no remote (git), we cannot compute this block
+                src_path = scm.get_repo_root()
+                url_root = SCM(scm_data, src_path, output).get_remote_url(remove_credentials=True)
+                scm_url = SCM.clean_url(scm_url)  # Remove peg_revision (SVN), ....
+                local_sources_path = os.path.join(src_path, os.path.relpath(scm_url, url_root))
+        else:
+            local_sources_path = None
+
+    if local_sources_path and conanfile.develop:
         excluded = SCM(scm_data, local_sources_path, output).excluded_files
         output.info("Getting sources from folder: %s" % local_sources_path)
         merge_directories(local_sources_path, dest_dir, excluded=excluded)
     else:
         output.info("Getting sources from url: '%s'" % scm_data.url)
         scm = SCM(scm_data, dest_dir, output)
         scm.checkout()
+
     if cache:
         # This is a bit weird. Why after a SCM should we remove files. Maybe check conan 2.0
         _clean_source_folder(dest_dir)

conans/model/scm.py

@@ -99,3 +99,11 @@ def get_qualified_remote_url(self, remove_credentials):
 
     def is_local_repository(self):
         return self.repo.is_local_repository()
+
+    @classmethod
+    def clean_url(cls, url):
+        _, last_chunk = url.rsplit('/', 1)
+        if '@' in last_chunk:  # Remove peg_revision
+            url, peg_revision = url.rsplit('@', 1)
+            return url
+        return url

conans/test/functional/command/source_test.py

@@ -20,8 +20,10 @@ class ScmtestConan(ConanFile):
 """
         client = TestClient()
         client.save({"conanfile.py": conanfile})
+        client.runner("git init .", cwd=client.current_folder)
         client.run("source .")
-        self.assertEqual(["conanfile.py"], os.listdir(client.current_folder))
+        self.assertEqual(sorted(["conanfile.py", '.git']),
+                         sorted(os.listdir(client.current_folder)))
 
     def local_flow_patch_test(self):
         # https://github.com/conan-io/conan/issues/2327

conans/test/functional/python_requires/python_requires_test.py

@@ -286,7 +286,7 @@ def reuse_scm_test(self):
 
         conanfile = """from conans import ConanFile
 scm = {"type" : "git",
-       "url" : "somerepo",
+       "url" : "http://somerepo",
        "revision" : "auto"}
 
 class MyConanfileBase(ConanFile):
@@ -298,13 +298,13 @@ class MyConanfileBase(ConanFile):
         client.run("get MyConanfileBase/1.1@lasote/testing")
         # The global scm is left as-is
         self.assertIn("""scm = {"type" : "git",
-       "url" : "somerepo",
+       "url" : "http://somerepo",
        "revision" : "auto"}""", client.out)
         # but the class one is replaced
         self.assertNotIn("scm = scm", client.out)
         self.assertIn('    scm = {"revision":', client.out)
         self.assertIn('"type": "git",', client.out)
-        self.assertIn('"url": "somerepo"', client.out)
+        self.assertIn('"url": "http://somerepo"', client.out)
 
         reuse = """from conans import python_requires
 base = python_requires("MyConanfileBase/1.1@lasote/testing")
@@ -320,7 +320,7 @@ def _my_method(self):
         self.assertNotIn("scm = base.scm", client.out)
         self.assertIn('scm = {"revision":', client.out)
         self.assertIn('"type": "git",', client.out)
-        self.assertIn('"url": "somerepo"', client.out)
+        self.assertIn('"url": "http://somerepo"', client.out)
 
     def reuse_exports_test(self):
         conanfile = """from conans import ConanFile

conans/test/functional/scm/scm_test.py

@@ -18,13 +18,17 @@
 import os
 from conans import ConanFile, tools
 
+def get_svn_remote(path_from_conanfile):
+    svn = tools.SVN(os.path.join(os.path.dirname(__file__), path_from_conanfile))
+    return svn.get_remote_url()
+
 class ConanLib(ConanFile):
     name = "lib"
     version = "0.1"
     short_paths = True
     scm = {{
         "type": "%s",
-        "url": "{url}",
+        "url": {url},
         "revision": "{revision}",
     }}
 
@@ -38,6 +42,10 @@ def build(self):
 base_svn = base % "svn"
 
 
+def _quoted(item):
+    return '"{}"'.format(item)
+
+
 @attr('git')
 class GitSCMTest(unittest.TestCase):
 
@@ -52,7 +60,7 @@ def _commit_contents(self):
         self.client.runner("git add .", cwd=self.client.current_folder)
         self.client.runner('git commit -m  "commiting"', cwd=self.client.current_folder)
 
-    def test_scm_other_type_ignored(self):
+    def test_scm_other_type_ignored(self):  # FIXME: is this test ok?
         conanfile = '''
 from conans import ConanFile, tools
 
@@ -94,15 +102,15 @@ class ConanLib(ConanFile):
 
     def test_auto_filesystem_remote_git(self):
         # https://github.com/conan-io/conan/issues/3109
-        conanfile = base_git.format(directory="None", url="auto", revision="auto")
+        conanfile = base_git.format(directory="None", url=_quoted("auto"), revision="auto")
         repo = temp_folder()
         self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"}, repo)
         self.client.runner("git init .", cwd=repo)
         self.client.runner('git config user.email "you@example.com"', cwd=repo)
         self.client.runner('git config user.name "Your Name"', cwd=repo)
         self.client.runner("git add .", cwd=repo)
         self.client.runner('git commit -m  "commiting"', cwd=repo)
-        self.client.runner('git clone "%s" .' % repo, cwd=self.client.current_folder)
+        self.client.runner('git clone "%s" .' % repo.replace('\\', '/'), cwd=self.client.current_folder)
         self.client.run("export . user/channel")
         self.assertIn("WARN: Repo origin looks like a local path", self.client.out)
         os.remove(self.client.cache.scm_folder(self.ref))
@@ -111,7 +119,7 @@ def test_auto_filesystem_remote_git(self):
 
     def test_auto_git(self):
         curdir = get_cased_path(self.client.current_folder).replace("\\", "/")
-        conanfile = base_git.format(directory="None", url="auto", revision="auto")
+        conanfile = base_git.format(directory="None", url=_quoted("auto"), revision="auto")
         self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"})
         create_local_git_repo(folder=self.client.current_folder)
         self.client.run("export . user/channel", assert_error=True)
@@ -137,7 +145,7 @@ def build(self):""", self.client.out)
 
         # Export again but now with absolute reference, so no pointer file is created nor kept
         git = Git(curdir)
-        self.client.save({"conanfile.py": base_git.format(url=curdir, revision=git.get_revision())})
+        self.client.save({"conanfile.py": base_git.format(url=_quoted(curdir), revision=git.get_revision())})
         self.client.run("create . user/channel")
         sources_dir = self.client.cache.scm_folder(self.ref)
         self.assertFalse(os.path.exists(sources_dir))
@@ -152,7 +160,7 @@ def test_auto_subfolder(self):
                                      '"revision": "{revision}",\n        '
                                      '"subfolder": "mysub"')
         conanfile = conanfile.replace("short_paths = True", "short_paths = False")
-        conanfile = conanfile.format(directory="None", url="auto", revision="auto")
+        conanfile = conanfile.format(directory="None", url=_quoted("auto"), revision="auto")
         self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"})
         create_local_git_repo(folder=self.client.current_folder)
         self.client.runner('git remote add origin https://myrepo.com.git', cwd=curdir)
@@ -167,7 +175,7 @@ def test_auto_conanfile_no_root(self):
         Conanfile is not in the root of the repo: https://github.com/conan-io/conan/issues/3465
         """
         curdir = get_cased_path(self.client.current_folder).replace("\\", "/")
-        conanfile = base_git.format(url="auto", revision="auto")
+        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
         self.client.save({"conan/conanfile.py": conanfile, "myfile.txt": "content of my file"})
         self._commit_contents()
         self.client.runner('git remote add origin https://myrepo.com.git', cwd=curdir)
@@ -180,7 +188,7 @@ def test_auto_conanfile_no_root(self):
     def test_deleted_source_folder(self):
         path, _ = create_local_git_repo({"myfile": "contents"}, branch="my_release")
         curdir = self.client.current_folder.replace("\\", "/")
-        conanfile = base_git.format(url="auto", revision="auto")
+        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
         self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"})
         create_local_git_repo(folder=self.client.current_folder)
         self.client.runner('git remote add origin "%s"' % path.replace("\\", "/"), cwd=curdir)
@@ -194,7 +202,7 @@ def test_deleted_source_folder(self):
         self.assertIn("Getting sources from url: '%s'" % path.replace("\\", "/"), self.client.out)
 
     def test_excluded_repo_fies(self):
-        conanfile = base_git.format(url="auto", revision="auto")
+        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
         conanfile = conanfile.replace("short_paths = True", "short_paths = False")
         path, _ = create_local_git_repo({"myfile": "contents",
                                          "ignored.pyc": "bin",
@@ -224,8 +232,8 @@ def test_excluded_repo_fies(self):
         self.assertFalse(os.path.exists(os.path.join(bf, "other_folder", "excluded_subfolder")))
 
     def test_local_source(self):
-        curdir = self.client.current_folder
-        conanfile = base_git.format(url="auto", revision="auto")
+        curdir = self.client.current_folder.replace("\\", "/")
+        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
@@ -239,11 +247,13 @@ def source(self):
         self.assertIn("SOURCE METHOD CALLED", self.client.out)
         # Even the not commited files are copied
         self.assertTrue(os.path.exists(os.path.join(curdir, "source", "aditional_file.txt")))
-        self.assertIn("Getting sources from folder: %s" % curdir, self.client.out)
+        self.assertIn("Getting sources from folder: %s" % curdir,
+                      str(self.client.out).replace("\\", "/"))
 
         # Export again but now with absolute reference, so no pointer file is created nor kept
         git = Git(curdir.replace("\\", "/"))
-        conanfile = base_git.format(url=curdir.replace("\\", "/"), revision=git.get_revision())
+        conanfile = base_git.format(url=_quoted(curdir.replace("\\", "/")),
+                                    revision=git.get_revision())
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
@@ -263,7 +273,7 @@ def test_local_source_subfolder(self):
         conanfile = base_git.replace('"revision": "{revision}"',
                                      '"revision": "{revision}",\n        '
                                      '"subfolder": "mysub"')
-        conanfile = conanfile.format(url="auto", revision="auto")
+        conanfile = conanfile.format(url=_quoted("auto"), revision="auto")
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
@@ -282,7 +292,7 @@ def test_install_checked_out(self):
         self.client = TestClient(servers=self.servers, users={"myremote": [("lasote", "mypass")]})
 
         curdir = self.client.current_folder.replace("\\", "/")
-        conanfile = base_git.format(url="auto", revision="auto")
+        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
         self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"})
         create_local_git_repo(folder=self.client.current_folder)
         cmd = 'git remote add origin "%s"' % curdir
@@ -602,7 +612,7 @@ def test_auto_filesystem_remote_svn(self):
         pass
 
     def test_auto_svn(self):
-        conanfile = base_svn.format(directory="None", url="auto", revision="auto")
+        conanfile = base_svn.format(directory="None", url=_quoted("auto"), revision="auto")
         project_url, _ = self.create_project(files={"conanfile.py": conanfile,
                                                     "myfile.txt": "My file is copied"})
         project_url = project_url.replace(" ", "%20")
@@ -622,7 +632,7 @@ def test_auto_svn(self):
 
         # Export again but now with absolute reference, so no pointer file is created nor kept
         svn = SVN(curdir)
-        self.client.save({"conanfile.py": base_svn.format(url=svn.get_remote_url(),
+        self.client.save({"conanfile.py": base_svn.format(url=_quoted(svn.get_remote_url()),
                                                           revision=svn.get_revision())})
         self.client.run("create . user/channel")
         sources_dir = self.client.cache.scm_folder(self.ref)
@@ -638,7 +648,7 @@ def test_auto_subfolder(self):
                                      '"revision": "{revision}",\n        '
                                      '"subfolder": "mysub"')
         conanfile = conanfile.replace("short_paths = True", "short_paths = False")
-        conanfile = conanfile.format(directory="None", url="auto", revision="auto")
+        conanfile = conanfile.format(directory="None", url=_quoted("auto"), revision="auto")
 
         project_url, _ = self.create_project(files={"conanfile.py": conanfile,
                                                     "myfile.txt": "My file is copied"})
@@ -656,7 +666,7 @@ def test_auto_conanfile_no_root(self):
         Conanfile is not in the root of the repo: https://github.com/conan-io/conan/issues/3465
         """
         curdir = self.client.current_folder
-        conanfile = base_svn.format(url="auto", revision="auto")
+        conanfile = base_svn.format(url="get_svn_remote('..')", revision="auto")
         project_url, _ = self.create_project(files={"conan/conanfile.py": conanfile,
                                                     "myfile.txt": "My file is copied"})
         self.client.runner('svn co "{url}" "{path}"'.format(url=project_url,
@@ -671,7 +681,7 @@ def test_deleted_source_folder(self):
         pass
 
     def test_excluded_repo_fies(self):
-        conanfile = base_svn.format(url="auto", revision="auto")
+        conanfile = base_svn.format(url=_quoted("auto"), revision="auto")
         conanfile = conanfile.replace("short_paths = True", "short_paths = False")
         # SVN ignores pyc files by default:
         # http://blogs.collab.net/subversion/repository-dictated-configuration-day-3-global-ignores
@@ -695,14 +705,14 @@ def test_excluded_repo_fies(self):
         self.assertFalse(os.path.exists(os.path.join(bf, "ignored.pyc")))
 
     def test_local_source(self):
-        curdir = self.client.current_folder
-        conanfile = base_svn.format(url="auto", revision="auto")
+        curdir = self.client.current_folder.replace("\\", "/")
+        conanfile = base_svn.format(url=_quoted("auto"), revision="auto")
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
 """
-        project_url, _ = self.create_project(files={"conanfile.py": conanfile, "myfile.txt":
-                                                    "My file is copied"})
+        project_url, _ = self.create_project(files={"conanfile.py": conanfile,
+                                                    "myfile.txt": "My file is copied"})
         project_url = project_url.replace(" ", "%20")
         self.client.runner('svn co "{url}" "{path}"'.format(url=project_url,
                                                             path=self.client.current_folder))
@@ -716,8 +726,8 @@ def source(self):
         self.assertIn("Getting sources from folder: %s" % curdir, self.client.out)
 
         # Export again but now with absolute reference, so no pointer file is created nor kept
-        svn = SVN(curdir.replace("\\", "/"))
-        conanfile = base_svn.format(url=svn.get_remote_url(), revision=svn.get_revision())
+        svn = SVN(curdir)
+        conanfile = base_svn.format(url=_quoted(svn.get_remote_url()), revision=svn.get_revision())
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
@@ -738,7 +748,7 @@ def test_local_source_subfolder(self):
         conanfile = base_svn.replace('"revision": "{revision}"',
                                      '"revision": "{revision}",\n        '
                                      '"subfolder": "mysub"')
-        conanfile = conanfile.format(url="auto", revision="auto")
+        conanfile = conanfile.format(url=_quoted("auto"), revision="auto")
         conanfile += """
     def source(self):
         self.output.warn("SOURCE METHOD CALLED")
@@ -759,7 +769,7 @@ def test_install_checked_out(self):
         self.servers = {"myremote": test_server}
         self.client = TestClient(servers=self.servers, users={"myremote": [("lasote", "mypass")]})
 
-        conanfile = base_svn.format(url="auto", revision="auto")
+        conanfile = base_svn.format(url=_quoted("auto"), revision="auto")
         project_url, _ = self.create_project(files={"conanfile.py": conanfile,
                                                     "myfile.txt": "My file is copied"})
         project_url = project_url.replace(" ", "%20")