#3812 Remove system requirements from cache (#4354)

* #3812 Remove cached system_reqs

- Add option to remove system_reqs from package cache.

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Add tests for remove --system_reqs

- conan remove --system_reqs should remove global requirement folder.
- Should we support by package id?

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 remove logger from cache

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Update system_reqs description

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 --system-reqs can raise error

- Conan remove --system-reqs forwards the exception message
  when an error occurs.
- Re-order remove command arguments
- Add test to check system-reqs error message

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Remove system-reqs by package id

- Add support to parse package reference when `remove --system-reqs -p
 <ref>` is executed and remove ONLY the cache for the specific
  package id.

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Revert "#3812 Remove system-reqs by package id"

This reverts commit 4d3e7fc.

* #3812 Ignore errors when remove system-reqs

- Package ID is NOT supported for system-reqs
- Ignore error, like permission or path not found

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Remove system-reqs can raise errors

- Add exception for permission denied and wrong pkg ref
- Add new test to check possible errors in system-reqs

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Fix test for Windows platform

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #3812 Replace rmtree by rmdir

Signed-off-by: Uilian Ries <uilianries@gmail.com>

conans/client/cache.py

@@ -14,12 +14,12 @@
 from conans.model.profile import Profile
 from conans.model.ref import ConanFileReference
 from conans.model.settings import Settings
-from conans.paths import PUT_HEADERS
+from conans.paths import PUT_HEADERS, SYSTEM_REQS_FOLDER
 from conans.paths.package_layouts.package_cache_layout import PackageCacheLayout
 from conans.paths.package_layouts.package_editable_layout import PackageEditableLayout
 from conans.paths.simple_paths import SimplePaths, check_ref_case
 from conans.unicode import get_cwd
-from conans.util.files import list_folder_subdirs, load, normalize, save
+from conans.util.files import list_folder_subdirs, load, normalize, save, rmdir
 from conans.util.locks import Lock, NoLock, ReadLock, SimpleLock, WriteLock
 
 
@@ -269,6 +269,19 @@ def delete_empty_dirs(self, deleted_refs):
                         break  # not empty
                 ref_path = os.path.dirname(ref_path)
 
+    def remove_package_system_reqs(self, reference):
+        assert isinstance(reference, ConanFileReference)
+        conan_folder = self.conan(reference)
+        system_reqs_folder = os.path.join(conan_folder, SYSTEM_REQS_FOLDER)
+        if not os.path.exists(conan_folder):
+            raise ValueError("%s does not exist" % repr(reference))
+        if not os.path.exists(system_reqs_folder):
+            return
+        try:
+            rmdir(system_reqs_folder)
+        except Exception as e:
+            raise ConanException("Unable to remove system requirements at %s: %s" % (system_reqs_folder, str(e)))
+
     def remove_locks(self):
         folders = list_folder_subdirs(self._store_folder, 4)
         for folder in folders:

conans/client/command.py

@@ -828,6 +828,8 @@ def remove(self, *args):
                                   "specifying the package ID"))
         parser.add_argument('-f', '--force', default=False, action='store_true',
                             help='Remove without requesting a confirmation')
+        parser.add_argument("-l", "--locks", default=False, action="store_true",
+                            help="Remove locks")
         parser.add_argument("-o", "--outdated", default=False, action="store_true",
                             help="Remove only outdated from recipe packages. " \
                                  "This flag can only be used with a reference")
@@ -838,8 +840,8 @@ def remove(self, *args):
                             help='Will remove from the specified remote')
         parser.add_argument('-s', '--src', default=False, action="store_true",
                             help='Remove source folders')
-        parser.add_argument("-l", "--locks", default=False, action="store_true",
-                            help="Remove locks")
+        parser.add_argument('-t', '--system-reqs', default=False, action="store_true",
+                            help='Remove system_reqs folders')
         args = parser.parse_args(*args)
 
         self._warn_python2()
@@ -863,6 +865,17 @@ def remove(self, *args):
             self._cache.remove_locks()
             self._user_io.out.info("Cache locks removed")
             return
+        elif args.system_reqs:
+            if not ref:
+                raise ConanException("Please specify a valid package reference to be cleaned")
+            if args.packages:
+                raise ConanException("'-t' and '-p' parameters can't be used at the same time")
+            try:
+                self._cache.remove_package_system_reqs(ref)
+                self._user_io.out.info("Cache system_reqs from %s has been removed" % repr(ref))
+                return
+            except Exception as error:
+                raise ConanException("Unable to remove system_reqs: %s" % error)
         else:
             if not args.pattern_or_reference:
                 raise ConanException('Please specify a pattern to be removed ("*" for all)')

conans/test/integration/system_reqs_test.py

@@ -1,6 +1,9 @@
 import os
+import stat
 import unittest
 
+from conans.paths import SYSTEM_REQS_FOLDER
+
 from conans.model.ref import ConanFileReference, PackageReference
 from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient
 from conans.util.files import load
@@ -22,7 +25,6 @@ def system_requirements(self):
 
 
 class SystemReqsTest(unittest.TestCase):
-
     def force_system_reqs_rerun_test(self):
         client = TestClient()
         files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}
@@ -92,8 +94,10 @@ def per_package_test(self):
 
     def global_test(self):
         client = TestClient()
-        files = {'conanfile.py': base_conanfile.replace("%GLOBAL%",
-                                                        "self.global_system_requirements=True")}
+        files = {
+            'conanfile.py': base_conanfile.replace("%GLOBAL%",
+                                                   "self.global_system_requirements=True")
+        }
         client.save(files)
         client.run("export . user/testing")
         client.run("install Test/0.1@user/testing --build missing")
@@ -128,8 +132,10 @@ def global_test(self):
 
     def wrong_output_test(self):
         client = TestClient()
-        files = {'conanfile.py':
-                 base_conanfile.replace("%GLOBAL%", "").replace('"Installed my stuff"', 'None')}
+        files = {
+            'conanfile.py':
+            base_conanfile.replace("%GLOBAL%", "").replace('"Installed my stuff"', 'None')
+        }
         client.save(files)
         client.run("export . user/testing")
         client.run("install Test/0.1@user/testing --build missing")
@@ -139,3 +145,110 @@ def wrong_output_test(self):
         pref = PackageReference(ref, "f0ba3ca2c218df4a877080ba99b65834b9413798")
         load_file = load(client.cache.system_reqs_package(pref))
         self.assertEqual('', load_file)
+
+    def remove_system_reqs_test(self):
+        ref = ConanFileReference.loads("Test/0.1@user/channel")
+        client = TestClient()
+        files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}
+        client.save(files)
+        system_reqs_path = os.path.join(
+            client.cache.package_layout(ref).conan(), SYSTEM_REQS_FOLDER)
+
+        # create package to populate system_reqs folder
+        self.assertFalse(os.path.exists(system_reqs_path))
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+        # a new build must not remove or re-run
+        client.run("create . user/channel")
+        self.assertNotIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+        # remove system_reqs global
+        client.run("remove --system-reqs Test/0.1@user/channel")
+        self.assertIn("Cache system_reqs from Test/0.1@user/channel has been removed",
+                      client.user_io.out)
+        self.assertFalse(os.path.exists(system_reqs_path))
+
+        # re-create system_reqs folder
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+    def invalid_remove_reqs_test(self):
+        client = TestClient()
+
+        with self.assertRaisesRegexp(
+                Exception, "ERROR: Please specify a valid package reference to be cleaned"):
+            client.run("remove --system-reqs")
+
+        # wrong file reference should be treated as error
+        with self.assertRaisesRegexp(
+                Exception, "ERROR: Unable to remove system_reqs: foo/version@bar/testing does not exist"):
+            client.run("remove --system-reqs foo/version@bar/testing")
+
+        # package is not supported with system_reqs
+        with self.assertRaisesRegexp(
+                Exception, "ERROR: '-t' and '-p' parameters can't be used at the same time"):
+            client.run("remove --system-reqs foo/bar@foo/bar -p f0ba3ca2c218df4a877080ba99b65834b9413798")
+
+    def permission_denied_remove_system_reqs_test(self):
+        ref = ConanFileReference.loads("Test/0.1@user/channel")
+        client = TestClient()
+        files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}
+        client.save(files)
+        system_reqs_path = os.path.join(
+            client.cache.package_layout(ref).conan(), SYSTEM_REQS_FOLDER)
+
+        # create package to populate system_reqs folder
+        self.assertFalse(os.path.exists(system_reqs_path))
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+        # remove write permission
+        current = stat.S_IMODE(os.lstat(system_reqs_path).st_mode)
+        os.chmod(system_reqs_path, current & ~stat.S_IWRITE)
+
+        # friendly message for permission error
+        with self.assertRaisesRegexp(
+                Exception, "ERROR: Unable to remove system_reqs:"):
+            client.run("remove --system-reqs Test/0.1@user/channel")
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+    def duplicate_remove_system_reqs_test(self):
+        ref = ConanFileReference.loads("Test/0.1@user/channel")
+        client = TestClient()
+        files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}
+        client.save(files)
+        system_reqs_path = os.path.join(
+            client.cache.package_layout(ref).conan(), SYSTEM_REQS_FOLDER)
+
+        # create package to populate system_reqs folder
+        self.assertFalse(os.path.exists(system_reqs_path))
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+        # a new build must not remove or re-run
+        client.run("create . user/channel")
+        self.assertNotIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))
+
+        # remove system_reqs global
+        client.run("remove --system-reqs Test/0.1@user/channel")
+        self.assertIn("Cache system_reqs from Test/0.1@user/channel has been removed",
+                      client.user_io.out)
+        self.assertFalse(os.path.exists(system_reqs_path))
+
+        # try to remove system_reqs global again
+        client.run("remove --system-reqs Test/0.1@user/channel")
+        self.assertIn("Cache system_reqs from Test/0.1@user/channel has been removed",
+                      client.user_io.out)
+        self.assertFalse(os.path.exists(system_reqs_path))
+
+        # re-create system_reqs folder
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(system_reqs_path))