bug fix: migrations do not respect core.cache:storage_path

migrations do not respect core.cache:storage_path.  If the user
has this option set, and tries to migrate from conan 2.0.13 to
2.0.14, the migration will fail.

This fix still assumes a 1-1 relationship between the $CONAN_HOME and
the local cache, but the forward and backward migrations are updated
to find the local cache in the correct way.

conan/api/conan_api.py

@@ -49,7 +49,7 @@ def __init__(self, cache_folder=None):
         self.graph = GraphAPI(self)
         self.export = ExportAPI(self)
         self.remove = RemoveAPI(self)
-        self.config = ConfigAPI(self)
+        self.config = ConfigAPI(self.home_folder)
         self.new = NewAPI(self)
         self.upload = UploadAPI(self)
         self.download = DownloadAPI(self)

conan/api/subapi/config.py

@@ -14,18 +14,18 @@
 
 class ConfigAPI:
 
-    def __init__(self, conan_api):
-        self.conan_api = conan_api
+    def __init__(self, home_folder):
+        self.home_folder = home_folder
         self._new_config = None
 
     def home(self):
-        return self.conan_api.cache_folder
+        return self.home_folder
 
     def install(self, path_or_url, verify_ssl, config_type=None, args=None,
                 source_folder=None, target_folder=None):
         # TODO: We probably want to split this into git-folder-http cases?
         from conans.client.conf.config_installer import configuration_install
-        app = ConanApp(self.conan_api.cache_folder, self.conan_api.config.global_conf)
+        app = ConanApp(self.home_folder, self.global_conf)
         return configuration_install(app, path_or_url, verify_ssl,
                                      config_type=config_type, args=args,
                                      source_folder=source_folder, target_folder=target_folder)
@@ -44,18 +44,17 @@ def global_conf(self):
         """
         if self._new_config is None:
             self._new_config = ConfDefinition()
-            cache_folder = self.conan_api.cache_folder
-            home_paths = HomePaths(cache_folder)
+            home_paths = HomePaths(self.home_folder)
             global_conf_path = home_paths.global_conf_path
             if os.path.exists(global_conf_path):
                 text = load(global_conf_path)
                 distro = None
                 if platform.system() in ["Linux", "FreeBSD"]:
                     import distro
-                template = Environment(loader=FileSystemLoader(cache_folder)).from_string(text)
+                template = Environment(loader=FileSystemLoader(self.home_folder)).from_string(text)
                 content = template.render({"platform": platform, "os": os, "distro": distro,
                                            "conan_version": conan_version,
-                                           "conan_home_folder": cache_folder,
+                                           "conan_home_folder": self.home_folder,
                                            "detect_api": detect_api})
                 self._new_config.loads(content)
             else:  # creation of a blank global.conf file for user convenience

conans/client/cache/cache.py

@@ -22,8 +22,7 @@ def __init__(self, cache_folder, global_conf):
         self.cache_folder = cache_folder
         self.editable_packages = EditablePackages(self.cache_folder)
         # paths
-        self._store_folder = global_conf.get("core.cache:storage_path") or \
-                             os.path.join(self.cache_folder, "p")
+        self._store_folder = self.find_store_folder(cache_folder, global_conf)
 
         try:
             mkdir(self._store_folder)
@@ -32,6 +31,10 @@ def __init__(self, cache_folder, global_conf):
         except Exception as e:
             raise ConanException(f"Couldn't initialize storage in {self._store_folder}: {e}")
 
+    @staticmethod
+    def find_store_folder(cache_folder, global_conf):
+        return global_conf.get("core.cache:storage_path") or os.path.join(cache_folder, "p")
+
     @property
     def temp_folder(self):
         """ temporary folder where Conan puts exports and packages before the final revision

conans/client/migrations.py

@@ -2,10 +2,13 @@
 import sqlite3
 import textwrap
 
+from conan.api.subapi.config import ConfigAPI
 from conan.api.output import ConanOutput
 from conans.migrations import Migrator
 from conans.util.dates import timestamp_now
 from conans.util.files import load, save
+from conans.client.cache.cache import ClientCache
+
 
 CONAN_GENERATED_COMMENT = "This file was generated by Conan"
 
@@ -35,31 +38,40 @@ def update_file(file_path, new_content):
 
 class ClientMigrator(Migrator):
 
-    def __init__(self, cache_folder, current_version):
-        self.cache_folder = cache_folder
-        super(ClientMigrator, self).__init__(cache_folder, current_version)
+    def __init__(self, home_folder, current_version):
+        self.home_folder = home_folder
+        super().__init__(home_folder, current_version)
 
     def _apply_migrations(self, old_version):
         # Migrate the settings if they were the default for that version
         # Time for migrations!
         # Update settings.yml
         from conans.client.conf import migrate_settings_file
-        migrate_settings_file(self.cache_folder)
+        migrate_settings_file(self.home_folder)
         # Update compatibility.py, app_compat.py, and cppstd_compat.py.
         from conans.client.graph.compatibility import migrate_compatibility_files
-        migrate_compatibility_files(self.cache_folder)
+        migrate_compatibility_files(self.home_folder)
         # Update profile plugin
         from conans.client.profile_loader import migrate_profile_plugin
-        migrate_profile_plugin(self.cache_folder)
+        migrate_profile_plugin(self.home_folder)
+
+        # Find the local cache
+        config = ConfigAPI(self.home_folder)
+        store_folder = ClientCache.find_store_folder(self.home_folder, config.global_conf)
 
-        if old_version and old_version < "2.0.14-":
-            _migrate_pkg_db_lru(self.cache_folder, old_version)
+        # Migrate the local cache
+        if old_version and old_version < "2.0.15-":
+            if old_version and old_version < "2.0.14-":
+                _migrate_pkg_db_lru(self.home_folder, store_folder, old_version)
+            else:
+                # database is already migrated, just rewrite the undo file
+                _save_pkg_db_lru_migration(self.home_folder)
 
 
-def _migrate_pkg_db_lru(cache_folder, old_version):
+def _migrate_pkg_db_lru(home_folder, store_folder, old_version):
     ConanOutput().warning(f"Upgrade cache from Conan version '{old_version}'")
     ConanOutput().warning("Running 2.0.14 Cache DB migration to add LRU column")
-    db_filename = os.path.join(cache_folder, 'p', 'cache.sqlite3')
+    db_filename = os.path.join(store_folder, 'cache.sqlite3')
     connection = sqlite3.connect(db_filename, isolation_level=None,
                                  timeout=1, check_same_thread=False)
     try:
@@ -71,26 +83,44 @@ def _migrate_pkg_db_lru(cache_folder, old_version):
         ConanOutput().error(f"Could not complete the 2.0.14 DB migration."
                             " Please manually remove your .conan2 cache and reinstall packages")
         raise
-    else:  # generate the back-migration script
-        undo_lru = textwrap.dedent("""\
-            import os
-            import sqlite3
-            def migrate(cache_folder):
-                db = os.path.join(cache_folder, 'p', 'cache.sqlite3')
-                connection = sqlite3.connect(db, isolation_level=None, timeout=1,
-                                             check_same_thread=False)
-                rec_cols = 'reference, rrev, path, timestamp'
-                pkg_cols = 'reference, rrev, pkgid, prev, path, timestamp, build_id'
-                try:
-                    for table in ("recipes", "packages"):
-                        columns = pkg_cols if table == "packages" else rec_cols
-                        connection.execute(f"CREATE TABLE {table}_backup AS SELECT {columns} FROM {table};")
-                        connection.execute(f"DROP TABLE {table};")
-                        connection.execute(f"ALTER TABLE {table}_backup RENAME TO {table};")
-                finally:
-                    connection.close()
-            """)
-        path = os.path.join(cache_folder, "migrations", f"2.0.14_1-migrate.py")
-        save(path, undo_lru)
+    else:
+        _save_pkg_db_lru_migration(home_folder)
     finally:
         connection.close()
+
+# generate the back-migration script
+def _save_pkg_db_lru_migration(home_folder):
+    undo_lru = textwrap.dedent("""\
+        from conan.api.subapi.config import ConfigAPI
+        import os
+        import sqlite3
+
+        class MockAPI:
+            def __init__(self, home_folder):
+                self.cache_path = self.cache_folder = self.home_folder = home_folder
+                self.config = ConfigAPI(self)
+
+        def migrate(home_folder):
+            api = MockAPI(home_folder)
+            config = ConfigAPI(api)
+            store_folder = (config.get("core.cache:storage_path") or
+                os.path.join(home_folder, "p"))
+            version_txt = os.path.join(store_folder, "version.txt")
+            db = os.path.join(store_folder, 'cache.sqlite3')
+            connection = sqlite3.connect(db, isolation_level=None, timeout=1,
+                                            check_same_thread=False)
+            rec_cols = 'reference, rrev, path, timestamp'
+            pkg_cols = 'reference, rrev, pkgid, prev, path, timestamp, build_id'
+            try:
+                for table in ("recipes", "packages"):
+                    columns = pkg_cols if table == "packages" else rec_cols
+                    connection.execute(f"CREATE TABLE {table}_backup AS SELECT {columns} FROM {table};")
+                    connection.execute(f"DROP TABLE {table};")
+                    connection.execute(f"ALTER TABLE {table}_backup RENAME TO {table};")
+                if os.path.exists(version_txt):
+                    os.unlink(version_txt)
+            finally:
+                connection.close()
+    """)
+    path = os.path.join(home_folder, "migrations", f"2.0.14_1-migrate.py")
+    save(path, undo_lru)