Cache LRU removal

conan/api/subapi/list.py

@@ -6,6 +6,7 @@
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
 from conans.search.search import get_cache_packages_binary_info, filter_packages
+from conans.util.dates import timelimit
 
 
 class ListAPI:
@@ -84,13 +85,18 @@ def filter_packages_configurations(pkg_configurations, query):
         """
         return filter_packages(query, pkg_configurations)
 
-    def select(self, pattern, package_query=None, remote=None):
+    def select(self, pattern, package_query=None, remote=None, lru=None):
         if package_query and pattern.package_id and "*" not in pattern.package_id:
             raise ConanException("Cannot specify '-p' package queries, "
                                  "if 'package_id' is not a pattern")
+        if remote and lru:
+            raise ConanException("'--lru' cannot be used in remotes, only in cache")
+
         select_bundle = PackagesList()
         # Avoid doing a ``search`` of recipes if it is an exact ref and it will be used later
         search_ref = pattern.search_ref
+        app = ConanApp(self.conan_api.cache_folder)
+        limit_time = timelimit(lru) if lru else None
         if search_ref:
             refs = self.conan_api.search.recipes(search_ref, remote=remote)
             refs = pattern.filter_versions(refs)
@@ -114,6 +120,10 @@ def select(self, pattern, package_query=None, remote=None):
                 rrevs = self.recipe_revisions(r, remote)
                 rrevs = pattern.filter_rrevs(rrevs)
                 rrevs = list(reversed(rrevs))  # Order older revisions first
+
+            if lru and pattern.package_id is None:  # Filter LRUs
+                rrevs = [r for r in rrevs if app.cache.get_recipe_lru(r) < limit_time]
+
             select_bundle.add_refs(rrevs)
 
             if pattern.package_id is None:  # Stop if not displaying binaries
@@ -148,6 +158,9 @@ def select(self, pattern, package_query=None, remote=None):
                             new_prefs.extend(prevs)
                     prefs = new_prefs
 
+                if lru:  # Filter LRUs
+                    prefs = [r for r in prefs if app.cache.get_package_lru(r) < limit_time]
+
                 select_bundle.add_prefs(rrev, prefs)
                 select_bundle.add_configurations(packages)
         return select_bundle

conan/cli/commands/list.py

@@ -108,15 +108,23 @@ def list(conan_api: ConanAPI, parser, *args):
     parser.add_argument("-g", "--graph", help="Graph json file")
     parser.add_argument("-gb", "--graph-binaries", help="Which binaries are listed", action="append")
     parser.add_argument("-gr", "--graph-recipes", help="Which recipes are listed", action="append")
+    parser.add_argument('--lru', default=None, action=OnceArgument,
+                        help="List recipes and binaries that have not been recently used. Use a"
+                             " time limit like --lru=5d (days) or --lru=4w (weeks),"
+                             " h (hours), m(minutes)")
 
     args = parser.parse_args(*args)
 
     if args.pattern is None and args.graph is None:
         raise ConanException("Missing pattern or graph json file")
     if args.pattern and args.graph:
         raise ConanException("Cannot define both the pattern and the graph json file")
+    if args.graph and args.lru:
+        raise ConanException("Cannot define lru when loading a graph json file")
     if (args.graph_recipes or args.graph_binaries) and not args.graph:
         raise ConanException("--graph-recipes and --graph-binaries require a --graph input")
+    if args.remote and args.lru:
+        raise ConanException("'--lru' cannot be used in remotes, only in cache")
 
     if args.graph:
         graphfile = make_abs_path(args.graph)
@@ -127,7 +135,8 @@ def list(conan_api: ConanAPI, parser, *args):
         pkglist = MultiPackagesList()
         if args.cache or not args.remote:
             try:
-                cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None)
+                cache_list = conan_api.list.select(ref_pattern, args.package_query, remote=None,
+                                                   lru=args.lru)
             except Exception as e:
                 pkglist.add_error("Local Cache", str(e))
             else:

conan/cli/commands/remove.py

@@ -51,6 +51,10 @@ def remove(conan_api: ConanAPI, parser, *args):
     parser.add_argument('-r', '--remote', action=OnceArgument,
                         help='Will remove from the specified remote')
     parser.add_argument("-l", "--list", help="Package list file")
+    parser.add_argument('--lru', default=None, action=OnceArgument,
+                        help="Remove recipes and binaries that have not been recently used. Use a"
+                             " time limit like --lru=5d (days) or --lru=4w (weeks),"
+                             " h (hours), m(minutes)")
     parser.add_argument("--dry-run", default=False, action="store_true",
                         help="Do not remove any items, only print those which would be removed")
     args = parser.parse_args(*args)
@@ -61,6 +65,10 @@ def remove(conan_api: ConanAPI, parser, *args):
         raise ConanException("Cannot define both the pattern and the package list file")
     if args.package_query and args.list:
         raise ConanException("Cannot define package-query and the package list file")
+    if args.remote and args.lru:
+        raise ConanException("'--lru' cannot be used in remotes, only in cache")
+    if args.list and args.lru:
+        raise ConanException("'--lru' cannot be used with input package list")
 
     ui = UserInput(conan_api.config.get("core:non_interactive"))
     remote = conan_api.remotes.get(args.remote) if args.remote else None
@@ -80,7 +88,7 @@ def confirmation(message):
         ref_pattern = ListPattern(args.pattern, rrev="*", prev="*")
         if ref_pattern.package_id is None and args.package_query is not None:
             raise ConanException('--package-query supplied but the pattern does not match packages')
-        package_list = conan_api.list.select(ref_pattern, args.package_query, remote)
+        package_list = conan_api.list.select(ref_pattern, args.package_query, remote, lru=args.lru)
         multi_package_list = MultiPackagesList()
         multi_package_list.add(cache_name, package_list)
 

conan/internal/cache/cache.py

@@ -232,3 +232,15 @@ def assign_rrev(self, layout: RecipeLayout):
             # This was exported before, making it latest again, update timestamp
             ref = layout.reference
             self._db.update_recipe_timestamp(ref)
+
+    def get_recipe_lru(self, ref):
+        return self._db.get_recipe_lru(ref)
+
+    def update_recipe_lru(self, ref):
+        self._db.update_recipe_lru(ref)
+
+    def get_package_lru(self, pref):
+        return self._db.get_package_lru(pref)
+
+    def update_package_lru(self, pref):
+        self._db.update_package_lru(pref)

conan/internal/cache/db/cache_database.py

@@ -30,6 +30,18 @@ def update_recipe_timestamp(self, ref):
     def update_package_timestamp(self, pref: PkgReference, path: str, build_id: str):
         self._packages.update_timestamp(pref, path=path, build_id=build_id)
 
+    def get_recipe_lru(self, ref):
+        return self._recipes.get_recipe(ref)["lru"]
+
+    def get_package_lru(self, pref: PkgReference):
+        return self._packages.get(pref)["lru"]
+
+    def update_recipe_lru(self, ref):
+        self._recipes.update_lru(ref)
+
+    def update_package_lru(self, pref):
+        self._packages.update_lru(pref)
+
     def remove_recipe(self, ref: RecipeReference):
         # Removing the recipe must remove all the package binaries too from DB
         self._recipes.remove(ref)

conan/internal/cache/db/packages_table.py

@@ -4,6 +4,7 @@
 from conans.errors import ConanReferenceDoesNotExistInDB, ConanReferenceAlreadyExistsInDB
 from conans.model.package_ref import PkgReference
 from conans.model.recipe_ref import RecipeReference
+from conans.util.dates import timestamp_now
 
 
 class PackagesDBTable(BaseDbTable):
@@ -14,7 +15,8 @@ class PackagesDBTable(BaseDbTable):
                            ('prev', str, True),
                            ('path', str, False, None, True),
                            ('timestamp', float),
-                           ('build_id', str, True)]
+                           ('build_id', str, True),
+                           ('lru', int)]
     unique_together = ('reference', 'rrev', 'pkgid', 'prev')
 
     @staticmethod
@@ -26,6 +28,7 @@ def _as_dict(row):
             "pref": pref,
             "build_id": row.build_id,
             "path": row.path,
+            "lru": row.lru
         }
 
     def _where_clause(self, pref: PkgReference):
@@ -73,12 +76,13 @@ def create(self, path, pref: PkgReference, build_id):
         # are saved with the temporary uuid one, we don't want to consider these
         # not yet built packages for search and so on
         placeholders = ', '.join(['?' for _ in range(len(self.columns))])
+        lru = timestamp_now()
         with self.db_connection() as conn:
             try:
                 conn.execute(f'INSERT INTO {self.table_name} '
-                                   f'VALUES ({placeholders})',
-                                   [str(pref.ref), pref.ref.revision, pref.package_id, pref.revision,
-                                    path, pref.timestamp, build_id])
+                             f'VALUES ({placeholders})',
+                             [str(pref.ref), pref.ref.revision, pref.package_id, pref.revision,
+                              path, pref.timestamp, build_id, lru])
             except sqlite3.IntegrityError:
                 raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(pref)}' already exists")
 
@@ -96,6 +100,18 @@ def update_timestamp(self, pref: PkgReference, path: str, build_id: str):
             except sqlite3.IntegrityError:
                 raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(pref)}' already exists")
 
+    def update_lru(self, pref):
+        assert pref.revision is not None
+        # TODO: InstallGraph is dropping the pref.timestamp, cannot be checked here yet
+        # assert pref.timestamp is not None, f"PREF _TIMESSTAMP IS NONE {repr(pref)}"
+        where_clause = self._where_clause(pref)
+        lru = timestamp_now()
+        query = f"UPDATE {self.table_name} " \
+                f'SET {self.columns.lru} = "{lru}" ' \
+                f"WHERE {where_clause};"
+        with self.db_connection() as conn:
+            conn.execute(query)
+
     def remove_build_id(self, pref):
         where_clause = self._where_clause(pref)
         query = f"UPDATE {self.table_name} " \
@@ -133,7 +149,8 @@ def get_package_revisions_references(self, pref: PkgReference, only_latest_prev=
                     f'{self.columns.prev}, ' \
                     f'{self.columns.path}, ' \
                     f'MAX({self.columns.timestamp}), ' \
-                    f'{self.columns.build_id} ' \
+                    f'{self.columns.build_id}, ' \
+                    f'{self.columns.lru} ' \
                     f'FROM {self.table_name} ' \
                     f'WHERE {self.columns.rrev} = "{pref.ref.revision}" ' \
                     f'AND {self.columns.reference} = "{str(pref.ref)}" ' \
@@ -165,7 +182,8 @@ def get_package_references(self, ref: RecipeReference, only_latest_prev=True):
                     f'{self.columns.prev}, ' \
                     f'{self.columns.path}, ' \
                     f'MAX({self.columns.timestamp}), ' \
-                    f'{self.columns.build_id} ' \
+                    f'{self.columns.build_id}, ' \
+                    f'{self.columns.lru} ' \
                     f'FROM {self.table_name} ' \
                     f'WHERE {self.columns.rrev} = "{ref.revision}" ' \
                     f'AND {self.columns.reference} = "{str(ref)}" ' \

conan/internal/cache/db/recipes_table.py

@@ -3,14 +3,16 @@
 from conan.internal.cache.db.table import BaseDbTable
 from conans.errors import ConanReferenceDoesNotExistInDB, ConanReferenceAlreadyExistsInDB
 from conans.model.recipe_ref import RecipeReference
+from conans.util.dates import timestamp_now
 
 
 class RecipesDBTable(BaseDbTable):
     table_name = 'recipes'
     columns_description = [('reference', str),
                            ('rrev', str),
                            ('path', str, False, None, True),
-                           ('timestamp', float)]
+                           ('timestamp', float),
+                           ('lru', int)]
     unique_together = ('reference', 'rrev')
 
     @staticmethod
@@ -21,6 +23,7 @@ def _as_dict(row):
         return {
             "ref": ref,
             "path": row.path,
+            "lru": row.lru,
         }
 
     def _where_clause(self, ref):
@@ -38,11 +41,12 @@ def create(self, path, ref: RecipeReference):
         assert ref.revision is not None
         assert ref.timestamp is not None
         placeholders = ', '.join(['?' for _ in range(len(self.columns))])
+        lru = timestamp_now()
         with self.db_connection() as conn:
             try:
                 conn.execute(f'INSERT INTO {self.table_name} '
                              f'VALUES ({placeholders})',
-                             [str(ref), ref.revision, path, ref.timestamp])
+                             [str(ref), ref.revision, path, ref.timestamp, lru])
             except sqlite3.IntegrityError:
                 raise ConanReferenceAlreadyExistsInDB(f"Reference '{repr(ref)}' already exists")
 
@@ -56,6 +60,17 @@ def update_timestamp(self, ref: RecipeReference):
         with self.db_connection() as conn:
             conn.execute(query)
 
+    def update_lru(self, ref):
+        assert ref.revision is not None
+        assert ref.timestamp is not None
+        where_clause = self._where_clause(ref)
+        lru = timestamp_now()
+        query = f"UPDATE {self.table_name} " \
+                f'SET {self.columns.lru} = "{lru}" ' \
+                f"WHERE {where_clause};"
+        with self.db_connection() as conn:
+            conn.execute(query)
+
     def remove(self, ref: RecipeReference):
         where_clause = self._where_clause(ref)
         query = f"DELETE FROM {self.table_name} " \
@@ -68,9 +83,11 @@ def all_references(self):
         query = f'SELECT DISTINCT {self.columns.reference}, ' \
                     f'{self.columns.rrev}, ' \
                     f'{self.columns.path} ,' \
-                    f'{self.columns.timestamp} ' \
+                    f'{self.columns.timestamp}, ' \
+                    f'{self.columns.lru} ' \
                     f'FROM {self.table_name} ' \
                     f'ORDER BY {self.columns.timestamp} DESC'
+
         with self.db_connection() as conn:
             r = conn.execute(query)
             result = [self._as_dict(self.row_type(*row)) for row in r.fetchall()]
@@ -92,7 +109,8 @@ def get_latest_recipe(self, ref: RecipeReference):
         query = f'SELECT {self.columns.reference}, ' \
                 f'{self.columns.rrev}, ' \
                 f'{self.columns.path}, ' \
-                f'MAX({self.columns.timestamp}) ' \
+                f'MAX({self.columns.timestamp}), ' \
+                f'{self.columns.lru} ' \
                 f'FROM {self.table_name} ' \
                 f'WHERE {self.columns.reference} = "{str(ref)}" ' \
                 f'GROUP BY {self.columns.reference} '  # OTHERWISE IT FAILS THE MAX()

conans/client/cache/cache.py

@@ -123,6 +123,18 @@ def get_matching_build_id(self, ref, build_id):
     def get_latest_package_reference(self, pref):
         return self._data_cache.get_latest_package_reference(pref)
 
+    def get_recipe_lru(self, ref):
+        return self._data_cache.get_recipe_lru(ref)
+
+    def update_recipe_lru(self, ref):
+        self._data_cache.update_recipe_lru(ref)
+
+    def get_package_lru(self, pref):
+        return self._data_cache.get_package_lru(pref)
+
+    def update_package_lru(self, pref):
+        self._data_cache.update_package_lru(pref)
+
     @property
     def store(self):
         return self._store_folder

conans/client/graph/proxy.py

@@ -45,6 +45,8 @@ def _get_recipe(self, reference, remotes, update, check_update):
             conanfile_path = recipe_layout.conanfile()
             return conanfile_path, status, remote, new_ref
 
+        self._cache.update_recipe_lru(ref)
+
         # TODO: cache2.0: check with new --update flows
         conanfile_path = recipe_layout.conanfile()
 

conans/client/installer.py

@@ -323,6 +323,7 @@ def _handle_package(self, package, install_reference, handled_count, total_count
         elif package.binary == BINARY_CACHE:
             node = package.nodes[0]
             pref = node.pref
+            self._cache.update_package_lru(pref)
             assert node.prev, "PREV for %s is None" % str(pref)
             node.conanfile.output.success(f'Already installed! ({handled_count} of {total_count})')