diff --git a/setup.py b/setup.py index f645c06c1c..b42aa292a4 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def package_files(directory): # What packages are required for this module to be executed? REQUIRED = [ f'aim-ui=={__version__}', - 'aimrocks==0.4.0', + 'aimrocks==0.5.1', 'khash==0.5.0a5', 'cachetools>=4.0.0', 'click>=7.0', diff --git a/src/python/aim/_core/storage/rockscontainer.pyx b/src/python/aim/_core/storage/rockscontainer.pyx index 32bfa81b52..039137a351 100644 --- a/src/python/aim/_core/storage/rockscontainer.pyx +++ b/src/python/aim/_core/storage/rockscontainer.pyx @@ -1,6 +1,8 @@ import logging import os +import tempfile from pathlib import Path +from cachetools.func import ttl_cache import aimrocks @@ -131,9 +133,17 @@ class RocksContainer(Container): def _lock(self, value): self._resources._lock = value + @ttl_cache(ttl=5) + def _refresh(self): + if not self.read_only: + return + assert self._db is not None + self._db.try_catch_up_with_primary() + @property def db(self) -> aimrocks.DB: if self._db is not None: + self._refresh() return self._db logger.debug(f'opening {self.path} as aimrocks db') @@ -144,15 +154,17 @@ class RocksContainer(Container): lock_cls = self.get_lock_cls() self._lock = lock_cls(self._lock_path, timeout) self._lock.acquire() + secondary_path = None else: self.optimize_for_read() - + secondary_path = tempfile.mkdtemp() self._db = aimrocks.DB(str(self.path), aimrocks.Options(**self._db_opts), - read_only=self.read_only) + read_only=self.read_only, secondary_path=secondary_path) return self._db + def finalize(self, index: Container): """Finalize the Container.