Merge pull request #1469 from nipeone/master

Add redis as cache backend option

.circleci/config.yml

@@ -103,6 +103,10 @@ commands:
           name: start memcached
           command: |
             docker run --rm -d -p 11211:11211 memcached -m 64
+      - run:
+          name: start redis
+          command: |
+            docker run --rm -d -p 6379:6379 redis
       - run:
           name: Use nvm
           # see https://discuss.circleci.com/t/nvm-does-not-change-node-version-on-machine/28973/14

docs/config_options.rst

@@ -19,6 +19,12 @@ Configuration parameters:
 
 - ``cache_memcached_password``: A password for the memcached server.  Default ``None``.
 
+- ``cache_redis_url``: If tiles are cached in redis, the url or list of urls where the redis server is located.  Default '127.0.0.1:6379'.
+
+- ``cache_redis_username``: A username for the redis server.  Default ``None``.
+
+- ``cache_redis_password``: A password for the redis server.  Default ``None``.
+
 - ``cache_tilesource_memory_portion``: Tilesources are cached on open so that subsequent accesses can be faster.  These use file handles and memory.  This limits the maximum based on a memory estimation and using no more than 1 / (``cache_tilesource_memory_portion``) of the available memory.
 
 - ``cache_tilesource_maximum``: If this is non-zero, this further limits the number of tilesources than can be cached to this value.

large_image/cache_util/__init__.py

@@ -22,10 +22,15 @@
 from .cachefactory import CacheFactory, pickAvailableCache
 
 MemCache: Any
+RedisCache: Any
 try:
     from .memcache import MemCache
 except ImportError:
     MemCache = None
+try:
+    from .rediscache import RedisCache
+except ImportError:
+    RedisCache = None
 
 _cacheClearFuncs: List[Callable] = []
 
@@ -99,6 +104,6 @@ def cachesInfo(*args, **kwargs) -> Dict[str, Dict[str, int]]:
     return info
 
 
-__all__ = ('CacheFactory', 'getTileCache', 'isTileCacheSetup', 'MemCache',
+__all__ = ('CacheFactory', 'getTileCache', 'isTileCacheSetup', 'MemCache', 'RedisCache',
            'strhash', 'LruCacheMetaclass', 'pickAvailableCache', 'methodcache',
            'CacheProperties')

large_image/cache_util/cachefactory.py

@@ -24,6 +24,7 @@
 from .. import config
 from ..exceptions import TileCacheError
 from .memcache import MemCache
+from .rediscache import RedisCache
 
 # DO NOT MANUALLY ADD ANYTHING TO `_availableCaches`
 #  use entrypoints and let loadCaches fill in `_availableCaches`
@@ -59,6 +60,8 @@ def loadCaches(
     if MemCache is not None:
         # TODO: put this in an entry point for a new package
         _availableCaches['memcached'] = MemCache
+    if RedisCache is not None:
+        _availableCaches['redis'] = RedisCache
     # NOTE: `python` cache is viewed as a fallback and isn't listed in `availableCaches`
 
 

large_image/cache_util/rediscache.py

@@ -0,0 +1,177 @@
+#############################################################################
+#  Copyright Kitware Inc.
+#
+#  Licensed under the Apache License, Version 2.0 ( the "License" );
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#############################################################################
+
+import pickle
+import threading
+import time
+from typing import Any, Callable, Iterable, List, Optional, Sized, Tuple, TypeVar, Union, cast
+
+from typing_extensions import Buffer
+
+from .. import config
+from .base import BaseCache
+
+_VT = TypeVar('_VT')
+
+
+class RedisCache(BaseCache):
+    """Use redis as the backing cache."""
+
+    def __init__(
+            self, url: Union[str, List[str]] = '127.0.0.1:6379',
+            username: Optional[str] = None, password: Optional[str] = None,
+            getsizeof: Optional[Callable[[_VT], float]] = None,
+            mustBeAvailable: bool = False) -> None:
+        import redis
+        from redis.client import Redis
+
+        self.redis = redis
+        self._redisCls = Redis
+        super().__init__(0, getsizeof=getsizeof)
+        self._cache_key_prefix = 'large_image_'
+        self._clientParams = (f'redis://{url}', dict(
+            username=username, password=password, db=0, retry_on_timeout=1))
+        self._client: Redis = Redis.from_url(self._clientParams[0], **self._clientParams[1])
+        if mustBeAvailable:
+            # Try to ping server; this will throw an error if the server is
+            # unreachable, so we don't bother trying to use it.
+            self._client.ping()
+
+    def __repr__(self) -> str:
+        return "Redis doesn't list its keys"
+
+    def __iter__(self):
+        # return invalid iter
+        return None
+
+    def __len__(self) -> int:
+        # return invalid length
+        keys = self._client.keys(f'{self._cache_key_prefix}*')
+        return len(cast(Sized, keys))
+
+    def __contains__(self, key) -> bool:
+        # cache never contains key
+        _key = self._cache_key_prefix + self._hashKey(key)
+        return bool(self._client.exists(_key))
+
+    def __delitem__(self, key: str) -> None:
+        if not self.__contains__(key):
+            raise KeyError
+        _key = self._cache_key_prefix + self._hashKey(key)
+        self._client.delete(_key)
+
+    def __getitem__(self, key: str) -> Any:
+        _key = self._cache_key_prefix + self._hashKey(key)
+        try:
+            # must determine if tke key exists , otherwise cache_test can not be passed.
+            if not self.__contains__(key):
+                raise KeyError
+            return pickle.loads(cast(Buffer, self._client.get(_key)))
+        except KeyError:
+            return self.__missing__(key)
+        except self.redis.ConnectionError:
+            self.logError(self.redis.ConnectionError, config.getLogger('logprint').info,
+                          'redis ConnectionError')
+            self._reconnect()
+            return self.__missing__(key)
+        except self.redis.RedisError:
+            self.logError(self.redis.RedisError, config.getLogger('logprint').exception,
+                          'redis RedisError')
+            return self.__missing__(key)
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        _key = self._cache_key_prefix + self._hashKey(key)
+        try:
+            self._client.set(_key, pickle.dumps(value))
+        except (TypeError, KeyError) as exc:
+            valueSize = value.shape if hasattr(value, 'shape') else (
+                value.size if hasattr(value, 'size') else (
+                    len(value) if hasattr(value, '__len__') else None))
+            valueRepr = repr(value)
+            if len(valueRepr) > 500:
+                valueRepr = valueRepr[:500] + '...'
+            self.logError(
+                exc.__class__, config.getLogger('logprint').error,
+                '%s: Failed to save value (size %r) with key %s' % (
+                    exc.__class__.__name__, valueSize, key))
+        except self.redis.ConnectionError:
+            self.logError(self.redis.ConnectionError, config.getLogger('logprint').info,
+                          'redis ConnectionError')
+            self._reconnect()
+
+    @property
+    def curritems(self) -> int:
+        return cast(int, self._client.dbsize())
+
+    @property
+    def currsize(self) -> int:
+        return self._getStat('used_memory')
+
+    @property
+    def maxsize(self) -> int:
+        maxmemory = self._getStat('maxmemory')
+        if maxmemory:
+            return maxmemory
+        else:
+            return self._getStat('total_system_memory')
+
+    def _reconnect(self) -> None:
+        try:
+            self._lastReconnectBackoff = getattr(self, '_lastReconnectBackoff', 2)
+            if time.time() - getattr(self, '_lastReconnect', 0) > self._lastReconnectBackoff:
+                config.getLogger('logprint').info('Trying to reconnect to redis server')
+                self._client = self._redisCls.from_url(self._clientParams[0],
+                                                       **self._clientParams[1])
+                self._lastReconnectBackoff = min(self._lastReconnectBackoff + 1, 30)
+                self._lastReconnect = time.time()
+        except Exception:
+            pass
+
+    def _getStat(self, key: str) -> int:
+        try:
+            stats = self._client.info()
+            value = cast(dict, stats)[key]
+        except Exception:
+            return 0
+        return value
+
+    def clear(self) -> None:
+        keys = self._client.keys(f'{self._cache_key_prefix}*')
+        if keys:
+            self._client.delete(*list(cast(Iterable[Any], keys)))
+
+    @staticmethod
+    def getCache() -> Tuple[Optional['RedisCache'], threading.Lock]:
+        cacheLock = threading.Lock()
+
+        # check if credentials and location exist, otherwise assume
+        # location is 127.0.0.1 (localhost) with no password
+        url = config.getConfig('cache_redis_url')
+        if not url:
+            url = '127.0.0.1:6379'
+        redisUsername = config.getConfig('cache_redis_username')
+        if not redisUsername:
+            redisUsername = None
+        redisPassword = config.getConfig('cache_redis_password')
+        if not redisPassword:
+            redisPassword = None
+        try:
+            cache = RedisCache(url, redisUsername, redisPassword,
+                               mustBeAvailable=True)
+        except Exception:
+            config.getLogger().info('Cannot use redis for caching.')
+            cache = None
+        return cache, cacheLock

large_image/config.py

@@ -26,13 +26,15 @@
     'logprint': fallbackLogger,
 
     # For tiles
-    'cache_backend': None,  # 'python' or 'memcached'
+    'cache_backend': None,  # 'python', 'redis' or 'memcached'
     # 'python' cache can use 1/(val) of the available memory
     'cache_python_memory_portion': 32,
     # cache_memcached_url may be a list
     'cache_memcached_url': '127.0.0.1',
     'cache_memcached_username': None,
     'cache_memcached_password': None,
+    'cache_redis_url': '127.0.0.1:6379',
+    'cache_redis_password': None,
 
     # If set to False, the default will be to not cache tile sources.  This has
     # substantial performance penalties if sources are used multiple times, so

setup.py

@@ -36,6 +36,7 @@ def prerelease_local_scheme(version):
 
 extraReqs = {
     'memcached': ['pylibmc>=1.5.1 ; platform_system != "Windows"'],
+    'redis': ['redis>=4.5.5'],
     'converter': [f'large-image-converter{limit_version}'],
     'colormaps': ['matplotlib'],
     'tiledoutput': ['pyvips'],
@@ -74,7 +75,7 @@ def prerelease_local_scheme(version):
 # The common packages are ones that will install on Ubuntu, OSX, and Windows
 # from pypi with all needed dependencies.
 extraReqs['common'] = list(set(itertools.chain.from_iterable(extraReqs[key] for key in {
-    'memcached', 'colormaps', 'performance',
+    'memcached', 'redis', 'colormaps', 'performance',
     'deepzoom', 'dicom', 'multi', 'nd2', 'test', 'tifffile', 'zarr',
 })) | {
     f'large-image-source-pil[all]{limit_version}',

test/test_cache.py

@@ -7,9 +7,9 @@
 
 import large_image.cache_util.cache
 from large_image import config
-from large_image.cache_util import (LruCacheMetaclass, MemCache, cachesClear,
-                                    cachesInfo, getTileCache, methodcache,
-                                    strhash)
+from large_image.cache_util import (LruCacheMetaclass, MemCache, RedisCache,
+                                    cachesClear, cachesInfo, getTileCache,
+                                    methodcache, strhash)
 
 
 class Fib:
@@ -51,6 +51,23 @@ def testCheckCacheMemcached():
     assert val == 354224848179261915075
 
 
+@pytest.mark.singular()
+def testCacheRedis():
+    cache_test(RedisCache())
+
+
+@pytest.mark.singular()
+def testCheckCacheRedis():
+    cache = RedisCache()
+
+    cache_test(cache)
+
+    val = cache['(2,)']
+    assert val == 1
+    val = cache['(100,)']
+    assert val == 354224848179261915075
+
+
 def testBadMemcachedUrl():
     # go though and check if all 100 fib numbers are in cache
     # it is stored in cache as ('fib', #)
@@ -79,6 +96,15 @@ def testGetTileCacheMemcached():
     assert isinstance(tileCache, MemCache)
 
 
+@pytest.mark.singular()
+def testGetTileCacheRedis():
+    large_image.cache_util.cache._tileCache = None
+    large_image.cache_util.cache._tileLock = None
+    config.setConfig('cache_backend', 'redis')
+    tileCache, tileLock = getTileCache()
+    assert isinstance(tileCache, RedisCache)
+
+
 class TestClass:
     def testLRUThreadSafety(self):
         # The cachetools LRU cache is not thread safe, and if two threads ask

test/test_cached_tiles.py

@@ -161,6 +161,14 @@ def setup_class(cls):
         config.setConfig('cache_backend', 'memcached')
 
 
+class TestRedisCache(LargeImageCachedTilesTest):
+    @classmethod
+    def setup_class(cls):
+        large_image.cache_util.cache._tileCache = None
+        large_image.cache_util.cache._tileLock = None
+        config.setConfig('cache_backend', 'redis')
+
+
 class TestPythonCache(LargeImageCachedTilesTest):
     @classmethod
     def setup_class(cls):

test/test_config.py

@@ -7,6 +7,8 @@ def testConfigFunctions():
     assert getConfig('cache_backend') == 'python'
     setConfig('cache_backend', 'memcached')
     assert getConfig('cache_backend') == 'memcached'
+    setConfig('cache_backend', 'redis')
+    assert getConfig('cache_backend') == 'redis'
     setConfig('cache_backend', None)
     assert getConfig('cache_backend') is None
     assert getConfig('unknown', 'python') == 'python'

test/test_files/sample.girder.cfg

@@ -8,6 +8,8 @@ cache_python_memory_portion = 32
 cache_memcached_url = "127.0.0.1"
 cache_memcached_username = None
 cache_memcached_password = None
+cache_redis_url = "127.0.0.1:6379"
+cache_redis_password = None
 # The tilesource cache uses the lesser of a value based on available file
 # handles, the memory portion, and the maximum (if not 0)
 cache_tilesource_memory_portion = 8

tox.ini

@@ -13,6 +13,7 @@ toxworkdir = {toxinidir}/build/tox
 passenv = PYTEST_*,DICOMWEB_TEST_URL,DICOMWEB_TEST_TOKEN
 extras =
   memcached
+  redis
   performance
 setenv =
   PIP_FIND_LINKS=https://girder.github.io/large_image_wheels