Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add redis as cache backend option #1469

Merged
merged 11 commits into from
May 24, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,10 @@ commands:
name: start memcached
command: |
docker run --rm -d -p 11211:11211 memcached -m 64
- run:
name: start redis
command: |
docker run --rm -d -p 6379:6379 redis
- run:
name: Use nvm
# see https://discuss.circleci.com/t/nvm-does-not-change-node-version-on-machine/28973/14
Expand Down
6 changes: 6 additions & 0 deletions docs/config_options.rst
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ Configuration parameters:

- ``cache_memcached_password``: A password for the memcached server. Default ``None``.

- ``cache_redis_url``: If tiles are cached in redis, the url or list of urls where the redis server is located. Default '127.0.0.1:6379'.

- ``cache_redis_username``: A username for the redis server. Default ``None``.

- ``cache_redis_password``: A password for the redis server. Default ``None``.

- ``cache_tilesource_memory_portion``: Tilesources are cached on open so that subsequent accesses can be faster. These use file handles and memory. This limits the maximum based on a memory estimation and using no more than 1 / (``cache_tilesource_memory_portion``) of the available memory.

- ``cache_tilesource_maximum``: If this is non-zero, this further limits the number of tilesources than can be cached to this value.
Expand Down
7 changes: 6 additions & 1 deletion large_image/cache_util/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,15 @@
from .cachefactory import CacheFactory, pickAvailableCache

MemCache: Any
RedisCache: Any
try:
from .memcache import MemCache
except ImportError:
MemCache = None
try:
from .rediscache import RedisCache
except ImportError:
RedisCache = None

Check warning on line 33 in large_image/cache_util/__init__.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/__init__.py#L32-L33

Added lines #L32 - L33 were not covered by tests

_cacheClearFuncs: List[Callable] = []

Expand Down Expand Up @@ -99,6 +104,6 @@
return info


__all__ = ('CacheFactory', 'getTileCache', 'isTileCacheSetup', 'MemCache',
__all__ = ('CacheFactory', 'getTileCache', 'isTileCacheSetup', 'MemCache', 'RedisCache',
'strhash', 'LruCacheMetaclass', 'pickAvailableCache', 'methodcache',
'CacheProperties')
3 changes: 3 additions & 0 deletions large_image/cache_util/cachefactory.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from .. import config
from ..exceptions import TileCacheError
from .memcache import MemCache
from .rediscache import RedisCache

# DO NOT MANUALLY ADD ANYTHING TO `_availableCaches`
# use entrypoints and let loadCaches fill in `_availableCaches`
Expand Down Expand Up @@ -59,6 +60,8 @@ def loadCaches(
if MemCache is not None:
# TODO: put this in an entry point for a new package
_availableCaches['memcached'] = MemCache
if RedisCache is not None:
_availableCaches['redis'] = RedisCache
# NOTE: `python` cache is viewed as a fallback and isn't listed in `availableCaches`


Expand Down
177 changes: 177 additions & 0 deletions large_image/cache_util/rediscache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
#############################################################################
# Copyright Kitware Inc.
#
# Licensed under the Apache License, Version 2.0 ( the "License" );
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#############################################################################

import pickle
import threading
import time
from typing import Any, Callable, Iterable, List, Optional, Sized, Tuple, TypeVar, Union, cast

from typing_extensions import Buffer

from .. import config
from .base import BaseCache

_VT = TypeVar('_VT')


class RedisCache(BaseCache):
"""Use redis as the backing cache."""

def __init__(
self, url: Union[str, List[str]] = '127.0.0.1:6379',
username: Optional[str] = None, password: Optional[str] = None,
getsizeof: Optional[Callable[[_VT], float]] = None,
mustBeAvailable: bool = False) -> None:
import redis
from redis.client import Redis

self.redis = redis
self._redisCls = Redis
super().__init__(0, getsizeof=getsizeof)
self._cache_key_prefix = 'large_image_'
self._clientParams = (f'redis://{url}', dict(
username=username, password=password, db=0, retry_on_timeout=1))
self._client: Redis = Redis.from_url(self._clientParams[0], **self._clientParams[1])
if mustBeAvailable:
# Try to ping server; this will throw an error if the server is
# unreachable, so we don't bother trying to use it.
self._client.ping()

def __repr__(self) -> str:
return "Redis doesn't list its keys"

Check warning on line 54 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L54

Added line #L54 was not covered by tests

def __iter__(self):
# return invalid iter
return None

Check warning on line 58 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L58

Added line #L58 was not covered by tests

def __len__(self) -> int:
# return invalid length
keys = self._client.keys(f'{self._cache_key_prefix}*')
return len(cast(Sized, keys))

Check warning on line 63 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L62-L63

Added lines #L62 - L63 were not covered by tests

def __contains__(self, key) -> bool:
# cache never contains key
_key = self._cache_key_prefix + self._hashKey(key)
return bool(self._client.exists(_key))

def __delitem__(self, key: str) -> None:
if not self.__contains__(key):
raise KeyError
_key = self._cache_key_prefix + self._hashKey(key)
self._client.delete(_key)

Check warning on line 74 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L72-L74

Added lines #L72 - L74 were not covered by tests

def __getitem__(self, key: str) -> Any:
_key = self._cache_key_prefix + self._hashKey(key)
try:
# must determine if tke key exists , otherwise cache_test can not be passed.
if not self.__contains__(key):
raise KeyError
return pickle.loads(cast(Buffer, self._client.get(_key)))
except KeyError:
return self.__missing__(key)
except self.redis.ConnectionError:
self.logError(self.redis.ConnectionError, config.getLogger('logprint').info,

Check warning on line 86 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L86

Added line #L86 was not covered by tests
'redis ConnectionError')
self._reconnect()
return self.__missing__(key)
except self.redis.RedisError:
self.logError(self.redis.RedisError, config.getLogger('logprint').exception,

Check warning on line 91 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L88-L91

Added lines #L88 - L91 were not covered by tests
'redis RedisError')
return self.__missing__(key)

Check warning on line 93 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L93

Added line #L93 was not covered by tests

def __setitem__(self, key: str, value: Any) -> None:
_key = self._cache_key_prefix + self._hashKey(key)
try:
self._client.set(_key, pickle.dumps(value))
except (TypeError, KeyError) as exc:
valueSize = value.shape if hasattr(value, 'shape') else (

Check warning on line 100 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L100

Added line #L100 was not covered by tests
value.size if hasattr(value, 'size') else (
len(value) if hasattr(value, '__len__') else None))
valueRepr = repr(value)

Check warning on line 103 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L103

Added line #L103 was not covered by tests
if len(valueRepr) > 500:
valueRepr = valueRepr[:500] + '...'
self.logError(

Check warning on line 106 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L105-L106

Added lines #L105 - L106 were not covered by tests
exc.__class__, config.getLogger('logprint').error,
'%s: Failed to save value (size %r) with key %s' % (
exc.__class__.__name__, valueSize, key))
except self.redis.ConnectionError:
self.logError(self.redis.ConnectionError, config.getLogger('logprint').info,

Check warning on line 111 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L110-L111

Added lines #L110 - L111 were not covered by tests
'redis ConnectionError')
self._reconnect()

Check warning on line 113 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L113

Added line #L113 was not covered by tests

@property
def curritems(self) -> int:
return cast(int, self._client.dbsize())

Check warning on line 117 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L117

Added line #L117 was not covered by tests

@property
def currsize(self) -> int:
return self._getStat('used_memory')

Check warning on line 121 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L121

Added line #L121 was not covered by tests

@property
def maxsize(self) -> int:
maxmemory = self._getStat('maxmemory')

Check warning on line 125 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L125

Added line #L125 was not covered by tests
if maxmemory:
return maxmemory

Check warning on line 127 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L127

Added line #L127 was not covered by tests
else:
return self._getStat('total_system_memory')

Check warning on line 129 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L129

Added line #L129 was not covered by tests

def _reconnect(self) -> None:
try:
self._lastReconnectBackoff = getattr(self, '_lastReconnectBackoff', 2)

Check warning on line 133 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L132-L133

Added lines #L132 - L133 were not covered by tests
if time.time() - getattr(self, '_lastReconnect', 0) > self._lastReconnectBackoff:
config.getLogger('logprint').info('Trying to reconnect to redis server')
self._client = self._redisCls.from_url(self._clientParams[0],

Check warning on line 136 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L135-L136

Added lines #L135 - L136 were not covered by tests
**self._clientParams[1])
self._lastReconnectBackoff = min(self._lastReconnectBackoff + 1, 30)
self._lastReconnect = time.time()
except Exception:
pass

Check warning on line 141 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L138-L141

Added lines #L138 - L141 were not covered by tests

def _getStat(self, key: str) -> int:
try:
stats = self._client.info()
value = cast(dict, stats)[key]
except Exception:
return 0
return value

Check warning on line 149 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L144-L149

Added lines #L144 - L149 were not covered by tests

def clear(self) -> None:
keys = self._client.keys(f'{self._cache_key_prefix}*')
if keys:
self._client.delete(*list(cast(Iterable[Any], keys)))

@staticmethod
def getCache() -> Tuple[Optional['RedisCache'], threading.Lock]:
cacheLock = threading.Lock()

# check if credentials and location exist, otherwise assume
# location is 127.0.0.1 (localhost) with no password
url = config.getConfig('cache_redis_url')
if not url:
url = '127.0.0.1:6379'

Check warning on line 164 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L164

Added line #L164 was not covered by tests
redisUsername = config.getConfig('cache_redis_username')
if not redisUsername:
redisUsername = None
redisPassword = config.getConfig('cache_redis_password')
if not redisPassword:
redisPassword = None
try:
cache = RedisCache(url, redisUsername, redisPassword,
mustBeAvailable=True)
except Exception:
config.getLogger().info('Cannot use redis for caching.')
cache = None

Check warning on line 176 in large_image/cache_util/rediscache.py

View check run for this annotation

Codecov / codecov/patch

large_image/cache_util/rediscache.py#L174-L176

Added lines #L174 - L176 were not covered by tests
return cache, cacheLock
4 changes: 3 additions & 1 deletion large_image/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,15 @@
'logprint': fallbackLogger,

# For tiles
'cache_backend': None, # 'python' or 'memcached'
'cache_backend': None, # 'python', 'redis' or 'memcached'
# 'python' cache can use 1/(val) of the available memory
'cache_python_memory_portion': 32,
# cache_memcached_url may be a list
'cache_memcached_url': '127.0.0.1',
'cache_memcached_username': None,
'cache_memcached_password': None,
'cache_redis_url': '127.0.0.1:6379',
'cache_redis_password': None,

# If set to False, the default will be to not cache tile sources. This has
# substantial performance penalties if sources are used multiple times, so
Expand Down
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ def prerelease_local_scheme(version):

extraReqs = {
'memcached': ['pylibmc>=1.5.1 ; platform_system != "Windows"'],
'redis': ['redis>=4.5.5'],
'converter': [f'large-image-converter{limit_version}'],
'colormaps': ['matplotlib'],
'tiledoutput': ['pyvips'],
Expand Down Expand Up @@ -74,7 +75,7 @@ def prerelease_local_scheme(version):
# The common packages are ones that will install on Ubuntu, OSX, and Windows
# from pypi with all needed dependencies.
extraReqs['common'] = list(set(itertools.chain.from_iterable(extraReqs[key] for key in {
'memcached', 'colormaps', 'performance',
'memcached', 'redis', 'colormaps', 'performance',
'deepzoom', 'dicom', 'multi', 'nd2', 'test', 'tifffile', 'zarr',
})) | {
f'large-image-source-pil[all]{limit_version}',
Expand Down
32 changes: 29 additions & 3 deletions test/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

import large_image.cache_util.cache
from large_image import config
from large_image.cache_util import (LruCacheMetaclass, MemCache, cachesClear,
cachesInfo, getTileCache, methodcache,
strhash)
from large_image.cache_util import (LruCacheMetaclass, MemCache, RedisCache,
cachesClear, cachesInfo, getTileCache,
methodcache, strhash)


class Fib:
Expand Down Expand Up @@ -51,6 +51,23 @@ def testCheckCacheMemcached():
assert val == 354224848179261915075


@pytest.mark.singular()
def testCacheRedis():
cache_test(RedisCache())


@pytest.mark.singular()
def testCheckCacheRedis():
cache = RedisCache()

cache_test(cache)

val = cache['(2,)']
assert val == 1
val = cache['(100,)']
assert val == 354224848179261915075


def testBadMemcachedUrl():
# go though and check if all 100 fib numbers are in cache
# it is stored in cache as ('fib', #)
Expand Down Expand Up @@ -79,6 +96,15 @@ def testGetTileCacheMemcached():
assert isinstance(tileCache, MemCache)


@pytest.mark.singular()
def testGetTileCacheRedis():
large_image.cache_util.cache._tileCache = None
large_image.cache_util.cache._tileLock = None
config.setConfig('cache_backend', 'redis')
tileCache, tileLock = getTileCache()
assert isinstance(tileCache, RedisCache)


class TestClass:
def testLRUThreadSafety(self):
# The cachetools LRU cache is not thread safe, and if two threads ask
Expand Down
8 changes: 8 additions & 0 deletions test/test_cached_tiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,14 @@ def setup_class(cls):
config.setConfig('cache_backend', 'memcached')


class TestRedisCache(LargeImageCachedTilesTest):
@classmethod
def setup_class(cls):
large_image.cache_util.cache._tileCache = None
large_image.cache_util.cache._tileLock = None
config.setConfig('cache_backend', 'redis')


class TestPythonCache(LargeImageCachedTilesTest):
@classmethod
def setup_class(cls):
Expand Down
2 changes: 2 additions & 0 deletions test/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ def testConfigFunctions():
assert getConfig('cache_backend') == 'python'
setConfig('cache_backend', 'memcached')
assert getConfig('cache_backend') == 'memcached'
setConfig('cache_backend', 'redis')
assert getConfig('cache_backend') == 'redis'
setConfig('cache_backend', None)
assert getConfig('cache_backend') is None
assert getConfig('unknown', 'python') == 'python'
2 changes: 2 additions & 0 deletions test/test_files/sample.girder.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ cache_python_memory_portion = 32
cache_memcached_url = "127.0.0.1"
cache_memcached_username = None
cache_memcached_password = None
cache_redis_url = "127.0.0.1:6379"
cache_redis_password = None
# The tilesource cache uses the lesser of a value based on available file
# handles, the memory portion, and the maximum (if not 0)
cache_tilesource_memory_portion = 8
Expand Down
1 change: 1 addition & 0 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ toxworkdir = {toxinidir}/build/tox
passenv = PYTEST_*,DICOMWEB_TEST_URL,DICOMWEB_TEST_TOKEN
extras =
memcached
redis
performance
setenv =
PIP_FIND_LINKS=https://girder.github.io/large_image_wheels
Expand Down