Merge pull request #1104 from girder/reconnect-memcached

Reconnect to memcached if the connection fails

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Improvements
 - Better cache handling with Etags ([#1097](../../pull/1097))
 - Reduce duplicate computation of slow cached values ([#1100](../../pull/1100))
+- Reconnect to memcached if the connection fails ([#1102](../../pull/1102))
 
 ### Bug Fixes
 - Tile serving can bypass loading a source if it is in memory ([#1102](../../pull/1102))

large_image/cache_util/memcache.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #############################################################################
 
+import copy
 import threading
 import time
 from typing import Tuple
@@ -47,15 +48,13 @@ def __init__(self, url='127.0.0.1', username=None, password=None,
         if len(url) > 1:
             behaviors['remove_failed'] = 1
         # name mangling to override 'private variable' __data in cache
-        self._client = pylibmc.Client(
-            url, binary=True, username=username, password=password,
-            behaviors=behaviors)
+        self._clientParams = (url, dict(
+            binary=True, username=username, password=password, behaviors=behaviors))
+        self._client = pylibmc.Client(self._clientParams[0], **self._clientParams[1])
         if mustBeAvailable:
             # Try to set a value; this will throw an error if the server is
             # unreachable, so we don't bother trying to use it.
             self._client['large_image_cache_test'] = time.time()
-        self._clientParams = (url, dict(
-            binary=True, username=username, password=password, behaviors=behaviors))
 
     def __repr__(self):
         return "Memcache doesn't list its keys"
@@ -85,6 +84,7 @@ def __getitem__(self, key):
         except pylibmc.ServerDown:
             self.logError(pylibmc.ServerDown, config.getConfig('logprint').info,
                           'Memcached ServerDown')
+            self._reconnect()
             return self.__missing__(key)
         except pylibmc.Error:
             self.logError(pylibmc.Error, config.getConfig('logprint').exception,
@@ -109,6 +109,7 @@ def __setitem__(self, key, value):
         except pylibmc.ServerDown:
             self.logError(pylibmc.ServerDown, config.getConfig('logprint').info,
                           'Memcached ServerDown')
+            self._reconnect()
         except pylibmc.TooBig:
             pass
         except pylibmc.Error as exc:
@@ -130,9 +131,21 @@ def currsize(self):
     def maxsize(self):
         return self._getStat('limit_maxbytes')
 
+    def _reconnect(self):
+        try:
+            self._lastReconnectBackoff = getattr(self, '_lastReconnectBackoff', 2)
+            if time.time() - getattr(self, '_lastReconnect', 0) > self._lastReconnectBackoff:
+                config.getConfig('logprint').info('Trying to reconnect to memcached server')
+                self._client = pylibmc.Client(self._clientParams[0], **self._clientParams[1])
+                self._lastReconnectBackoff = min(self._lastReconnectBackoff + 1, 30)
+                self._lastReconnect = time.time()
+        except Exception:
+            pass
+
     def _blockingClient(self):
-        self._clientParams[1]['behaviors']['no_block'] = False
-        return pylibmc.Client(self._clientParams[0], **self._clientParams[1])
+        params = copy.deepcopy(self._clientParams)
+        params[1]['behaviors']['no_block'] = False
+        return pylibmc.Client(params[0], **params[1])
 
     def _getStat(self, key):
         try: