From ef60b9501617b18ac4a83675249aebc60aa07f70 Mon Sep 17 00:00:00 2001 From: Martin Pitt Date: Thu, 22 Sep 2022 07:20:23 +0200 Subject: [PATCH] appservice: Retry redis connection The pod starts up in parallel with appservice, so redis may not yet be ready. This caused an occasional CI failure, which can be reproduced with delaying the redis startup in webconsoleapp-local.yaml: command: ["sh", "-ec", "sleep 2; redis-server"] --- appservice/multiplexer.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/appservice/multiplexer.py b/appservice/multiplexer.py index 4e506dd..1615b7b 100644 --- a/appservice/multiplexer.py +++ b/appservice/multiplexer.py @@ -6,7 +6,8 @@ import uuid import httpx -import redis.asyncio as redis +import redis.exceptions +import redis.asyncio import uvicorn import websockets @@ -25,7 +26,7 @@ # states: wait_target or running SESSIONS = {} -REDIS = redis.Redis(host=os.environ['REDIS_SERVICE_HOST'], port=int(os.environ.get('REDIS_SERVICE_PORT', '6379'))) +REDIS = redis.asyncio.Redis(host=os.environ['REDIS_SERVICE_HOST'], port=int(os.environ.get('REDIS_SERVICE_PORT', '6379'))) logger = logging.getLogger('multiplexer') app = Starlette() @@ -217,7 +218,17 @@ async def init_sessions(): global SESSIONS pubsub = REDIS.pubsub() - await pubsub.subscribe('sessions') + # wait for Redis service to be up + for retry in range(10): + try: + await pubsub.subscribe('sessions') + break + except redis.exceptions.ConnectionError as e: + logger.warning('Failed to connect to Redis, retry %i: %s', retry, e) + await asyncio.sleep(retry * retry + 1) + else: + raise RuntimeError('timed out trying to connect to Redis') + asyncio.create_task(watch_redis(pubsub)) sessions = await REDIS.get('sessions')