diff --git a/query/deployment.yaml b/query/deployment.yaml index c76f7727926..7e9a7118be7 100644 --- a/query/deployment.yaml +++ b/query/deployment.yaml @@ -16,6 +16,7 @@ spec: app: query hail.is/sha: "{{ code.sha }}" spec: + terminationGracePeriodSeconds: 28800 # 8 hours serviceAccountName: query {% if deploy %} priorityClassName: production diff --git a/query/query/query.py b/query/query/query.py index 9151f5f96e1..2e0d24c1c19 100644 --- a/query/query/query.py +++ b/query/query/query.py @@ -220,6 +220,15 @@ async def on_cleanup(app): await asyncio.gather(*(t for t in asyncio.all_tasks() if t is not asyncio.current_task())) +async def on_shutdown(_): + # Filter the asyncio.current_task(), because if we await + # the current task we'll end up in a deadlock + remaining_tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] + log.info(f"On shutdown request received, with {len(remaining_tasks)} remaining tasks") + await asyncio.wait(*remaining_tasks) + log.info("All tasks on shutdown have completed") + + def run(): app = web.Application() @@ -229,6 +238,7 @@ def run(): app.on_startup.append(on_startup) app.on_cleanup.append(on_cleanup) + app.on_shutdown.append(on_shutdown) deploy_config = get_deploy_config() web.run_app(