From 384a38418bce7b1fa57091765585a8f7c50e4846 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Fri, 10 Jan 2025 12:59:34 -0800 Subject: [PATCH] test set_spawn_method and handle exceptions --- backend/onyx/background/celery/apps/heavy.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/onyx/background/celery/apps/heavy.py b/backend/onyx/background/celery/apps/heavy.py index 1a8b6587447..c49ccfa751b 100644 --- a/backend/onyx/background/celery/apps/heavy.py +++ b/backend/onyx/background/celery/apps/heavy.py @@ -60,7 +60,17 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None: all_start_methods: list[str] = multiprocessing.get_all_start_methods() logger.info(f"Multiprocessing all start methods: {all_start_methods}") - multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn + try: + multiprocessing.set_start_method("spawn") # fork is unsafe, set to spawn + except Exception: + logger.info("multiprocessing.set_start_method exceptioned.") + try: + multiprocessing.set_start_method( + "spawn", force=True + ) # fork is unsafe, set to spawn + except Exception: + logger.info("multiprocessing.set_start_method force=True exceptioned.") + logger.info( f"Multiprocessing selected start method: {multiprocessing.get_start_method()}" )