From 7533d6f663714bcf69b08c55308e501dec969a31 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 4 Oct 2022 13:50:30 -0700 Subject: [PATCH 1/6] [ds launcher] un-hijack PYTHONPATH --- src/accelerate/commands/launch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py index 982293a3310..b20b387cc89 100644 --- a/src/accelerate/commands/launch.py +++ b/src/accelerate/commands/launch.py @@ -573,7 +573,13 @@ def deepspeed_launcher(args): warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning) mixed_precision = "fp16" - current_env["PYTHONPATH"] = sys.executable + def env_path_add(env_var_name, new_path): + """extend path-based env variable with a new path""" + paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] + paths.append(str(new_path)) + return ":".join(paths) + + current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable) current_env["MIXED_PRECISION"] = str(mixed_precision) current_env["USE_DEEPSPEED"] = "true" current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage) From 03e455ec5ac3a04bce0242b191a1e94f8a854d24 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 4 Oct 2022 14:44:33 -0700 Subject: [PATCH 2/6] move to utils --- src/accelerate/commands/launch.py | 9 ++------- src/accelerate/utils/launch.py | 9 +++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py index b20b387cc89..b93e1f32d3a 100644 --- a/src/accelerate/commands/launch.py +++ b/src/accelerate/commands/launch.py @@ -45,6 +45,7 @@ ) from accelerate.utils.constants import DEEPSPEED_MULTINODE_LAUNCHERS from accelerate.utils.dataclasses import SageMakerDistributedType +from accelerate.utils.launch import env_var_path_add if is_rich_available(): @@ -573,13 +574,7 @@ def deepspeed_launcher(args): warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning) mixed_precision = "fp16" - def env_path_add(env_var_name, new_path): - """extend path-based env variable with a new path""" - paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] - paths.append(str(new_path)) - return ":".join(paths) - - current_env["PYTHONPATH"] = env_path_add("PYTHONPATH", sys.executable) + current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", sys.executable) current_env["MIXED_PRECISION"] = str(mixed_precision) current_env["USE_DEEPSPEED"] = "true" current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage) diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py index 31d71ad2128..1a57868b1d1 100644 --- a/src/accelerate/utils/launch.py +++ b/src/accelerate/utils/launch.py @@ -52,6 +52,15 @@ def _filter_args(args): return new_args +def env_var_path_add(env_var_name, new_path): + """ + Extends path-based env variable with a new path + """ + paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] + paths.append(str(new_path)) + return ":".join(paths) + + class PrepareForLaunch: """ Prepare a function that will launched in a distributed setup. From 0b5542d2b71f805142909bcc21266d4b598d374d Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 4 Oct 2022 16:22:42 -0700 Subject: [PATCH 3/6] improve doc, arg names --- src/accelerate/utils/launch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py index 1a57868b1d1..5dfc60a0156 100644 --- a/src/accelerate/utils/launch.py +++ b/src/accelerate/utils/launch.py @@ -52,12 +52,13 @@ def _filter_args(args): return new_args -def env_var_path_add(env_var_name, new_path): +def env_var_path_add(env_var_name, path_to_add): """ - Extends path-based env variable with a new path + Extends path-based environment variable's value with a new path and returns the updated value. It's up to the + caller to set it in os.environ. """ paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] - paths.append(str(new_path)) + paths.append(str(path_to_add)) return ":".join(paths) From 8cf35c36aae0ad383d1bf1cdb3dd53c70e3f4652 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Tue, 4 Oct 2022 16:23:20 -0700 Subject: [PATCH 4/6] fix --- src/accelerate/utils/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/accelerate/utils/launch.py b/src/accelerate/utils/launch.py index 5dfc60a0156..8642a441aff 100644 --- a/src/accelerate/utils/launch.py +++ b/src/accelerate/utils/launch.py @@ -54,7 +54,7 @@ def _filter_args(args): def env_var_path_add(env_var_name, path_to_add): """ - Extends path-based environment variable's value with a new path and returns the updated value. It's up to the + Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the caller to set it in os.environ. """ paths = [p for p in os.environ.get(env_var_name, "").split(":") if len(p) > 0] From 5ae6010b73dc40b37f8862b8e5d912b060d0180b Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 6 Oct 2022 08:51:45 -0700 Subject: [PATCH 5/6] Update src/accelerate/commands/launch.py Co-authored-by: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> --- src/accelerate/commands/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py index b93e1f32d3a..333b79d37dc 100644 --- a/src/accelerate/commands/launch.py +++ b/src/accelerate/commands/launch.py @@ -574,7 +574,7 @@ def deepspeed_launcher(args): warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning) mixed_precision = "fp16" - current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", sys.executable) + current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath('.')) current_env["MIXED_PRECISION"] = str(mixed_precision) current_env["USE_DEEPSPEED"] = "true" current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage) From 0c153c4e4df1d9f868d11dffbc93e951636c0da8 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Thu, 6 Oct 2022 09:07:24 -0700 Subject: [PATCH 6/6] style --- src/accelerate/commands/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py index 333b79d37dc..f348bb0a663 100644 --- a/src/accelerate/commands/launch.py +++ b/src/accelerate/commands/launch.py @@ -574,7 +574,7 @@ def deepspeed_launcher(args): warnings.warn('--fp16 flag is deprecated. Use "--mixed_precision fp16" instead.', DeprecationWarning) mixed_precision = "fp16" - current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath('.')) + current_env["PYTHONPATH"] = env_var_path_add("PYTHONPATH", os.path.abspath(".")) current_env["MIXED_PRECISION"] = str(mixed_precision) current_env["USE_DEEPSPEED"] = "true" current_env["DEEPSPEED_ZERO_STAGE"] = str(args.zero_stage)