Fix multinode with GPU ids when each node has 1 (#1127)

* Fix multinode * Assert * Reverse logic * Use <= and not "not" Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> * All on a single statement --------- Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
huggingface · Mar 1, 2023 · 9b5877d · 9b5877d
1 parent 586941d
commit 9b5877d
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/src/accelerate/commands/launch.py b/src/accelerate/commands/launch.py
@@ -776,8 +776,16 @@ def _validate_launch_command(args):
                     args.gpu_ids = defaults.gpu_ids
                 else:
                     args.gpu_ids = "all"
-            if len(args.gpu_ids.split(",")) < 2 and args.multi_gpu and (args.gpu_ids != "all"):
-                args.multi_gpu = False
+            if (
+                len(args.gpu_ids.split(",")) < 2
+                and (args.gpu_ids != "all")
+                and args.multi_gpu
+                and args.num_machines <= 1
+            ):
+                raise ValueError(
+                    "Less than two GPU ids were configured and tried to run on on multiple GPUs. "
+                    "Please ensure at least two are specified for `--gpu_ids`, or use `--gpu_ids='all'`."
+                )
         if defaults.compute_environment == ComputeEnvironment.LOCAL_MACHINE:
             # Update args with the defaults
             for name, attr in defaults.__dict__.items():