Add documentation for docker + multinode

mila-iqia · Jan 17, 2025 · 3adab0f · 3adab0f
1 parent 40c35bd
commit 3adab0f
Show file tree

Hide file tree

Showing 2 changed files with 80 additions and 2 deletions.
diff --git a/docs/Contributing/recipes.rst b/docs/Contributing/recipes.rst
@@ -255,6 +255,74 @@ Nightly
 Multi Node & Docker
 ^^^^^^^^^^^^^^^^^^^
 
+1. Create a system file with the right docker configuration
+
+.. code-block:: yaml
+
+   system:
+      # Default arch
+      arch: cuda
+
+      # sshkey used in remote milabench operations
+      sshkey: ~/.ssh/id_ed25519
+
+      # Configures how to use docker 
+      docker:
+         executable: podman
+         image: ghcr.io/mila-iqia/milabench:${system.arch}-nightly
+         base: /tmp/workspace
+         args: [
+            -it, --rm, --ipc=host, --gpus=all, --network, host, --privileged,
+            -e, MILABENCH_HF_TOKEN=<TOKEN>,
+            -v, "${system.docker.base}/data:/milabench/envs/data",
+            -v, "${system.docker.base}/runs:/milabench/envs/runs",
+         ]
+
+      # Nodes list
+      nodes:
+         # Alias used to reference the node
+         - name: manager
+            ip: 192.168.11.11
+            port: 5000
+            # Use this node as the master node or not
+            main: true
+            # User to use in remote milabench operations
+            user: manager
+
+         - name: node1
+            ip: 192.168.11.12
+            main: false
+            user: username
+
+
+2. Use ``milabench docker`` to suggest the command to use to execute the benchmark
+
+.. code-block:: bash
+
+   cp system.yaml /tmp/workspace/data/system.yaml
+   milabench docker --system system.yaml
+
+
+.. code-block::
+
+   podman run -it --rm --ipc=host --gpus=all --network host --privileged   \
+      -e MILABENCH_HF_TOKEN=<TOKEN>                                        \
+      -v /tmp/workspace/data:/milabench/envs/data                          \
+      -v /tmp/workspace/runs:/milabench/envs/runs                          \
+      -e OMP_NUM_THREADS='12' ghcr.io/mila-iqia/milabench:cuda-nightly     \
+      milabench prepare --system /milabench/envs/data/system.yaml
+   
+   podman run -it --rm --ipc=host --gpus=all --network host --privileged   \
+      -e MILABENCH_HF_TOKEN=<TOKEN>                                        \
+      -v /tmp/workspace/data:/milabench/envs/data                          \
+      -v /tmp/workspace/runs:/milabench/envs/runs                          \
+      -e OMP_NUM_THREADS='12' ghcr.io/mila-iqia/milabench:cuda-nightly     \
+      milabench run --system /milabench/envs/data/system.yaml
+
+
+3. execute prepare
+
+4. execute run
 
 
 Example Reports

diff --git a/milabench/cli/docker.py b/milabench/cli/docker.py
@@ -42,14 +42,24 @@ def cli_docker(args=None):
     system = pack.config["system"]
     config = DockerConfig(**system.get("docker"))
 
+    # TODO: how can we generate this
+    extra_args = [
+        "--system", "/milabench/envs/data/system.yaml"
+    ]
+    print()
+    print()
+
     # milabench prepare
-    plan = DockerRunCommand(CmdCommand(pack, "milabench", "prepare"), config)
+    plan = DockerRunCommand(CmdCommand(pack, "milabench", "prepare", *extra_args), config)
 
     for pack, argv, _ in plan.commands():
         print(" ".join(argv))
 
+    print()
     # milabench run
-    plan = DockerRunCommand(CmdCommand(pack, "milabench", "run"), config)
+    plan = DockerRunCommand(CmdCommand(pack, "milabench", "run", *extra_args), config)
 
     for pack, argv, _ in plan.commands():
         print(" ".join(argv))
+
+    print()