Test merge: run collaborator in docker

docs/mermaid/director_aggregator.mmd

@@ -0,0 +1,22 @@
+sequenceDiagram
+    participant D as Director
+    participant A as Aggregator
+    rect rgb(0, 255, 0,.1)
+        Note over D: An Experiment's start
+        D->>D: Get next experiment<br>from Experiment Registry
+        opt Docker specific logic
+            D->>D: Create aggregator docker build<br>context from experiment workspace.<br>(Add Dockerfile and execution script<br>to context specific for aggregator)
+            D->>D: Build aggregator docker image
+            D->>D: Create aggregator docker container
+            D->>D: Run aggregator docker container
+            D->>D: Monitor aggregator docker container
+        end
+        loop every round
+            A->>D: Send last/best model to director
+            D->>D: Save model on director
+        end
+        opt Docker specific logic
+            D->>D: Delete aggregator docker container<br>when experiment was finished
+        end
+    end
+    Note over D: The Experiment ended. <br> The Federation keeps existing.

docs/mermaid/director_envoy.mmd

@@ -0,0 +1,55 @@
+sequenceDiagram
+    participant N as NoteBook
+    participant D as Director
+    participant E as Envoy
+    rect rgb(0, 255, 0,.1)
+        Note over D,E: A Federation startup process
+        D->D: Starts
+        E->E: Adapting a dataset
+        E->E: Starts
+        Note over D,E: Exchange certs
+        E-->>D: Connects using FQDN and pwd
+        E-->>D: Communicates dataset info to
+        D-->D: Keeps a list of connected Envoys
+        D-->D: Ensures unified data interface
+    end
+    Note over D,E: We consider a Federation set up
+    rect rgb(0, 255, 0,.1)
+    Note over N,D: Create new experiment
+        N->>N: Prepare experiment in Notebook
+        N->>N: Connect to federation
+        N->>N: Run experiment
+        N->>D: Send Experiment workspace
+        D->>D: Create new experiment
+        D->>D: Add experiment to regestry
+    end
+    rect rgb(0, 255, 0,.1)
+        Note over D,E: An Experiment's start
+        D->>D: Get next experiment<br>from Experiment Registry
+        opt Docker specific logic
+            D->>D: Create aggregator docker build<br>context from experiment workspace.<br>(Add Dockerfile and execution script<br>to context specific for aggregator)
+            D->>D: Build aggregator docker image
+            D->>D: Create aggregator docker container
+            D->>D: Run aggregator docker container
+            D->>D: Monitor aggregator docker container
+        end
+        E->>D: WaitExperiment
+        D-->>E: Send Experiment name
+        E->>D: GetExperimentData(experiment_name)
+        D-->>E: Send Experiment workspace
+        opt Docker specific logic
+            E->>E: Create collaborator docker build<br>context from Experiment workspace.<br>(Add Dockerfile and execution script<br>to context specific for collaborator)
+            E->>E: Build collaborator docker image
+            E->>E: Create collaborator docker container
+            E->>E: Run collaborator docker container
+            E->>E: Monitor collaborator docker container
+        end
+            Note over D,E: Wait for last round finished
+        opt Docker specific logic
+            E->>E: Delete collaborator docker container<br>when experiment was finished
+            D->>D: Delete aggregator docker container<br>when experiment was finished
+        end
+    end
+    N->>D: Get best model
+    D-->>N: Send best model
+    Note over D,E: The Experiment ended. <br> The Federation keeps existing.

openfl-docker/Dockerfile.aggregator

@@ -0,0 +1,7 @@
+FROM python:3.8
+
+RUN pip install --upgrade pip
+RUN pip install git+https://github.com/dmitryagapov/openfl.git@feature/run_collaborator_in_docker
+
+COPY . /code
+WORKDIR /code

openfl-docker/Dockerfile.collaborator

@@ -0,0 +1,10 @@
+FROM python:3.8
+
+RUN pip install --upgrade pip
+RUN pip install git+https://github.com/dmitryagapov/openfl.git@feature/run_collaborator_in_docker
+
+WORKDIR /code
+COPY ./requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .

openfl-tutorials/interactive_api/MXNet_landmarks/director/start_director_docker.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx director start --disable-tls -c director_config.yaml --use-docker

openfl-tutorials/interactive_api/MXNet_landmarks/envoy/start_envoy_docker.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+ENVOY_NAME=$1
+SHARD_CONF=$2
+
+fx envoy start -n "$ENVOY_NAME" --disable-tls --envoy-config-path "$SHARD_CONF" -dh localhost -dp 50051

openfl-tutorials/interactive_api/PyTorch_DogsCats_ViT/envoy/envoy_config_one.yaml

@@ -1,13 +1,16 @@
 params:
-  cuda_devices: [0]
-  
+  cuda_devices: [ 0 ]
+
 optional_plugin_components:
- cuda_device_monitor:
-   template: openfl.plugins.processing_units_monitor.pynvml_monitor.PynvmlCUDADeviceMonitor
-   settings: []
+  cuda_device_monitor:
+    template: openfl.plugins.processing_units_monitor.pynvml_monitor.PynvmlCUDADeviceMonitor
+    settings: [ ]
 
 shard_descriptor:
   template: dogs_cats_shard_descriptor.DogsCatsShardDescriptor
+  volumes:
+    - '~/.kaggle/kaggle.json'
+    - './data'
   params:
     data_folder: data
     rank_worldsize: 1,2

openfl-tutorials/interactive_api/PyTorch_Kvasir_UNet/director/director_config.yaml

@@ -1,6 +1,15 @@
 settings:
   listen_host: localhost
   listen_port: 50050
-  sample_shape: ['300', '400', '3']
-  target_shape: ['300', '400']
+  sample_shape: [ '300', '400', '3' ]
+  target_shape: [ '300', '400' ]
   envoy_health_check_period: 5  # in seconds
+  docker:
+    env:
+      http_proxy:
+      https_proxy:
+      no_proxy:
+    buildargs:
+      HTTP_PROXY:
+      HTTPS_PROXY:
+      NO_PROXY:

openfl-tutorials/interactive_api/PyTorch_Kvasir_UNet/director/start_director_docker.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx director start --disable-tls -c director_config.yaml --use-docker

openfl-tutorials/interactive_api/PyTorch_Kvasir_UNet/envoy/envoy_config.yaml

@@ -1,14 +1,27 @@
 params:
-  cuda_devices: [0,2]
-
+  cuda_devices: [ 0, 2 ]
+  docker:
+    env:
+      http_proxy:
+      https_rpoxy:
+      no_proxy:
+    buildargs:
+      HTTP_PROXY:
+      HTTPS_PROXY:
+      NO_PROXY:
+    volumes:
+      - './kvasir_data'
+
 optional_plugin_components:
- cuda_device_monitor:
-   template: openfl.plugins.processing_units_monitor.pynvml_monitor.PynvmlCUDADeviceMonitor
-   settings: []
+  cuda_device_monitor:
+    template: openfl.plugins.processing_units_monitor.pynvml_monitor.PynvmlCUDADeviceMonitor
+    settings: [ ]
 
 shard_descriptor:
   template: kvasir_shard_descriptor.KvasirShardDescriptor
   params:
     data_folder: kvasir_data
     rank_worldsize: 1,10
     enforce_image_hw: '300,400'
+
+

openfl-tutorials/interactive_api/PyTorch_Kvasir_UNet/envoy/envoy_config_no_gpu.yaml

@@ -1,7 +1,11 @@
 params:
-  cuda_devices: []
+  cuda_devices: [ ]
+  docker_env:
+    http_proxy:
+    https_rpoxy:
+    no_proxy:
 
-optional_plugin_components: {}
+optional_plugin_components: { }
 
 shard_descriptor:
   template: kvasir_shard_descriptor.KvasirShardDescriptor

openfl-tutorials/interactive_api/PyTorch_Kvasir_UNet/envoy/start_envoy_docker.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx envoy start -n env_one --disable-tls --envoy-config-path envoy_config.yaml -dh localhost -dp 50050 --use-docker

openfl-tutorials/interactive_api/numpy_linear_regression/envoy/linreg_shard_descriptor.py

@@ -33,7 +33,7 @@ def get_dataset(self, dataset_type: str) -> np.ndarray:
         """
         Return a shard dataset by type.
 
-        A simple list with elements (x, y) implemets the Shard Dataset interface.
+        A simple list with elements (x, y) implements the Shard Dataset interface.
         """
         if dataset_type == 'train':
             return self.data[:self.n_samples // 2]