diff --git a/.env b/.env index 43770b2e..f1ad1d9f 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -TAG=v0.2.0 +TAG=v0.3.0 SERVER_NAME=federatedai/fedlcm-server SERVER_IMG=${SERVER_NAME}:${TAG} diff --git a/.github/workflows/CodeQL.yml b/.github/workflows/CodeQL.yml new file mode 100644 index 00000000..e8fb4c00 --- /dev/null +++ b/.github/workflows/CodeQL.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + workflow_dispatch: + push: + branches: [ "main" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/check-crlf.yaml b/.github/workflows/check-crlf.yaml new file mode 100644 index 00000000..8f2abbc0 --- /dev/null +++ b/.github/workflows/check-crlf.yaml @@ -0,0 +1,14 @@ +name: Check CRLF + +on: pull_request + +jobs: + check-CRLF: + name: Check CRLF action + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@main + + - name: check-crlf + uses: erclu/check-crlf@v1 \ No newline at end of file diff --git a/.github/workflows/fedlcm-docker-build-and-push.yaml b/.github/workflows/fedlcm-docker-build-and-push.yaml new file mode 100644 index 00000000..957f4a7d --- /dev/null +++ b/.github/workflows/fedlcm-docker-build-and-push.yaml @@ -0,0 +1,50 @@ +name: FedLCM docker build and push + +on: + push: + # Publish `main` as Docker `latest` image. + branches: + - main + + # Publish `v1.2.3` tags as releases. + tags: + - v* + +jobs: + # no test is required + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@main + + - name: Prepare the TAG + id: prepare-the-tag + run: | + # strip git ref prefix from version + TAG="" + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [ $VERSION = "main" ]; then + TAG="latest" + fi + echo "TAG=${TAG}" + echo "TAG=${TAG}" >> $GITHUB_OUTPUT + - name: Build image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + make docker-build + + - name: Log into DockerHub + run: docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + make docker-push diff --git a/.github/workflows/fedlcm-unit-test.yaml b/.github/workflows/fedlcm-unit-test.yaml new file mode 100644 index 00000000..e23fe173 --- /dev/null +++ b/.github/workflows/fedlcm-unit-test.yaml @@ -0,0 +1,25 @@ +name: FedLCM server unit test + +on: + pull_request: + paths: + - ".github/workflows/fedlcm-unit-test.yaml" + - "server/**" + - “pkg/**” +jobs: + Unit-test: + name: Unit Test + runs-on: ubuntu-latest + steps: + - name: Setup + uses: actions/setup-go@v1 + with: + go-version: 1.19 + id: go + + - name: Code + uses: actions/checkout@main + + - name: Unit Test + run: | + make server-unittest \ No newline at end of file diff --git a/.github/workflows/fml-manager-docker-build-and push.yaml b/.github/workflows/fml-manager-docker-build-and push.yaml new file mode 100644 index 00000000..4a58ee66 --- /dev/null +++ b/.github/workflows/fml-manager-docker-build-and push.yaml @@ -0,0 +1,52 @@ +name: FML-Manager docker build and push + +on: + push: + # Publish `main` as Docker `latest` image. + branches: + - main + + # Publish `v1.2.3` tags as releases. + tags: + - v* + +jobs: + # no test is required + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@main + + - name: Prepare the TAG + id: prepare-the-tag + run: | + # strip git ref prefix from version + TAG="" + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [ $VERSION = "main" ]; then + TAG="latest" + fi + echo "TAG=${TAG}" + echo "TAG=${TAG}" >> $GITHUB_OUTPUT + - name: Build image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + cd fml-manager + make docker-build + + - name: Log into DockerHub + run: docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + cd fml-manager + make docker-push diff --git a/.github/workflows/fml-manager-unit-test.yaml b/.github/workflows/fml-manager-unit-test.yaml new file mode 100644 index 00000000..3fc5c1bf --- /dev/null +++ b/.github/workflows/fml-manager-unit-test.yaml @@ -0,0 +1,26 @@ +name: FML-Manager server unit test + +on: + pull_request: + paths: + - ".github/workflows/fml-manager-unit-test.yaml" + - "fml-manager/server/**" + +jobs: + Unit-test: + name: Unit Test + runs-on: ubuntu-latest + steps: + - name: Setup + uses: actions/setup-go@v1 + with: + go-version: 1.19 + id: go + + - name: Code + uses: actions/checkout@main + + - name: Unit Test + run: | + cd fml-manager + make server-unittest \ No newline at end of file diff --git a/.github/workflows/site-portal-docker-build-and-push.yaml b/.github/workflows/site-portal-docker-build-and-push.yaml new file mode 100644 index 00000000..443566d3 --- /dev/null +++ b/.github/workflows/site-portal-docker-build-and-push.yaml @@ -0,0 +1,52 @@ +name: Site-Portal docker build and push + +on: + push: + # Publish `main` as Docker `latest` image. + branches: + - main + + # Publish `v1.2.3` tags as releases. + tags: + - v* + +jobs: + # no test is required + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@main + + - name: Prepare the TAG + id: prepare-the-tag + run: | + # strip git ref prefix from version + TAG="" + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [ $VERSION = "main" ]; then + TAG="latest" + fi + echo "TAG=${TAG}" + echo "TAG=${TAG}" >> $GITHUB_OUTPUT + - name: Build image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + cd site-portal + make docker-build + + - name: Log into DockerHub + run: docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push image + run: | + TAG=${{steps.prepare-the-tag.outputs.TAG}} + if [ ! -z "$TAG" ]; then + export TAG=$TAG + fi + cd site-portal + make docker-push diff --git a/.github/workflows/site-portal-unit-test.yaml b/.github/workflows/site-portal-unit-test.yaml new file mode 100644 index 00000000..66b87be7 --- /dev/null +++ b/.github/workflows/site-portal-unit-test.yaml @@ -0,0 +1,26 @@ +name: Site-Portal server unit test + +on: + pull_request: + paths: + - ".github/workflows/site-portal-unit-test.yaml" + - "site-portal/server/**" + +jobs: + Unit-test: + name: Unit Test + runs-on: ubuntu-latest + steps: + - name: Setup + uses: actions/setup-go@v1 + with: + go-version: 1.19 + id: go + + - name: Code + uses: actions/checkout@main + + - name: Unit Test + run: | + cd site-portal + make server-unittest \ No newline at end of file diff --git a/Makefile b/Makefile index 93313266..73d0341b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: all clean format swag swag-bin server-unittest server frontend run upgrade openfl-device-agent release RELEASE_VERSION ?= ${shell git describe --tags} -TAG ?= v0.2.0 +TAG ?= v0.3.0 SERVER_NAME ?= federatedai/fedlcm-server SERVER_IMG ?= ${SERVER_NAME}:${TAG} @@ -78,7 +78,7 @@ ifeq (, $(shell which swag)) SWAG_BIN_TMP_DIR=$$(mktemp -d) ;\ cd $$SWAG_BIN_TMP_DIR ;\ go mod init tmp ;\ - go get -u github.com/swaggo/swag/cmd/swag ;\ + go install github.com/swaggo/swag/cmd/swag@v1.8.7 ;\ rm -rf $$SWAG_BIN_TMP_DIR ;\ } SWAG_BIN=$(GOBIN)/swag diff --git a/cmd/device-agent/cli/register.go b/cmd/device-agent/cli/register.go index b3cc6319..6b4849bd 100644 --- a/cmd/device-agent/cli/register.go +++ b/cmd/device-agent/cli/register.go @@ -38,6 +38,7 @@ type envoyRegistrationConfig struct { Labels valueobject.Labels `json:"labels"` SkipCommonPythonFiles bool `json:"skip_common_python_files" yaml:"skipCommonPythonFiles"` EnablePSP bool `json:"enable_psp" yaml:"enablePSP"` + LessPrivileged bool `json:"less_privileged" yaml:"lessPrivileged"` RegistryConfig valueobject.KubeRegistryConfig `json:"registry_config" yaml:"registryConfig"` } diff --git a/doc/OpenFL_Guide.md b/doc/OpenFL_Guide.md index 5f0387ae..4239dadd 100644 --- a/doc/OpenFL_Guide.md +++ b/doc/OpenFL_Guide.md @@ -1,6 +1,6 @@ # Managing OpenFL Federations -This document provides an end-to-end guide to set up an OpenFL federation using FedLCM service. Currently, director-based mode is supported. +This document provides an end-to-end guide to set up an OpenFL v1.5 federation using FedLCM service. Currently, director-based mode is supported. The overall deployment architecture is in below diagram:
@@ -12,7 +12,8 @@ The high-level steps are 1. FedLCM deploys the KubeFATE service into a central K8s cluster and use this KubeFATE service to deploy OpenFL director component, which includes a director service and a Jupyter Lab service. 2. On each device/node/machine that will do the actual FML training using their local data, a device-agent program is launched to register this device/node/machine to FedLCM. The registration information contains a KubeConfig file so the FedLCM can further operate the K8s cluster on the device/node/machine. 3. FedLCM deploys the KubeFATE service and the OpenFL envoy components onto the device/node/machine's K8s cluster. -4. the envoy is configured with the address of the director service, so it will register to the director service upon started. +4. The envoy is configured with the address of the director service, so it will register to the director service upon started. +5. Users can use the deployed Jupyter Lab service to work with the OpenFL federation to run FL experiments. > Currently the core images for FedLCM's OpenFL federations are not made public yet, please talk with the maintainer for the access details. @@ -59,12 +60,12 @@ With the service running and CA configured, we can start create the OpenFL feder ### Add Kubernetes Infrastructure Kubernetes clusters are considered as Infrastructures in the FedLCM service. All the other installation are performed on these K8s clusters. To deploy the director, one must firstly add the target K8s into the system. -Go to the "Infrastructure" section and click the "NEW" button. What needs to be filled is the KubeConfig content that FedLCM will use to connect the K8s cluster. +Go to the "Infrastructure" section and click the "NEW" button. What needs to be filled is the kubeconfig content that FedLCM will use to connect the K8s cluster. -**Even though for FATE we can support namespace-wide admin, the user configured in the KubeConfig for OpenFL should have the privilege to create all core K8s resource including namespace, deployment, configmap, role, secret, etc. We haven't tested the exact rules. If not sure, use the cluster-admin ClusterRole** +**By default, FedLCM expects the kubeconfig to have cluster-admin permission to operate the cluster. An alternative is using namespace wide admin permission. To use this less privileged config, enable the "Limited to certain namespaces" option and input the namespace(s) this kubeconfig can only use**
- +
Click "TEST" to make sure the cluster is available. And "SUBMIT" to save the new infrastructure. @@ -74,7 +75,7 @@ The "FATE Registry Configuration" section is for FATE usage and not for OpenFL, ### Install KubeFATE Endpoint In the "Endpoint" section, we can install KubeFATE service onto the K8s infrastructure. And later it can be used to deploy OpenFL components. -To add a new KubeFATE endpoint, select the infrastructure and the system will try to find if there is already a KubeFATE service running. +To add a new KubeFATE endpoint, select the infrastructure (and the namespace if the kubeconfig has less privilege) and then system will try to find if there is already a KubeFATE service running. If yes, the system will add the KubeFATE into its database directly. If no, the system will provide an installation step as shown below:
@@ -150,7 +151,7 @@ class FedLCMDummyShardDescriptor(DummyShardDescriptor): f'target shape: {self.target_shape}' ) """Return the dataset description.""" - return 'This is dummy data shard descriptor provided by FedLCM project. You should implement your own data ' + return 'This is dummy data shard descriptor provided by FedLCM project. You should implement your own data ' \ 'loader to load your local data for each Envoy.' ``` @@ -167,6 +168,7 @@ After the federation is created, we can create the director. Click "NEW" under t Several things to note: * Try to give a unique name and namespace for the director as it may cause some issue if the name and namespace conflicts with existing ones in the cluster. +* If the selected endpoint and infrastructure is using less privileged kubeconfig, the namespace is predefined and cannot be changed. * It is suggested to choose "Install certificates for me" in the Certificate section. Only select "I will manually install certificates" if you want to import your own certificate instead of using the CA to create a new one. Refer to the OpenFL helm chart guide on how to import existing one. * Choose "NodePort" if your cluster doesn't have any controller that can handle `LoadBalancer` type of service. * If your cluster doesn't enable [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/), you don't have to enable it in the "Pod Security Policy Configuration". @@ -186,59 +188,37 @@ You can click into the director details page and keep refreshing it. If things w Now, we can open up the deployed Jupyter Lab system by clicking the Jupyter Notebook link, input the password we just configured and open a notebook we want to use, or create a new notebook where we can write our own code. * For this example we use the `interactive_api/Tensorflow_MNIST/workspace/Tensorflow_MNIST.ipynb` notebook. -* If the federation is configured with the default Unbounded Shard Descriptor, you can use `interactive_api/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb` as an example on how to put real data reading logic in the `DataInterface`. +* If the federation is configured with the default Unbounded Shard Descriptor, you can use [Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb](./examples/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb) as an example on how to put real data reading logic in the `DataInterface`. Just upload this file to the Jupyter Lab instance and follow the guide there. -The content in the notebook is from OpenFL's [official repo](https://github.com/intel/openfl/tree/develop/openfl-tutorials). We assume you have basic knowledge on how to use the OpenFL sdk to work with the director API. +The existing content in the notebook service is from OpenFL's [official repo](https://github.com/intel/openfl/tree/develop/openfl-tutorials). We assume you have basic knowledge on how to use the OpenFL sdk to work with the director API. -For the `federation` creation part, most of the examples are using below code: +For the `federation` creation part, to connect to the director from the deployed Jupyter Lab instance, use the following code: ```python # Create a federation from openfl.interface.interactive_api.federation import Federation -# please use the same identificator that was used in signed certificate client_id = 'api' -cert_dir = 'cert' -director_node_fqdn = 'localhost' -director_port=50051 -# 1) Run with API layer - Director mTLS -# If the user wants to enable mTLS their must provide CA root chain, and signed key pair to the federation interface -# cert_chain = f'{cert_dir}/root_ca.crt' -# api_certificate = f'{cert_dir}/{client_id}.crt' -# api_private_key = f'{cert_dir}/{client_id}.key' - -# federation = Federation( -# client_id=client_id, -# director_node_fqdn=director_node_fqdn, -# director_port=director_port, -# cert_chain=cert_chain, -# api_cert=api_certificate, -# api_private_key=api_private_key -# ) - -# -------------------------------------------------------------------------------------------------------------------- - -# 2) Run with TLS disabled (trusted environment) -# Federation can also determine local fqdn automatically +director_node_fqdn = 'director' +director_port = 50051 +cert_chain = '/openfl/workspace/cert/root_ca.crt' +api_cert = '/openfl/workspace/cert/notebook.crt' +api_private_key = '/openfl/workspace/cert/priv.key' + federation = Federation( - client_id=client_id, - director_node_fqdn=director_node_fqdn, - director_port=director_port, - tls=False + client_id=client_id, + director_node_fqdn=director_node_fqdn, + director_port=director_port, + cert_chain=cert_chain, + api_cert=api_cert, + api_private_key=api_private_key ) ``` -But we actually don't need that to be this complicated, since we have internally configured the SDK to work with the deployed director by default. -So to create a federation that represent the director, use below code is sufficient: - -```python -# Create a federation -from openfl.interface.interactive_api.federation import Federation - -federation = Federation() -``` +The certificates, keys, director listening port and other settings are pre-configured by FedLCM so we can use them as shown above. And if we call the federation's `get_shard_registry()` API, we will notice the returned data is an empty dict. It is expected as current there is no "client (envoy)" created yet. + We can move on now. ## Register Device/Node/Machine to the Federation @@ -291,6 +271,7 @@ chartUUID: "type: string, default: " labels: "type: map, default:, the labels for the envoy will be a merge from this field and labels of the token" skipCommonPythonFiles: "type: bool, default: false, if true, the python shard descriptor files configured in the federation will not be used, and user will need to manually import the python files." enablePSP: "type: bool, default: false, if true, the deployd envoy pod will have PSP associated" +lessPrivileged: "type: bool, default: false, if ture, all the components will be installed in one namespace. Make sure the namespace already exists and the kubeconfig has the permission to operate in this namespace" registryConfig: useRegistry: "type: bool, default false" registry: "type: string, default , if set, the image will be /fedlcm-openfl:v0.1.0" @@ -303,6 +284,8 @@ registryConfig: > The registryConfig will affect both KubeFATE and OpenFL related images. Make sure you have all the images in your customized registry. +**If the kubeconfig used for Envoy registration only has namespace wide admin permissions, we must specify the namespace and set `lessPrivileged` to true in the extra-config file** + ### Start Registration and Envoy Deployment Assume on the device/node/machine we have the `openfl-device-agent` program, we can start the registration process @@ -379,4 +362,64 @@ Now, we have finished the whole process of installing FedLCM to deploying OpenFL ## Caveats * If there are errors when running experiment in the envoy side, the experiment may become "never finished". This is OpenFL's own issue. Currently, the workaround is restart the director and envoy. * There is no "unregister" support in OpenFL yet so if we delete an envoy, it may still show in the director's `federation.get_shard_registry()` API. But its status is offline so director won't send future experiment to this removed envoy. -* For the KubeConfig used in the infrastructure, We haven't tested what are exactly the minimal requirement permissions. \ No newline at end of file +* To facilitate the envoy's container image registry configuration, we can set the `LIFECYCLEMANAGER_OPENFL_ENVOY_REGISTRY_OVERRIDE` environment variable for FedLCM service, which will take precedence of the registry url configured in the `extra-config` file used by the device agent. + +### Preparing the fedlcm-openfl Image Locally & Using You Own Registry +The Director, Envoy and Jupyter Lab container deployed by FedLCM all use a same container image. This image is built using OpenFL's official dockerfile but with small modifications. Here is how to build this image locally and use your own image registry: + +1. Checkout OpenFL's v1.5 release code +```bash +git clone -b v1.5 https://github.com/securefederatedai/openfl.git +cd openfl +``` + +2. Run the following command to add the modification +```bash +patch -p1 </fedlcm-openfl:v0.3.0 -f openfl-docker/Dockerfile.base . +docker push /fedlcm-openfl:v0.3.0 +``` + +For example, assuming we want to use my dockerhub account "foobar", then the command would look like: +```bash +docker build -t foobar/fedlcm-openfl:v0.3.0 -f openfl-docker/Dockerfile.base . +docker push foobar/fedlcm-openfl:v0.3.0 +``` + +4. When deploying directors, we need to set the registry url to `foobar`. +5. When registering envoys, we need to configure the registry in `--extra-config` or set the `LIFECYCLEMANAGER_OPENFL_ENVOY_REGISTRY_OVERRIDE` environment variable of FedLCM service to `foobar`. + +With the above steps, we will be using our locally built image from our own container image registry. \ No newline at end of file diff --git a/doc/examples/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb b/doc/examples/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb new file mode 100644 index 00000000..49971b00 --- /dev/null +++ b/doc/examples/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM/Tensorflow_MNIST_With_Dummy_Envoy_Shard_FedLCM.ipynb @@ -0,0 +1,542 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "26fdd9ed", + "metadata": {}, + "source": [ + "# Federated Tensorflow Mnist Tutorial With Dummy Envoy Shard Descriptor\n", + "\n", + "This tutorial is based on the \"Tensorflow_MNIST\" tutorial but with changes to demostrate how to use the dummy shard descriptor. By default, OpenFL federation created by FedLCM will only use this dummy shard descriptor. And user have to specify how envoy can retreive local data by defining the `DataInterface`.\n", + "\n", + "**Understanding of the original \"Tensorflow_MNIST\" is highly recommended!**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0570122", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Install dependencies if not already installed\n", + "!python -m pip install tensorflow==2.8" + ] + }, + { + "cell_type": "markdown", + "id": "6479191e-bb78-44cd-9462-8e420de6aa63", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06915d29-07bc-47e2-8d4a-ff73428ac104", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "print('TensorFlow', tf.__version__)" + ] + }, + { + "cell_type": "markdown", + "id": "246f9c98", + "metadata": { + "tags": [] + }, + "source": [ + "## Connect to the Federation\n", + "\n", + "This cell connects this notebook to the Federation.\n", + "\n", + "Note *the parameters provided in the cell is for the Jupyter Lab instance deployed along with the director by FedLCM. Change the parameters if necessary if you are running this notebook in other environments.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d657e463", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create a federation\n", + "from openfl.interface.interactive_api.federation import Federation\n", + "\n", + "client_id = 'api'\n", + "director_node_fqdn = 'director'\n", + "director_port = 50051\n", + "cert_chain = '/openfl/workspace/cert/root_ca.crt'\n", + "api_cert = '/openfl/workspace/cert/notebook.crt'\n", + "api_private_key = '/openfl/workspace/cert/priv.key'\n", + "\n", + "federation = Federation(\n", + " client_id=client_id,\n", + " director_node_fqdn=director_node_fqdn,\n", + " director_port=director_port,\n", + " cert_chain=cert_chain,\n", + " api_cert=api_cert,\n", + " api_private_key=api_private_key\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "74c57e78-dfc9-47f7-8a4c-96fdc48f64fc", + "metadata": { + "tags": [] + }, + "source": [ + "## Query Datasets from Shard Registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47dcfab3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "shard_registry = federation.get_shard_registry()\n", + "shard_registry" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2a6c237", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# First, request a dummy_shard_desc that holds information about the federated dataset \n", + "# If the shard descriptor is the default dummy one, the shape of the sample and target will be \"1\", which is not the actual data shape\n", + "dummy_shard_desc = federation.get_dummy_shard_descriptor(size=10)\n", + "dummy_shard_dataset = dummy_shard_desc.get_dataset('train')\n", + "sample, target = dummy_shard_dataset[0]\n", + "f\"Sample shape: {sample.shape}, target shape: {target.shape}\"" + ] + }, + { + "cell_type": "markdown", + "id": "cc0dbdbd", + "metadata": {}, + "source": [ + "## Describing FL experimen" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc88700a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "from openfl.interface.interactive_api.experiment import TaskInterface\n", + "from openfl.interface.interactive_api.experiment import ModelInterface\n", + "from openfl.interface.interactive_api.experiment import FLExperiment" + ] + }, + { + "cell_type": "markdown", + "id": "3b468ae1", + "metadata": {}, + "source": [ + "### Register model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06545bbb", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Define model\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n", + " tf.keras.layers.MaxPooling2D((2, 2)),\n", + " tf.keras.layers.BatchNormalization(),\n", + " tf.keras.layers.Conv2D(64, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n", + " tf.keras.layers.MaxPooling2D((2, 2)),\n", + " tf.keras.layers.BatchNormalization(),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(10, activation=None),\n", + "], name='simplecnn')\n", + "model.summary()\n", + "\n", + "# Define optimizer\n", + "optimizer = tf.optimizers.Adam(learning_rate=1e-3)\n", + "\n", + "# Loss and metrics. These will be used later.\n", + "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n", + "train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n", + "val_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n", + "\n", + "# Create ModelInterface\n", + "framework_adapter = 'openfl.plugins.frameworks_adapters.keras_adapter.FrameworkAdapterPlugin'\n", + "MI = ModelInterface(model=model, optimizer=optimizer, framework_plugin=framework_adapter)" + ] + }, + { + "cell_type": "markdown", + "id": "b0979470", + "metadata": {}, + "source": [ + "### Register dataset\n", + "\n", + "This is the main difference between this tutorial and the original \"Tensorflow_MNIST\" tutorial. Instead of configuring each Envoy with a specific shard descriptor class, the dummy one is configuraed for them, and we move the data retrieval logic into this subclass from `DataInterface`. For other tasks not using the MNIST dataset, we can write our own local data retrival logic here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8c9eb50", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "import numpy as np\n", + "from tensorflow.keras.utils import Sequence\n", + "from openfl.interface.interactive_api.shard_descriptor import ShardDataset\n", + "\n", + "from openfl.interface.interactive_api.experiment import DataInterface\n", + "\n", + "class MnistShardDataset(ShardDataset):\n", + " \"\"\"Mnist Shard dataset class.\"\"\"\n", + "\n", + " def __init__(self, x, y, rank=1, worldsize=1):\n", + " \"\"\"Initialize TinyImageNetDataset.\"\"\"\n", + " self.rank = rank\n", + " self.worldsize = worldsize\n", + " self.x = x[self.rank - 1::self.worldsize]\n", + " self.y = y[self.rank - 1::self.worldsize]\n", + "\n", + " def __getitem__(self, index: int):\n", + " \"\"\"Return an item by the index.\"\"\"\n", + " return self.x[index], self.y[index]\n", + "\n", + " def __len__(self):\n", + " \"\"\"Return the len of the dataset.\"\"\"\n", + " return len(self.x)\n", + " \n", + "\n", + "class DataGenerator(Sequence):\n", + "\n", + " def __init__(self, data_set, batch_size):\n", + " self.data_set = data_set\n", + " self.batch_size = batch_size\n", + " self.indices = np.arange(len(data_set))\n", + " self.on_epoch_end()\n", + "\n", + " def __len__(self):\n", + " return len(self.indices) // self.batch_size\n", + "\n", + " def __getitem__(self, index):\n", + " index = self.indices[index * self.batch_size:(index + 1) * self.batch_size]\n", + " batch = [self.indices[k] for k in index]\n", + "\n", + " X, y = self.data_set[batch]\n", + " return X, y\n", + "\n", + " def on_epoch_end(self):\n", + " np.random.shuffle(self.indices)\n", + "\n", + "\n", + "class MnistFedDataset(DataInterface):\n", + "\n", + " def __init__(self, **kwargs):\n", + " super().__init__(**kwargs)\n", + "\n", + " @property\n", + " def shard_descriptor(self):\n", + " return self._shard_descriptor\n", + "\n", + " @shard_descriptor.setter\n", + " def shard_descriptor(self, shard_descriptor):\n", + " \"\"\"\n", + " Describe per-collaborator procedures or sharding.\n", + "\n", + " This method will be called during a collaborator initialization.\n", + " Local shard_descriptor will be set by Envoy.\n", + " \"\"\"\n", + " self._shard_descriptor = shard_descriptor\n", + " \n", + " (x_train, y_train), (x_test, y_test) = self.download_data()\n", + " self.train_set = MnistShardDataset(x_train, y_train)\n", + " self.valid_set = MnistShardDataset(x_test, y_test)\n", + " \n", + " def __getitem__(self, index):\n", + " return self.shard_descriptor[index]\n", + "\n", + " def __len__(self):\n", + " return len(self.shard_descriptor)\n", + " \n", + " def get_train_loader(self):\n", + " \"\"\"\n", + " Output of this method will be provided to tasks with optimizer in contract\n", + " \"\"\"\n", + " if self.kwargs['train_bs']:\n", + " batch_size = self.kwargs['train_bs']\n", + " else:\n", + " batch_size = 32\n", + " return DataGenerator(self.train_set, batch_size=batch_size)\n", + "\n", + " def get_valid_loader(self):\n", + " \"\"\"\n", + " Output of this method will be provided to tasks without optimizer in contract\n", + " \"\"\"\n", + " if self.kwargs['valid_bs']:\n", + " batch_size = self.kwargs['valid_bs']\n", + " else:\n", + " batch_size = 32\n", + " \n", + " return DataGenerator(self.valid_set, batch_size=batch_size)\n", + "\n", + " def get_train_data_size(self):\n", + " \"\"\"\n", + " Information for aggregation\n", + " \"\"\"\n", + " \n", + " return len(self.train_set)\n", + "\n", + " def get_valid_data_size(self):\n", + " \"\"\"\n", + " Information for aggregation\n", + " \"\"\"\n", + " return len(self.valid_set)\n", + " \n", + " def download_data(self):\n", + " \"\"\"Download prepared dataset.\"\"\"\n", + " local_file_path = 'mnist.npz'\n", + " mnist_url = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz'\n", + " response = requests.get(mnist_url)\n", + " with open(local_file_path, 'wb') as f:\n", + " f.write(response.content)\n", + "\n", + " with np.load(local_file_path) as f:\n", + " x_train, y_train = f['x_train'], f['y_train']\n", + " x_test, y_test = f['x_test'], f['y_test']\n", + " x_train = np.reshape(x_train, (-1, 28, 28, 1))\n", + " x_test = np.reshape(x_test, (-1, 28, 28, 1))\n", + "\n", + " os.remove(local_file_path) # remove mnist.npz\n", + " print('Mnist data was loaded!')\n", + " return (x_train, y_train), (x_test, y_test)" + ] + }, + { + "cell_type": "markdown", + "id": "b0dfb459", + "metadata": {}, + "source": [ + "### Create Mnist federated dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4af5c4c2", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "fed_dataset = MnistFedDataset(train_bs=64, valid_bs=512)" + ] + }, + { + "cell_type": "markdown", + "id": "849c165b", + "metadata": {}, + "source": [ + "## Define and register FL tasks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b9649385", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import time\n", + "\n", + "\n", + "\n", + "TI = TaskInterface()\n", + "\n", + "# from openfl.interface.aggregation_functions import AdagradAdaptiveAggregation # Uncomment this lines to use \n", + "# agg_fn = AdagradAdaptiveAggregation(model_interface=MI, learning_rate=0.4) # Adaptive Federated Optimization\n", + "# @TI.set_aggregation_function(agg_fn) # alghorithm!\n", + "# # See details in the:\n", + "# # https://arxiv.org/abs/2003.00295\n", + "\n", + "@TI.register_fl_task(model='model', data_loader='train_dataset', device='device', optimizer='optimizer') \n", + "def train(model, train_dataset, optimizer, device, loss_fn=loss_fn, warmup=False):\n", + " start_time = time.time()\n", + "\n", + " # Iterate over the batches of the dataset.\n", + " for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n", + " with tf.GradientTape() as tape:\n", + " logits = model(x_batch_train, training=True)\n", + " loss_value = loss_fn(y_batch_train, logits)\n", + " grads = tape.gradient(loss_value, model.trainable_weights)\n", + " optimizer.apply_gradients(zip(grads, model.trainable_weights))\n", + "\n", + " # Update training metric.\n", + " train_acc_metric.update_state(y_batch_train, logits)\n", + "\n", + " # Log every 200 batches.\n", + " if step % 200 == 0:\n", + " print(\n", + " \"Training loss (for one batch) at step %d: %.4f\"\n", + " % (step, float(loss_value))\n", + " )\n", + " print(\"Seen so far: %d samples\" % ((step + 1) * 64))\n", + " if warmup:\n", + " break\n", + "\n", + " # Display metrics at the end of each epoch.\n", + " train_acc = train_acc_metric.result()\n", + " print(\"Training acc over epoch: %.4f\" % (float(train_acc),))\n", + "\n", + " # Reset training metrics at the end of each epoch\n", + " train_acc_metric.reset_states()\n", + "\n", + " \n", + " return {'train_acc': train_acc,}\n", + "\n", + "\n", + "@TI.register_fl_task(model='model', data_loader='val_dataset', device='device') \n", + "def validate(model, val_dataset, device):\n", + " # Run a validation loop at the end of each epoch.\n", + " for x_batch_val, y_batch_val in val_dataset:\n", + " val_logits = model(x_batch_val, training=False)\n", + " # Update val metrics\n", + " val_acc_metric.update_state(y_batch_val, val_logits)\n", + " val_acc = val_acc_metric.result()\n", + " val_acc_metric.reset_states()\n", + " print(\"Validation acc: %.4f\" % (float(val_acc),))\n", + " \n", + " return {'validation_accuracy': val_acc,}" + ] + }, + { + "cell_type": "markdown", + "id": "8f0ebf2d", + "metadata": {}, + "source": [ + "## Time to start a federated learning experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d41b7896", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# create an experimnet in federation\n", + "experiment_name = 'mnist_experiment'\n", + "fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f2074ca-9dcc-48ad-93fe-be4f65479b7b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# print the default federated learning plan\n", + "import openfl.native as fx\n", + "print(fx.get_plan(fl_plan=fl_experiment.plan))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41b44de9", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# The following command zips the workspace and python requirements to be transfered to collaborator nodes\n", + "fl_experiment.start(model_provider=MI, \n", + " task_keeper=TI,\n", + " data_loader=fed_dataset,\n", + " rounds_to_train=5,\n", + " opt_treatment='CONTINUE_GLOBAL',\n", + " override_config={'aggregator.settings.db_store_rounds': 1, 'compression_pipeline.template': 'openfl.pipelines.KCPipeline', 'compression_pipeline.settings.n_clusters': 2})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01fa7cea", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "fl_experiment.stream_metrics()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba13c3d4-bc2f-4bdb-86f0-5a72d827d9c8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/doc/images/fedlcm-new-infra.jpg b/doc/images/fedlcm-new-infra.jpg deleted file mode 100644 index 023fc590..00000000 Binary files a/doc/images/fedlcm-new-infra.jpg and /dev/null differ diff --git a/doc/images/fedlcm-new-infra.png b/doc/images/fedlcm-new-infra.png new file mode 100644 index 00000000..65f040bd Binary files /dev/null and b/doc/images/fedlcm-new-infra.png differ diff --git a/docker-compose.yml b/docker-compose.yml index be5165f3..0b0d7af0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,7 @@ services: postgres: image: postgres:13.3 volumes: - - ./output/data/postgres:/var/lib/postgresql + - ./output/data/postgres:/var/lib/postgresql/data ports: - "5432:5432" restart: always diff --git a/fml-manager/.env b/fml-manager/.env index b6aa0c6a..843db873 100644 --- a/fml-manager/.env +++ b/fml-manager/.env @@ -1,4 +1,4 @@ -TAG=v0.2.0 +TAG=v0.3.0 SERVER_NAME=federatedai/fml-manager-server SERVER_IMG=${SERVER_NAME}:${TAG} \ No newline at end of file diff --git a/fml-manager/Makefile b/fml-manager/Makefile index bc098bfe..74a1a67a 100644 --- a/fml-manager/Makefile +++ b/fml-manager/Makefile @@ -1,7 +1,7 @@ .PHONY: all clean format swag swag-bin server-unittest server run RELEASE_VERSION ?= ${shell git describe --tags} -TAG ?= v0.2.0 +TAG ?= v0.3.0 SERVER_NAME ?= federatedai/fml-manager-server SERVER_IMG ?= ${SERVER_NAME}:${TAG} diff --git a/fml-manager/server/api/job.go b/fml-manager/server/api/job.go index d46d435a..2249d731 100644 --- a/fml-manager/server/api/job.go +++ b/fml-manager/server/api/job.go @@ -61,14 +61,14 @@ func (controller *JobController) Route(r *gin.RouterGroup) { } // handleJobCreation process a job creation request -// @Summary Process job creation -// @Tags Job -// @Produce json -// @Param project body service.JobRemoteJobCreationRequest true "job creation request" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /job/create [post] +// @Summary Process job creation +// @Tags Job +// @Produce json +// @Param project body service.JobRemoteJobCreationRequest true "job creation request" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /job/create [post] func (controller *JobController) handleJobCreation(c *gin.Context) { if err := func() error { creationRequest := &service.JobRemoteJobCreationRequest{} @@ -91,15 +91,15 @@ func (controller *JobController) handleJobCreation(c *gin.Context) { } // handleJobResponse process a job approval response -// @Summary Process job response -// @Tags Job -// @Produce json -// @Param uuid path string true "Job UUID" -// @Param project body service.JobApprovalContext true "job approval response" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /job/{uuid}/response [post] +// @Summary Process job response +// @Tags Job +// @Produce json +// @Param uuid path string true "Job UUID" +// @Param project body service.JobApprovalContext true "job approval response" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /job/{uuid}/response [post] func (controller *JobController) handleJobResponse(c *gin.Context) { if err := func() error { jobUUID := c.Param("uuid") @@ -123,15 +123,15 @@ func (controller *JobController) handleJobResponse(c *gin.Context) { } // handleJobStatusUpdate process a job status update request -// @Summary Process job status update -// @Tags Job -// @Produce json -// @Param uuid path string true "Job UUID" -// @Param project body service.JobStatusUpdateContext true "job status" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /job/{uuid}/status [post] +// @Summary Process job status update +// @Tags Job +// @Produce json +// @Param uuid path string true "Job UUID" +// @Param project body service.JobStatusUpdateContext true "job status" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /job/{uuid}/status [post] func (controller *JobController) handleJobStatusUpdate(c *gin.Context) { if err := func() error { jobUUID := c.Param("uuid") diff --git a/fml-manager/server/api/project.go b/fml-manager/server/api/project.go index 4be28e5e..91e043c4 100644 --- a/fml-manager/server/api/project.go +++ b/fml-manager/server/api/project.go @@ -76,14 +76,14 @@ func (controller *ProjectController) Route(r *gin.RouterGroup) { } // handleInvitation process a project invitation -// @Summary Process project invitation -// @Tags Project -// @Produce json -// @Param project body service.ProjectInvitationRequest true "invitation request" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/invitation [post] +// @Summary Process project invitation +// @Tags Project +// @Produce json +// @Param project body service.ProjectInvitationRequest true "invitation request" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/invitation [post] func (controller *ProjectController) handleInvitation(c *gin.Context) { if err := func() error { invitationRequest := &service.ProjectInvitationRequest{} @@ -106,14 +106,14 @@ func (controller *ProjectController) handleInvitation(c *gin.Context) { } // handleInvitationAcceptance process a project invitation acceptance -// @Summary Process invitation acceptance response -// @Tags Project -// @Produce json -// @Param uuid path string true "Invitation UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/invitation/{uuid}/accept [post] +// @Summary Process invitation acceptance response +// @Tags Project +// @Produce json +// @Param uuid path string true "Invitation UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/invitation/{uuid}/accept [post] func (controller *ProjectController) handleInvitationAcceptance(c *gin.Context) { if err := func() error { invitationUUID := c.Param("uuid") @@ -133,14 +133,14 @@ func (controller *ProjectController) handleInvitationAcceptance(c *gin.Context) } // handleInvitationRejection process a project invitation rejection -// @Summary Process invitation rejection response -// @Tags Project -// @Produce json -// @Param uuid path string true "Invitation UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/invitation/{uuid}/reject [post] +// @Summary Process invitation rejection response +// @Tags Project +// @Produce json +// @Param uuid path string true "Invitation UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/invitation/{uuid}/reject [post] func (controller *ProjectController) handleInvitationRejection(c *gin.Context) { if err := func() error { invitationUUID := c.Param("uuid") @@ -160,14 +160,14 @@ func (controller *ProjectController) handleInvitationRejection(c *gin.Context) { } // handleInvitationRevocation process a project invitation revocation -// @Summary Process invitation revocation request -// @Tags Project -// @Produce json -// @Param uuid path string true "Invitation UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/invitation/{uuid}/revoke [post] +// @Summary Process invitation revocation request +// @Tags Project +// @Produce json +// @Param uuid path string true "Invitation UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/invitation/{uuid}/revoke [post] func (controller *ProjectController) handleInvitationRevocation(c *gin.Context) { if err := func() error { invitationUUID := c.Param("uuid") @@ -187,14 +187,14 @@ func (controller *ProjectController) handleInvitationRevocation(c *gin.Context) } // handleParticipantInfoUpdate process a participant info update event -// @Summary Process participant info update event, called by this FML manager's site context only -// @Tags Project -// @Produce json -// @Param project body event.ProjectParticipantUpdateEvent true "Updated participant info" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/event/participant/update [post] +// @Summary Process participant info update event, called by this FML manager's site context only +// @Tags Project +// @Produce json +// @Param project body event.ProjectParticipantUpdateEvent true "Updated participant info" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/event/participant/update [post] func (controller *ProjectController) handleParticipantInfoUpdate(c *gin.Context) { if err := func() error { updateEvent := &event.ProjectParticipantUpdateEvent{} @@ -217,15 +217,15 @@ func (controller *ProjectController) handleParticipantInfoUpdate(c *gin.Context) } // handleDataAssociation process a new data association -// @Summary Process new data association from site -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Param project body service.ProjectDataAssociation true "Data association info" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/data/associate [post] +// @Summary Process new data association from site +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Param project body service.ProjectDataAssociation true "Data association info" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/data/associate [post] func (controller *ProjectController) handleDataAssociation(c *gin.Context) { if err := func() error { projectUUID := c.Param("uuid") @@ -249,15 +249,15 @@ func (controller *ProjectController) handleDataAssociation(c *gin.Context) { } // handleDataDismissal process project data dismissal -// @Summary Process data dismissal from site -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Param project body service.ProjectDataAssociationBase true "Data association info containing the data UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/data/dismiss [post] +// @Summary Process data dismissal from site +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Param project body service.ProjectDataAssociationBase true "Data association info containing the data UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/data/dismiss [post] func (controller *ProjectController) handleDataDismissal(c *gin.Context) { if err := func() error { projectUUID := c.Param("uuid") @@ -281,15 +281,15 @@ func (controller *ProjectController) handleDataDismissal(c *gin.Context) { } // handleParticipantLeaving process project participant leaving -// @Summary Process participant leaving -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Param siteUUID path string true "Site UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/participant/{siteUUID}/leave [post] +// @Summary Process participant leaving +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Param siteUUID path string true "Site UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/participant/{siteUUID}/leave [post] func (controller *ProjectController) handleParticipantLeaving(c *gin.Context) { if err := func() error { projectUUID := c.Param("uuid") @@ -310,15 +310,15 @@ func (controller *ProjectController) handleParticipantLeaving(c *gin.Context) { } // handleParticipantDismissal process project participant dismissal -// @Summary Process participant dismissal, called by the managing site only -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Param siteUUID path string true "Site UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/participant/{siteUUID}/dismiss [post] +// @Summary Process participant dismissal, called by the managing site only +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Param siteUUID path string true "Site UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/participant/{siteUUID}/dismiss [post] func (controller *ProjectController) handleParticipantDismissal(c *gin.Context) { if err := func() error { projectUUID := c.Param("uuid") @@ -339,14 +339,14 @@ func (controller *ProjectController) handleParticipantDismissal(c *gin.Context) } // handleProjectClosing process project closing -// @Summary Process project closing -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/close [post] +// @Summary Process project closing +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/close [post] func (controller *ProjectController) handleProjectClosing(c *gin.Context) { if err := func() error { projectUUID := c.Param("uuid") @@ -366,14 +366,14 @@ func (controller *ProjectController) handleProjectClosing(c *gin.Context) { } // list returns all projects or project related to the specified participant -// @Summary List all project -// @Tags Project -// @Produce json -// @Param participant query string false "participant uuid, if set, only returns the projects containing the participant" -// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectInfoWithStatus} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project [get] +// @Summary List all project +// @Tags Project +// @Produce json +// @Param participant query string false "participant uuid, if set, only returns the projects containing the participant" +// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectInfoWithStatus} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project [get] func (controller *ProjectController) list(c *gin.Context) { // TODO: use token to extract participant uuid and do authz check participantUUID := c.DefaultQuery("participant", "") @@ -395,14 +395,14 @@ func (controller *ProjectController) list(c *gin.Context) { } // listData returns all data association in a project -// @Summary List all data association in a project -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectDataAssociation} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/data [get] +// @Summary List all data association in a project +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectDataAssociation} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/data [get] func (controller *ProjectController) listData(c *gin.Context) { // TODO: use token to verify the requester can access these info projectUUID := c.Param("uuid") @@ -424,14 +424,14 @@ func (controller *ProjectController) listData(c *gin.Context) { } // listParticipant returns all participants info in a project -// @Summary List all participants in a project -// @Tags Project -// @Produce json -// @Param uuid path string true "Project UUID" -// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectDataAssociation} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/{uuid}/participant [get] +// @Summary List all participants in a project +// @Tags Project +// @Produce json +// @Param uuid path string true "Project UUID" +// @Success 200 {object} GeneralResponse{data=map[string]service.ProjectDataAssociation} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/{uuid}/participant [get] func (controller *ProjectController) listParticipant(c *gin.Context) { // TODO: use token to verify the requester can access these info projectUUID := c.Param("uuid") @@ -453,14 +453,14 @@ func (controller *ProjectController) listParticipant(c *gin.Context) { } // handleParticipantUnregistration process a participant unregistration event -// @Summary Process participant unregistration event, called by this FML manager's site context only -// @Tags Project -// @Produce json -// @Param site body event.ProjectParticipantUnregistrationEvent true "Unregistered site info" -// @Success 200 {object} GeneralResponse{} "Success" -// @Failure 401 {object} GeneralResponse "Unauthorized operation" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /project/event/participant/unregister [post] +// @Summary Process participant unregistration event, called by this FML manager's site context only +// @Tags Project +// @Produce json +// @Param site body event.ProjectParticipantUnregistrationEvent true "Unregistered site info" +// @Success 200 {object} GeneralResponse{} "Success" +// @Failure 401 {object} GeneralResponse "Unauthorized operation" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /project/event/participant/unregister [post] func (controller *ProjectController) handleParticipantUnregistration(c *gin.Context) { if err := func() error { unregistrationEvent := &event.ProjectParticipantUnregistrationEvent{} diff --git a/fml-manager/server/api/site.go b/fml-manager/server/api/site.go index bc689c65..e5725283 100644 --- a/fml-manager/server/api/site.go +++ b/fml-manager/server/api/site.go @@ -53,12 +53,12 @@ func (controller *SiteController) Route(r *gin.RouterGroup) { } // getSite returns the sites list -// @Summary Return sites list -// @Tags Site -// @Produce json -// @Success 200 {object} GeneralResponse{data=[]entity.Site} "Success" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /site [get] +// @Summary Return sites list +// @Tags Site +// @Produce json +// @Success 200 {object} GeneralResponse{data=[]entity.Site} "Success" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /site [get] func (controller *SiteController) getSite(c *gin.Context) { siteList, err := controller.siteAppService.GetSiteList() if err != nil { @@ -79,13 +79,13 @@ func (controller *SiteController) getSite(c *gin.Context) { } // postSite creates or updates site information -// @Summary Create or update site info -// @Tags Site -// @Produce json -// @Param site body entity.Site true "The site information" -// @Success 200 {object} GeneralResponse "Success" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /site [post] +// @Summary Create or update site info +// @Tags Site +// @Produce json +// @Param site body entity.Site true "The site information" +// @Success 200 {object} GeneralResponse "Success" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /site [post] func (controller *SiteController) postSite(c *gin.Context) { if err := func() error { updatedSiteInfo := &entity.Site{} @@ -108,13 +108,13 @@ func (controller *SiteController) postSite(c *gin.Context) { } // deleteSite removes a site -// @Summary Remove a site, all related projects will be impacted -// @Tags Site -// @Produce json -// @Param uuid path string true "The site UUID" -// @Success 200 {object} GeneralResponse "Success" -// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" -// @Router /site/{uuid} [delete] +// @Summary Remove a site, all related projects will be impacted +// @Tags Site +// @Produce json +// @Param uuid path string true "The site UUID" +// @Success 200 {object} GeneralResponse "Success" +// @Failure 500 {object} GeneralResponse{code=int} "Internal server error" +// @Router /site/{uuid} [delete] func (controller *SiteController) deleteSite(c *gin.Context) { if err := func() error { siteUUID := c.Param("uuid") diff --git a/fml-manager/server/main.go b/fml-manager/server/main.go index f2cdf383..661094b8 100644 --- a/fml-manager/server/main.go +++ b/fml-manager/server/main.go @@ -43,13 +43,13 @@ import ( ) // main starts the API server -// @title fml manager API service -// @version v1 -// @description backend APIs of fml manager service -// @termsOfService http://swagger.io/terms/ -// @contact.name FedLCM team -// @BasePath /api/v1 -// @in header +// @title fml manager API service +// @version v1 +// @description backend APIs of fml manager service +// @termsOfService http://swagger.io/terms/ +// @contact.name FedLCM team +// @BasePath /api/v1 +// @in header func main() { viper.AutomaticEnv() replacer := strings.NewReplacer(".", "_") diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 39f4dc4d..c0a57fcf 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -34,6 +34,7 @@ import { TimeOutServiceComponent } from './components/time-out-service/time-out- import { DirectorNewComponent } from './view/openfl/director-new/director-new.component' import { DirectorDetailComponent } from './view/openfl/director-detail/director-detail.component' import { EnvoyDetailComponent } from './view/openfl/envoy-detail/envoy-detail.component'; +import { ExchangeClusterUpgradeComponent } from './view/federation/exchange-cluster-upgrade/exchange-cluster-upgrade.component'; import { AuthService } from './services/common/auth.service'; import { RouterGuard } from './router-guard'; @@ -171,6 +172,14 @@ const routes: Routes = [ component: ClusterDetailComponent }, + { + path: 'federation/fate/:id/detail/:uuid/:version/:name/upgrade', + data: { + preload: true + }, + + component: ExchangeClusterUpgradeComponent + }, { path: 'federation/openfl/:id/envoy/detail/:envoy_uuid', data: { diff --git a/frontend/src/app/app.component.ts b/frontend/src/app/app.component.ts index 9f754d13..eb50e708 100644 --- a/frontend/src/app/app.component.ts +++ b/frontend/src/app/app.component.ts @@ -10,10 +10,10 @@ // limitations under the License. import { Component } from '@angular/core'; -import { addTextIcon, ClarityIcons, userIcon, vmBugIcon, alignBottomIcon, barsIcon, certificateIcon, cogIcon, nodeGroupIcon, organizationIcon, usersIcon, hostGroupIcon, trashIcon, checkCircleIcon, angleIcon, plusCircleIcon, clusterIcon, routerIcon, cloudTrafficIcon, nvmeIcon, refreshIcon, worldIcon, detailsIcon, popOutIcon, timesCircleIcon, searchIcon, recycleIcon, nodesIcon, infoCircleIcon } from '@cds/core/icon'; +import { addTextIcon, ClarityIcons, userIcon, vmBugIcon, alignBottomIcon, barsIcon, certificateIcon, cogIcon, nodeGroupIcon, organizationIcon, usersIcon, hostGroupIcon, trashIcon, checkCircleIcon, angleIcon, plusCircleIcon, clusterIcon, routerIcon, cloudTrafficIcon, nvmeIcon, refreshIcon, worldIcon, detailsIcon, popOutIcon, timesCircleIcon, searchIcon, recycleIcon, nodesIcon, infoCircleIcon, uploadIcon, warningStandardIcon, minusCircleIcon, minusIcon, plusIcon} from '@cds/core/icon'; import { thinClientIcon } from '@cds/core/icon/shapes/thin-client'; import { updateIcon } from '@cds/core/icon/shapes/update'; -ClarityIcons.addIcons(addTextIcon, vmBugIcon, userIcon, alignBottomIcon, cogIcon, certificateIcon, organizationIcon, barsIcon, nodeGroupIcon, usersIcon, hostGroupIcon, trashIcon, checkCircleIcon, angleIcon, plusCircleIcon, clusterIcon, routerIcon, cloudTrafficIcon, nvmeIcon, updateIcon, refreshIcon, worldIcon, detailsIcon, popOutIcon, timesCircleIcon, searchIcon, recycleIcon, nodesIcon, thinClientIcon,infoCircleIcon); +ClarityIcons.addIcons(addTextIcon, vmBugIcon, userIcon, alignBottomIcon, cogIcon, certificateIcon, organizationIcon, barsIcon, nodeGroupIcon, usersIcon, hostGroupIcon, trashIcon, checkCircleIcon, angleIcon, plusCircleIcon, clusterIcon, routerIcon, cloudTrafficIcon, nvmeIcon, updateIcon, refreshIcon, worldIcon, detailsIcon, popOutIcon, timesCircleIcon, searchIcon, recycleIcon, nodesIcon, thinClientIcon,infoCircleIcon, uploadIcon, warningStandardIcon, minusCircleIcon, minusIcon, plusIcon); @Component({ selector: 'app-root', diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 7cfa8b8d..66a85d67 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -53,6 +53,7 @@ import { TimeOutServiceComponent } from './components/time-out-service/time-out- import { FilterComponent } from './components/filter/filter.component'; import { EnvoyDetailComponent } from './view/openfl/envoy-detail/envoy-detail.component'; import { CreateOpenflComponent } from './view/openfl/create-openfl-fed/create-openfl-fed.component'; +import { ExchangeClusterUpgradeComponent } from './view/federation/exchange-cluster-upgrade/exchange-cluster-upgrade.component'; @NgModule({ declarations: [ @@ -85,7 +86,8 @@ import { CreateOpenflComponent } from './view/openfl/create-openfl-fed/create-op DirectorDetailComponent, FilterComponent, EnvoyDetailComponent, - CreateOpenflComponent + CreateOpenflComponent, + ExchangeClusterUpgradeComponent ], imports: [ BrowserModule, diff --git a/frontend/src/app/services/federation-fate/fed.service.ts b/frontend/src/app/services/federation-fate/fed.service.ts index d37da851..5a17cec1 100644 --- a/frontend/src/app/services/federation-fate/fed.service.ts +++ b/frontend/src/app/services/federation-fate/fed.service.ts @@ -103,4 +103,12 @@ export class FedService { createExternalCluster(fed_uuid:string, externalCluster:any): Observable { return this.http.post('/federation/fate/'+ fed_uuid +'/cluster/external', externalCluster); } + + getExchangeClusterUpgradeVersionList(fed_uuid:string, upgrade_uuid: string, type: 'cluster' | 'exchange') { + return this.http.get(`/federation/fate/${fed_uuid}/${type}/${upgrade_uuid}/upgrade`) + } + + upgradeExchangeCluster(fed_uuid:string, upgrade_uuid: string, type: 'cluster' | 'exchange', data: {upgradeVersion: string}) { + return this.http.post(`/federation/fate/${fed_uuid}/${type}/${upgrade_uuid}/upgrade?upgradeVersion=${data.upgradeVersion}`, {}); + } } \ No newline at end of file diff --git a/frontend/src/app/view/endpoint/endpoint-new/endpoint-new.component.ts b/frontend/src/app/view/endpoint/endpoint-new/endpoint-new.component.ts index 0b45c7f8..407e0373 100644 --- a/frontend/src/app/view/endpoint/endpoint-new/endpoint-new.component.ts +++ b/frontend/src/app/view/endpoint/endpoint-new/endpoint-new.component.ts @@ -288,6 +288,7 @@ export class EndpointNewComponent implements OnInit { }) this.codeMirror.on('change', (cm: any) => { this.codeMirror.save() + this.form.get('install')?.get('yaml')?.setValue(cm.getValue()) }) this.hasYAMLTextAreaDOM = true } diff --git a/frontend/src/app/view/federation/cluster-detail/cluster-detail.component.html b/frontend/src/app/view/federation/cluster-detail/cluster-detail.component.html index 1d08ae32..9cad3380 100644 --- a/frontend/src/app/view/federation/cluster-detail/cluster-detail.component.html +++ b/frontend/src/app/view/federation/cluster-detail/cluster-detail.component.html @@ -1,5 +1,5 @@
- <<{{'CommonlyUse.back'|translate}} + <<{{'CommonlyUse.back'|translate}}

{{'ClusterDetail.detail'|translate}}