Merge branch 'master' into binyli/log-url

.github/workflows/continuous-integration.yml

@@ -112,6 +112,36 @@ jobs:
         yarn install --frozen-lockfiles
         yarn test
 
+  code-coverage:
+    name: Run code coverage of rest-server on node-${{ matrix.node }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node: [10]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Use Node ${{ matrix.node }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node }}
+    - name: yarn install and test
+      run: |
+        cd src/rest-server
+        yarn install --frozen-lockfiles
+        yarn test
+        mkdir ./coverage
+        yarn run coveralls
+
+    - name: Coveralls GitHub Action
+      uses: coverallsapp/github-action@v1.1.1
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        # Path to lcov file
+        path-to-lcov: ./src/rest-server/coverage/lcov.info
+
   webportal:
     name: Test webportal on node-${{ matrix.node }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
@@ -160,3 +190,24 @@ jobs:
         cd contrib/submit-job-v2
         yarn install --frozen-lockfiles
         yarn build
+
+  frameworklauncher:
+    name: Test frameworklauncher on java-${{ matrix.java }}-${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        java: [8.0]
+        os: [ubuntu-16.04, ubuntu-latest]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Setup java
+      uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.java }}
+    - name: Test frameworklauncher
+      run: |
+        cd subprojects/frameworklauncher/yarn
+        mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+        mvn clean test jacoco:report

.github/workflows/lint.yml

@@ -64,3 +64,25 @@ jobs:
     - name: Lint
       run: |
         pylint contrib/kubespray/script --rcfile=contrib/kubespray/script/pylintrc
+
+  swagger-validate:
+    name: Validate swagger
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        node: [10]
+        os: [ubuntu-16.04]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Use Node ${{ matrix.node }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node }}
+    - name: Install swagger-cli
+      run: |
+        npm install -g @apidevtools/swagger-cli
+    - name: validate
+      run: |
+        swagger-cli validate src/rest-server/docs/swagger.yaml

README.md

@@ -2,7 +2,7 @@
 
 [logo]: ./pailogo.jpg "OpenPAI"
 
-[![Build Status](https://travis-ci.org/microsoft/pai.svg?branch=master)](https://travis-ci.org/microsoft/pai)
+[![Build Status](https://github.com/microsoft/pai/workflows/CI/badge.svg)](https://github.com/microsoft/pai/actions)
 [![Join the chat at https://gitter.im/Microsoft/pai](https://badges.gitter.im/Microsoft/pai.svg)](https://gitter.im/Microsoft/pai?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Version](https://img.shields.io/github/release/Microsoft/pai.svg)](https://github.com/Microsoft/pai/releases/latest)
 

docs/index.md

@@ -1,6 +1,6 @@
 # OpenPAI Handbook
 
-[![Build Status](https://travis-ci.org/microsoft/pai.svg?branch=master)](https://travis-ci.org/microsoft/pai)
+[![Build Status](https://github.com/microsoft/pai/workflows/CI/badge.svg)](https://github.com/microsoft/pai/actions)
 [![Join the chat at https://gitter.im/Microsoft/pai](https://badges.gitter.im/Microsoft/pai.svg)](https://gitter.im/Microsoft/pai?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Version](https://img.shields.io/github/release/Microsoft/pai.svg)](https://github.com/Microsoft/pai/releases/latest)
 
@@ -16,4 +16,4 @@ To set up a new cluster, learn how to manage cluster on OpenPAI, please follow [
 
 To view a general introduction of OpenPAI, please refer to the [Github Readme](https://github.com/microsoft/pai/blob/master/README.md).
 
-For any issue/bug/feature request, please submit it to [GitHub](https://github.com/microsoft/pai).
+For any issue/bug/feature request, please submit it to [GitHub](https://github.com/microsoft/pai).

docs/manual/cluster-admin/how-to-add-and-remove-nodes.md

@@ -14,7 +14,7 @@ Log in to your dev box machine, find [the pre-kept folder `~/pai-deploy`](./inst
 
 Find the file `~/pai-deploy/kubespray/inventory/pai/hosts.yml`, and follow the steps below to modify it. 
 
-Supposing you want to add 2 worker nodes into your cluster and their hostnames are `a` and `b`.  Add these 2 nodes into the `hosts.yml`. An example:
+Supposing you want to add 2 worker nodes into your cluster and their hostnames are `new-worker-node-0` and `new-worker-node-1`.  Add these 2 nodes into the `hosts.yml`. An example:
 
 ```yaml
 all:
@@ -35,15 +35,15 @@ all:
       ...
 
 ############# Example start ################### 
-    a:
+    new-worker-node-0:
       ip: x.x.x.x
       access_ip: x.x.x.x
       ansible_host: x.x.x.x
       ansible_ssh_user: "username"
       ansible_ssh_pass: "your-password-here"
       ansible_become_pass: "your-password-here"
       ansible_ssh_extra_args: '-o StrictHostKeyChecking=no'
-    b:
+    new-worker-node-1:
       ip: x.x.x.x
       access_ip: x.x.x.x
       ansible_host: x.x.x.x
@@ -65,8 +65,8 @@ all:
         origin4:
 
 ############# Example start ################### 
-        a:
-        b:
+        new-worker-node-0:
+        new-worker-node-1:
 ############## Example end #################### 
 
     gpu:
@@ -75,8 +75,8 @@ all:
 
 ############# Example start ################### 
 #### If the worker doesn't have GPU, please don't add them here.
-        a:
-        b:
+        new-worker-node-0:
+        new-worker-node-1:
 ############## Example end #################### 
 
     etcd:
@@ -95,38 +95,38 @@ all:
 Go into folder `~/pai-deploy/kubespray/`, run:
 
 ```bash
-ansible-playbook -i inventory/pai/hosts.yml scale.yml -b --become-user=root -e "node=a,b" -e "@inventory/pai/openpai.yml"
+ansible-playbook -i inventory/pai/hosts.yml cluster.yml -b --become-user=root --limit=new-worker-node-0,new-worker-node-1 -e "@inventory/pai/openpai.yml"
 ```
 
-The nodes to add are specified with `-e` flag.
+The nodes to add are specified with the `--limit` flag.
 
 ### Update OpenPAI Service Configuration
 
 Find your [service configuration file `layout.yaml` and `services-configuration.yaml`](./basic-management-operations.md#pai-service-management-and-paictl) in  `~/pai-deploy/cluster-cfg`.
 
-- Add the new node into `machine-list` field in `layout.yaml`
+- Add the new node into `machine-list` field in `layout.yaml`, create a new `machine-sku` if necessary. Refer to [layout.yaml](./installation-guide.md#layoutyaml-format) for schema requirements.
 
 ```yaml
 machine-list:
-  - hostname: a
+  - hostname: new-worker-node--0
     hostip: x.x.x.x
     machine-type: xxx-sku
     pai-worker: "true"
 
-  - hostname: b
+  - hostname: new-worker-node-1
     hostip: x.x.x.x
     machine-type: xxx-sku
     pai-worker: "true"
 ```
 
 - If you are using hived scheduler, you should modify its setting in `services-configuration.yaml` properly. Please refer to [how to set up virtual clusters](./how-to-set-up-virtual-clusters.md) and the [hived scheduler doc](https://github.com/microsoft/hivedscheduler/blob/master/doc/user-manual.md) for details. If you are using Kubernetes default scheduler, you can skip this step.
 
-- Stop the service, push the latest configuration, and then start services:
+- Stop the service, push the latest configuration, and then start related services:
 
 ```bash
-./paictl.py service stop -n cluster-configuration hivedscheduler rest-server
+./paictl.py service stop -n cluster-configuration hivedscheduler rest-server job-exporter
 ./paictl.py config push -p <config-folder> -m service
-./paictl.py service start -n cluster-configuration hivedscheduler rest-server
+./paictl.py service start -n cluster-configuration hivedscheduler rest-server job-exporter
 ```
 
 If you have configured any PV/PVC storage, please confirm the added worker node meets the PV's requirements. See [Confirm Worker Nodes Environment](./how-to-set-up-storage.md#confirm-environment-on-worker-nodes) for details.
@@ -139,17 +139,17 @@ To remove nodes from the cluster, there is no need to modify `hosts.yml`.
 Go into `~/pai-deploy/kubespray/`, run
 
 ```bash
-ansible-playbook -i inventory/pai/hosts.yml remove-node.yml -b --become-user=root -e "node=a,b" -e "@inventory/pai/openpai.yml"
+ansible-playbook -i inventory/pai/hosts.yml remove-node.yml -b --become-user=root -e "node=worker-node-to-remove-0,worker-node-to-remove-1" -e "@inventory/pai/openpai.yml"
 ``` 
 
-The nodes to remove are specified with `-e` flag.
+The nodes to remove are specified with the `-e` flag.
 
 Modify the `layout.yaml` and `services-configuration.yaml`.
 
-Stop the service, push the latest configuration, and then start services:
+Stop the service, push the latest configuration, and then start related services:
 
 ```bash
-./paictl.py service stop -n cluster-configuration hivedscheduler rest-server
+./paictl.py service stop -n cluster-configuration hivedscheduler rest-server job-exporter
 ./paictl.py config push -p <config-folder> -m service
-./paictl.py service start -n cluster-configuration hivedscheduler rest-server
+./paictl.py service start -n cluster-configuration hivedscheduler rest-server job-exporter
 ```

docs/manual/cluster-admin/installation-guide.md

@@ -170,9 +170,7 @@ Please edit `layout.yaml` and a `config.yaml` file under `<pai-code-dir>/contrib
 These two files spedify the cluster layout and the customized configuration, respectively.
 The following is the format and example of these 2 files.
 
-#### Tips for China Users
-
-If you are a China user, before you edit these files, please refer to [here](./configuration-for-china.md) first.
+**Tips for China Users**: If you are a China user, before you edit these files, please refer to [here](./configuration-for-china.md) first.
 
 #### `layout.yaml` format
 

docs_zh_CN/index.md

@@ -1,6 +1,6 @@
 # OpenPAI手册
 
-[![Build Status](https://travis-ci.org/microsoft/pai.svg?branch=master)](https://travis-ci.org/microsoft/pai)
+[![Build Status](https://github.com/microsoft/pai/workflows/CI/badge.svg)](https://github.com/microsoft/pai/actions)
 [![Join the chat at https://gitter.im/Microsoft/pai](https://badges.gitter.im/Microsoft/pai.svg)](https://gitter.im/Microsoft/pai?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Version](https://img.shields.io/github/release/Microsoft/pai.svg)](https://github.com/Microsoft/pai/releases/latest)
 

src/database-controller/sdk/index.js

@@ -39,11 +39,12 @@ class DatabaseModel {
           type: Sequelize.DATE,
           allowNull: false,
         },
-        // `dockerSecretDef`, `configSecretDef`, and `priorityClassDef` is the definition of job add-ons.
+        // `dockerSecretDef`, `configSecretDef`, `tokenSecretDef` and `priorityClassDef` is the definition of job add-ons.
         // They are generated by rest-server and recorded into database by write-merger.
         // These add-ons are created by poller or the short-cut in write-merger.
         dockerSecretDef: Sequelize.TEXT,
         configSecretDef: Sequelize.TEXT,
+        tokenSecretDef: Sequelize.TEXT,
         priorityClassDef: Sequelize.TEXT,
         retries: Sequelize.INTEGER,
         retryDelayTime: Sequelize.INTEGER,