From 14eda62f03ef97e31cda62322727847f4968b673 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 18 Mar 2019 14:32:13 -0700 Subject: [PATCH 1/3] add component.yaml to every ibm-components --- .../commons/config/component.yaml | 33 ++++++++++++++ .../ibm-components/ffdl/serve/component.yaml | 37 +++++++++++++++ .../ibm-components/ffdl/train/component.yaml | 31 +++++++++++++ .../watson/deploy/component.yaml | 33 ++++++++++++++ .../manage/monitor_fairness/component.yaml | 3 +- .../manage/monitor_quality/component.yaml | 3 +- .../watson/manage/subscribe/component.yaml | 5 ++- .../watson/store/component.yaml | 31 +++++++++++++ .../watson/train/component.yaml | 45 +++++++++++++++++++ 9 files changed, 217 insertions(+), 4 deletions(-) create mode 100644 components/ibm-components/commons/config/component.yaml create mode 100644 components/ibm-components/ffdl/serve/component.yaml create mode 100644 components/ibm-components/ffdl/train/component.yaml create mode 100644 components/ibm-components/watson/deploy/component.yaml create mode 100644 components/ibm-components/watson/store/component.yaml create mode 100644 components/ibm-components/watson/train/component.yaml diff --git a/components/ibm-components/commons/config/component.yaml b/components/ibm-components/commons/config/component.yaml new file mode 100644 index 00000000000..5485a2de095 --- /dev/null +++ b/components/ibm-components/commons/config/component.yaml @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Kubernetes Cluster: Create secret' +description: | + Create secret to store pipeline credentials on Kubernetes Cluster +inputs: + - {name: token, description: 'Required. GitHub token for accessing private repository'} + - {name: url, description: 'Required. GitHub raw path for accessing the credential file'} + - {name: name, description: 'Required. Secret Name to be stored in Kubernetes'} +outputs: + - {name: secret_name, description: 'Kubernetes secret name'} +implementation: + container: + image: docker.io/aipipeline/wml-config:latest + command: ['python3'] + args: [ + /app/config.py, + --token, {inputValue: token}, + --url, {inputValue: url}, + --name, {inputValue: name} + ] + fileOutputs: + secret_name: /tmp/ai-pipeline-creds diff --git a/components/ibm-components/ffdl/serve/component.yaml b/components/ibm-components/ffdl/serve/component.yaml new file mode 100644 index 00000000000..f0247d462be --- /dev/null +++ b/components/ibm-components/ffdl/serve/component.yaml @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Seldon Core: Serve Model' +description: | + Serve Machine Learning and Deep Learning Models remotely as web service using Seldon Core +inputs: + - {name: model_id, description: 'Required. Model training_id from Fabric for Deep Learning'} + - {name: deployment_name, description: 'Required. Deployment name for the seldon service'} + - {name: model_class_name, description: 'PyTorch model class name', default: 'ModelClass'} + - {name: model_class_file, description: 'File that contains the PyTorch model class', default: 'model_class.py'} + - {name: serving_image, description: 'Model serving images', default: 'aipipeline/seldon-pytorch:0.1'} +outputs: + - {name: output, description: 'Model Serving status'} +implementation: + container: + image: docker.io/aipipeline/ffdl-serve:latest + command: ['python'] + args: [ + -u, serve.py, + --model_id, {inputValue: model_id}, + --deployment_name, {inputValue: deployment_name}, + --model_class_name, {inputValue: model_class_name}, + --model_class_file, {inputValue: model_class_file}, + --serving_image, {inputValue: serving_image} + ] + fileOutputs: + output: /tmp/deployment_result.txt diff --git a/components/ibm-components/ffdl/train/component.yaml b/components/ibm-components/ffdl/train/component.yaml new file mode 100644 index 00000000000..402fe720f93 --- /dev/null +++ b/components/ibm-components/ffdl/train/component.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Fabric for Deep Learning: Train Model' +description: | + Train Machine Learning and Deep Learning Models remotely using Fabric for Deep Learning +inputs: + - {name: model_def_file_path, description: 'Required. Path for model training code in object storage'} + - {name: manifest_file_path, description: 'Required. Path for model manifest definition in object storage'} +outputs: + - {name: output, description: 'Model training_id'} +implementation: + container: + image: docker.io/aipipeline/ffdl-train:latest + command: ['python'] + args: [ + -u, train.py, + --model_def_file_path, {inputValue: model_def_file_path}, + --manifest_file_path, {inputValue: manifest_file_path} + ] + fileOutputs: + output: /tmp/training_id.txt diff --git a/components/ibm-components/watson/deploy/component.yaml b/components/ibm-components/watson/deploy/component.yaml new file mode 100644 index 00000000000..a3e24b0c6f7 --- /dev/null +++ b/components/ibm-components/watson/deploy/component.yaml @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Watson Machine Learning: Deploy Model' +description: | + Deploy stored model on Watson Machine Learning as a web service. +inputs: + - {name: model_uid, description: 'Required. UID for the Watson Machine Learning model'} + - {name: model_name, description: 'Required. Model Name on Watson Machine Learning'} + - {name: scoring_payload, description: 'Required. Sample Payload file name in the object storage'} +outputs: + - {name: output, description: 'Link to the deployed model web service'} +implementation: + container: + image: docker.io/aipipeline/wml-deploy:latest + command: ['python3'] + args: [ + /app/wml-deploy.py, + --model-uid, {inputValue: model_uid}, + --model-name, {inputValue: model_name}, + --scoring-payload, {inputValue: scoring_payload} + ] + fileOutputs: + output: /tmp/output diff --git a/components/ibm-components/watson/manage/monitor_fairness/component.yaml b/components/ibm-components/watson/manage/monitor_fairness/component.yaml index d6b74a0b3e5..cdf71b9ab57 100644 --- a/components/ibm-components/watson/manage/monitor_fairness/component.yaml +++ b/components/ibm-components/watson/manage/monitor_fairness/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Enable fairness monitoring on Watson OpenScale +name: 'Watson OpenScale: Monitor Fairness' description: | Enable model fairness monitoring on Watson OpenScale. inputs: @@ -22,6 +22,7 @@ inputs: implementation: container: image: docker.io/aipipeline/monitor_fairness:latest + command: ['python'] args: [ -u, monitor_fairness.py, --model_name, {inputValue: model_name}, diff --git a/components/ibm-components/watson/manage/monitor_quality/component.yaml b/components/ibm-components/watson/manage/monitor_quality/component.yaml index cbfaa8d575f..69577f58d68 100644 --- a/components/ibm-components/watson/manage/monitor_quality/component.yaml +++ b/components/ibm-components/watson/manage/monitor_quality/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Enable quality monitoring on Watson OpenScale +name: 'Watson OpenScale: Monitor quality' description: | Enable model quality monitoring on Watson OpenScale. inputs: @@ -21,6 +21,7 @@ inputs: implementation: container: image: docker.io/aipipeline/monitor_quality:latest + command: ['python'] args: [ -u, monitor_quality.py, --model_name, {inputValue: model_name}, diff --git a/components/ibm-components/watson/manage/subscribe/component.yaml b/components/ibm-components/watson/manage/subscribe/component.yaml index c19c86745f8..acce52caa1a 100644 --- a/components/ibm-components/watson/manage/subscribe/component.yaml +++ b/components/ibm-components/watson/manage/subscribe/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Subscribe Watson OpenScale +name: 'Watson OpenScale: Subscribe' description: | Binding deployed models and subscribe them to Watson OpenScale service. inputs: @@ -23,6 +23,7 @@ outputs: implementation: container: image: docker.io/aipipeline/subscribe:latest + command: ['python'] args: [ -u, subscribe.py, --model_name, {inputValue: model_name}, @@ -31,4 +32,4 @@ implementation: --label_column, {inputValue: label_column} ] fileOutputs: - job_id: /tmp/model_name.txt + model_name: /tmp/model_name.txt diff --git a/components/ibm-components/watson/store/component.yaml b/components/ibm-components/watson/store/component.yaml new file mode 100644 index 00000000000..383821e25ac --- /dev/null +++ b/components/ibm-components/watson/store/component.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Watson Machine Learning: Store model' +description: | + Store and persistent trained model on Watson Machine Learning. +inputs: + - {name: run_uid, description: 'Required. UID for the Watson Machine Learning training-runs'} + - {name: model_name, description: 'Required. Model Name to store on Watson Machine Learning'} +outputs: + - {name: model_uid, description: 'UID for the stored model on Watson Machine Learning'} +implementation: + container: + image: docker.io/aipipeline/wml-store:latest + command: ['python3'] + args: [ + /app/wml-store.py, + --run-uid, {inputValue: run_uid}, + --model-name, {inputValue: model_name} + ] + fileOutputs: + model_uid: /tmp/model_uid diff --git a/components/ibm-components/watson/train/component.yaml b/components/ibm-components/watson/train/component.yaml new file mode 100644 index 00000000000..2a78a60d37c --- /dev/null +++ b/components/ibm-components/watson/train/component.yaml @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: 'Watson Machine Learning: Train Model' +description: | + Train Machine Learning and Deep Learning Models in the Cloud using Watson Machine Learning +inputs: + - {name: train_code, description: 'Required. Code for training ML/DL models'} + - {name: execution_command, description: 'Required. Execution command to start the model training.'} + - {name: config, description: 'Required. Credential configfile is properly created.'} + - {name: framework, description: 'ML/DL Model Framework', default: 'tensorflow'} + - {name: framework_version, description: 'Model Framework version', default: '1.5'} + - {name: runtime, description: 'Model Code runtime language', default: 'python'} + - {name: runtime_version, description: 'Model Code runtime version', default: '3.5'} + - {name: run_definition, description: 'Name for the Watson Machine Learning training definition', default: 'python-tensorflow-definition'} + - {name: run_name, description: 'Name for the Watson Machine Learning training-runs', default: 'python-tensorflow-run'} +outputs: + - {name: run_uid, description: 'UID for the Watson Machine Learning training-runs'} +implementation: + container: + image: docker.io/aipipeline/wml-train:latest + command: ['python3'] + args: [ + /app/wml-train.py, + --config, {inputValue: config}, + --train-code, {inputValue: train_code}, + --execution-command, {inputValue: execution_command}, + --framework, {inputValue: framework}, + --framework-version, {inputValue: framework_version}, + --runtime, {inputValue: runtime}, + --runtime-version, {inputValue: runtime_version}, + --run-definition, {inputValue: run_definition}, + --run-name, {inputValue: run_name} + ] + fileOutputs: + run_uid: /tmp/run_uid From 05f4700f635041de87d4f11f27fdb102470499f6 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 18 Mar 2019 16:22:12 -0700 Subject: [PATCH 2/3] replace colon conflicts with dashes --- components/ibm-components/commons/config/component.yaml | 2 +- components/ibm-components/ffdl/serve/component.yaml | 2 +- components/ibm-components/ffdl/train/component.yaml | 2 +- components/ibm-components/watson/deploy/component.yaml | 2 +- .../watson/manage/monitor_fairness/component.yaml | 2 +- .../ibm-components/watson/manage/monitor_quality/component.yaml | 2 +- .../ibm-components/watson/manage/subscribe/component.yaml | 2 +- components/ibm-components/watson/store/component.yaml | 2 +- components/ibm-components/watson/train/component.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/ibm-components/commons/config/component.yaml b/components/ibm-components/commons/config/component.yaml index 5485a2de095..7b217842d27 100644 --- a/components/ibm-components/commons/config/component.yaml +++ b/components/ibm-components/commons/config/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Kubernetes Cluster: Create secret' +name: 'Kubernetes Cluster - Create secret' description: | Create secret to store pipeline credentials on Kubernetes Cluster inputs: diff --git a/components/ibm-components/ffdl/serve/component.yaml b/components/ibm-components/ffdl/serve/component.yaml index f0247d462be..649a5ae2149 100644 --- a/components/ibm-components/ffdl/serve/component.yaml +++ b/components/ibm-components/ffdl/serve/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Seldon Core: Serve Model' +name: 'Seldon Core - Serve Model' description: | Serve Machine Learning and Deep Learning Models remotely as web service using Seldon Core inputs: diff --git a/components/ibm-components/ffdl/train/component.yaml b/components/ibm-components/ffdl/train/component.yaml index 402fe720f93..b40a2f65ec7 100644 --- a/components/ibm-components/ffdl/train/component.yaml +++ b/components/ibm-components/ffdl/train/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Fabric for Deep Learning: Train Model' +name: 'Fabric for Deep Learning - Train Model' description: | Train Machine Learning and Deep Learning Models remotely using Fabric for Deep Learning inputs: diff --git a/components/ibm-components/watson/deploy/component.yaml b/components/ibm-components/watson/deploy/component.yaml index a3e24b0c6f7..0f6311c67d8 100644 --- a/components/ibm-components/watson/deploy/component.yaml +++ b/components/ibm-components/watson/deploy/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson Machine Learning: Deploy Model' +name: 'Watson Machine Learning - Deploy Model' description: | Deploy stored model on Watson Machine Learning as a web service. inputs: diff --git a/components/ibm-components/watson/manage/monitor_fairness/component.yaml b/components/ibm-components/watson/manage/monitor_fairness/component.yaml index cdf71b9ab57..ecf6822163d 100644 --- a/components/ibm-components/watson/manage/monitor_fairness/component.yaml +++ b/components/ibm-components/watson/manage/monitor_fairness/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson OpenScale: Monitor Fairness' +name: 'Watson OpenScale - Monitor Fairness' description: | Enable model fairness monitoring on Watson OpenScale. inputs: diff --git a/components/ibm-components/watson/manage/monitor_quality/component.yaml b/components/ibm-components/watson/manage/monitor_quality/component.yaml index 69577f58d68..4695731d787 100644 --- a/components/ibm-components/watson/manage/monitor_quality/component.yaml +++ b/components/ibm-components/watson/manage/monitor_quality/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson OpenScale: Monitor quality' +name: 'Watson OpenScale - Monitor quality' description: | Enable model quality monitoring on Watson OpenScale. inputs: diff --git a/components/ibm-components/watson/manage/subscribe/component.yaml b/components/ibm-components/watson/manage/subscribe/component.yaml index acce52caa1a..1932f32d590 100644 --- a/components/ibm-components/watson/manage/subscribe/component.yaml +++ b/components/ibm-components/watson/manage/subscribe/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson OpenScale: Subscribe' +name: 'Watson OpenScale - Subscribe' description: | Binding deployed models and subscribe them to Watson OpenScale service. inputs: diff --git a/components/ibm-components/watson/store/component.yaml b/components/ibm-components/watson/store/component.yaml index 383821e25ac..d2e29898dde 100644 --- a/components/ibm-components/watson/store/component.yaml +++ b/components/ibm-components/watson/store/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson Machine Learning: Store model' +name: 'Watson Machine Learning - Store model' description: | Store and persistent trained model on Watson Machine Learning. inputs: diff --git a/components/ibm-components/watson/train/component.yaml b/components/ibm-components/watson/train/component.yaml index 2a78a60d37c..78c92874204 100644 --- a/components/ibm-components/watson/train/component.yaml +++ b/components/ibm-components/watson/train/component.yaml @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Watson Machine Learning: Train Model' +name: 'Watson Machine Learning - Train Model' description: | Train Machine Learning and Deep Learning Models in the Cloud using Watson Machine Learning inputs: From 7d4d05b20dd50f463e397dac7d2e2b037dafad13 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Tue, 19 Mar 2019 09:45:41 -0700 Subject: [PATCH 3/3] update ffdl serve component.yaml description --- components/ibm-components/ffdl/serve/component.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/ibm-components/ffdl/serve/component.yaml b/components/ibm-components/ffdl/serve/component.yaml index 649a5ae2149..fb585f1090f 100644 --- a/components/ibm-components/ffdl/serve/component.yaml +++ b/components/ibm-components/ffdl/serve/component.yaml @@ -10,9 +10,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: 'Seldon Core - Serve Model' +name: 'Seldon Core - Serve PyTorch Model' description: | - Serve Machine Learning and Deep Learning Models remotely as web service using Seldon Core + Serve PyTorch Models remotely as web service using Seldon Core inputs: - {name: model_id, description: 'Required. Model training_id from Fabric for Deep Learning'} - {name: deployment_name, description: 'Required. Deployment name for the seldon service'}