From e8117196055d9be074062086acdfbe53df475004 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Tue, 8 Oct 2019 20:29:22 +0800 Subject: [PATCH 1/5] add sample test for multiple output --- ... - basics.ipynb => multiple_outputs.ipynb} | 27 ++++++++++++------- .../sample-test/configs/multiple_outputs.yaml | 19 +++++++++++++ test/sample_test.yaml | 1 + 3 files changed, 37 insertions(+), 10 deletions(-) rename samples/core/multiple_outputs/{Multiple outputs - basics.ipynb => multiple_outputs.ipynb} (89%) create mode 100644 test/sample-test/configs/multiple_outputs.yaml diff --git a/samples/core/multiple_outputs/Multiple outputs - basics.ipynb b/samples/core/multiple_outputs/multiple_outputs.ipynb similarity index 89% rename from samples/core/multiple_outputs/Multiple outputs - basics.ipynb rename to samples/core/multiple_outputs/multiple_outputs.ipynb index 33c96800c67..8b54ae0e3af 100644 --- a/samples/core/multiple_outputs/Multiple outputs - basics.ipynb +++ b/samples/core/multiple_outputs/multiple_outputs.ipynb @@ -50,14 +50,20 @@ }, "outputs": [], "source": [ - "GCS_BUCKET = 'gs://[BUCKET-NAME]' # GCS bucket name\n", - "PROJECT_NAME = '[PROJECT-NAME]' # GCP project name\n", - "\n", - "STAGING_GCS_PATH = GCS_BUCKET + '/multiple-output-sample'\n", - "TARGET_IMAGE = 'gcr.io/%s/multi-output:latest' % PROJECT_NAME\n", - "\n", - "BASE_IMAGE = 'tensorflow/tensorflow:1.11.0-py3'\n", - "EXPERIMENT_NAME = 'Multiple Outputs Sample'" + "output = 'gs://[BUCKET-NAME]' # GCS bucket name\n", + "project_id = '[PROJECT-NAME]' # GCP project name\n", + "experiment_name = 'Multiple Outputs Sample'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "STAGING_GCS_PATH = output + '/multiple-output-sample'\n", + "TARGET_IMAGE = 'gcr.io/%s/multi-output:latest' % project_id\n", + "BASE_IMAGE = 'tensorflow/tensorflow:1.11.0-py3'" ] }, { @@ -160,11 +166,12 @@ " 'b': 2.5,\n", " 'c': 3.0,\n", "}\n", - "run_result = kfp.Client().create_run_from_pipeline_func(pipeline, arguments=arguments, experiment_name=EXPERIMENT_NAME)" + "run_result = kfp.Client().create_run_from_pipeline_func(pipeline, arguments=arguments, experiment_name=experiment_name)" ] } ], "metadata": { + "celltoolbar": "Tags", "kernelspec": { "display_name": "Python 3", "language": "python", @@ -180,7 +187,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.6.4" } }, "nbformat": 4, diff --git a/test/sample-test/configs/multiple_outputs.yaml b/test/sample-test/configs/multiple_outputs.yaml new file mode 100644 index 00000000000..c50f9bfcb40 --- /dev/null +++ b/test/sample-test/configs/multiple_outputs.yaml @@ -0,0 +1,19 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +test_name: multiple_outputs +notebook_params: + output: + project: ml-pipeline-test + experiment_name: multiple_outputs-test \ No newline at end of file diff --git a/test/sample_test.yaml b/test/sample_test.yaml index a70385125d9..1893a46ebfd 100644 --- a/test/sample_test.yaml +++ b/test/sample_test.yaml @@ -86,6 +86,7 @@ spec: - loop_parameter - loop_static - resource_ops + - multiple_outputs # Build and push image - name: build-image-by-dockerfile inputs: From 786f7d718cb11be7bc5014d16f5cad0d50eebe7c Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Wed, 9 Oct 2019 10:33:03 +0800 Subject: [PATCH 2/5] adjust the output path --- samples/core/multiple_outputs/multiple_outputs.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/core/multiple_outputs/multiple_outputs.ipynb b/samples/core/multiple_outputs/multiple_outputs.ipynb index 8b54ae0e3af..c5c03c3b60d 100644 --- a/samples/core/multiple_outputs/multiple_outputs.ipynb +++ b/samples/core/multiple_outputs/multiple_outputs.ipynb @@ -61,7 +61,8 @@ "metadata": {}, "outputs": [], "source": [ - "STAGING_GCS_PATH = output + '/multiple-output-sample'\n", + "import os\n", + "STAGING_GCS_PATH = os.path.join(output, 'multiple-output-sample')\n", "TARGET_IMAGE = 'gcr.io/%s/multi-output:latest' % project_id\n", "BASE_IMAGE = 'tensorflow/tensorflow:1.11.0-py3'" ] From 7f5263ff4679b938074c7699cf5507eaf8300b5f Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Wed, 9 Oct 2019 10:41:16 +0800 Subject: [PATCH 3/5] rename config file --- .../{multiple_outputs.yaml => multiple_outputs.config.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/sample-test/configs/{multiple_outputs.yaml => multiple_outputs.config.yaml} (100%) diff --git a/test/sample-test/configs/multiple_outputs.yaml b/test/sample-test/configs/multiple_outputs.config.yaml similarity index 100% rename from test/sample-test/configs/multiple_outputs.yaml rename to test/sample-test/configs/multiple_outputs.config.yaml From 7bea8f16b38102f11f5034de991b6c76b0b90d76 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Wed, 9 Oct 2019 12:22:05 +0800 Subject: [PATCH 4/5] fix param --- test/sample-test/configs/multiple_outputs.config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sample-test/configs/multiple_outputs.config.yaml b/test/sample-test/configs/multiple_outputs.config.yaml index c50f9bfcb40..00f400f68ec 100644 --- a/test/sample-test/configs/multiple_outputs.config.yaml +++ b/test/sample-test/configs/multiple_outputs.config.yaml @@ -15,5 +15,5 @@ test_name: multiple_outputs notebook_params: output: - project: ml-pipeline-test + project_id: ml-pipeline-test experiment_name: multiple_outputs-test \ No newline at end of file From 9269c6b18b5ba0b3a8046948386ac05d2a789ea9 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Wed, 9 Oct 2019 14:24:06 +0800 Subject: [PATCH 5/5] remove the python_component decorator --- samples/core/multiple_outputs/multiple_outputs.ipynb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/samples/core/multiple_outputs/multiple_outputs.ipynb b/samples/core/multiple_outputs/multiple_outputs.ipynb index c5c03c3b60d..61a7dd0b522 100644 --- a/samples/core/multiple_outputs/multiple_outputs.ipynb +++ b/samples/core/multiple_outputs/multiple_outputs.ipynb @@ -93,12 +93,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Returns a*b and a+b\n", - "@dsl.python_component(\n", - " name='product_sum',\n", - " description='Calculates the product and the sum',\n", - " base_image=BASE_IMAGE\n", - ")\n", "def product_sum(a: float, b: float) -> NamedTuple(\n", " 'output', [('product', float), ('sum', float)]):\n", " '''Returns the product and sum of two numbers'''\n",