From bf3c48aa5eb90723b7b2abaef26e3ab75b36ea92 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Tue, 4 Jun 2019 10:46:35 -0700 Subject: [PATCH 1/2] expose add_op_transformer in the PipelineConf and add an example --- samples/basic/pipeline_transformers.py | 44 ++++++++++++++++++++++++++ sdk/python/kfp/dsl/_pipeline.py | 8 +++++ 2 files changed, 52 insertions(+) create mode 100644 samples/basic/pipeline_transformers.py diff --git a/samples/basic/pipeline_transformers.py b/samples/basic/pipeline_transformers.py new file mode 100644 index 00000000000..73a7aadddb6 --- /dev/null +++ b/samples/basic/pipeline_transformers.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import kfp +from kfp import dsl + + +def random_failure_op(exit_codes): + """A component that fails randomly.""" + return dsl.ContainerOp( + name='random_failure', + image='python:alpine3.6', + command=['python', '-c'], + arguments=['import random; import sys; exit_code = random.choice(sys.argv[1].split(",")); print(exit_code); sys.exit(exit_code)', exit_codes] + ) + +def add_retry(op): + op.set_retry(5) + return op + +@dsl.pipeline( + name='Retry random failures', + description='The pipeline includes two steps which fail randomly. It shows how to use ContainerOp(...).set_retry(...).' +) +def retry_sample_pipeline(): + op1 = random_failure_op('0,1,2,3') + op2 = random_failure_op('0,1') + dsl.get_pipeline_conf().add_op_transformer(add_retry) + +if __name__ == '__main__': + kfp.compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.zip') diff --git a/sdk/python/kfp/dsl/_pipeline.py b/sdk/python/kfp/dsl/_pipeline.py index 91a9584a04d..081adb8ca2a 100644 --- a/sdk/python/kfp/dsl/_pipeline.py +++ b/sdk/python/kfp/dsl/_pipeline.py @@ -104,6 +104,14 @@ def foo_pipeline(tag: str, pull_image_policy: str): self.artifact_location = artifact_location return self + def add_op_transformer(self, transformer): + """Configures the op_transformers which will be applied to all ops in the pipeline. + + Args: + transformer: a function that takes a ContainOp as input and returns a ContainerOp + """ + self.op_transformers.append(transformer) + def get_pipeline_conf(): """Configure the pipeline level setting to the current pipeline Note: call the function inside the user defined pipeline function. From 69e9a4ede96c972a61122832359932d589be12fa Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Thu, 6 Jun 2019 12:00:09 -0700 Subject: [PATCH 2/2] fix frontend integ test version --- test/frontend-integration-test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/frontend-integration-test/Dockerfile b/test/frontend-integration-test/Dockerfile index 8b931081d22..e76250fa878 100644 --- a/test/frontend-integration-test/Dockerfile +++ b/test/frontend-integration-test/Dockerfile @@ -1,4 +1,4 @@ -FROM selenium/standalone-chrome +FROM selenium/standalone-chrome:3.141.59-oxygen USER root