Enhance Any Sequencer to hanle cases that some tasks skipped (kubeflo…

…w#653)
red-hat-data-services · Jul 5, 2021 · 77963c3 · 77963c3
1 parent 54e8aaf
commit 77963c3
Show file tree

Hide file tree

Showing 11 changed files with 127 additions and 118 deletions.
diff --git a/frontend/src/lib/StaticGraphParser.ts b/frontend/src/lib/StaticGraphParser.ts
@@ -152,7 +152,7 @@ function buildTektonDag(graph: dagre.graphlib.Graph, template: any): void {
       task['taskSpec']['steps'][0]['args'] &&
       task['taskSpec']['steps'][0]['command'] &&
       task['taskSpec']['steps'][0]['command'][0] &&
-      task['taskSpec']['steps'][0]['command'][0] === 'any-taskrun'
+      task['taskSpec']['steps'][0]['command'][0] === 'any-task'
     ) {
       let isNextTaskList = false;
       let isNextCondition = false;

diff --git a/frontend/src/lib/WorkflowParser.ts b/frontend/src/lib/WorkflowParser.ts
@@ -140,7 +140,7 @@ export default class WorkflowParser {
           task['taskSpec']['steps'][0]['args'] &&
           task['taskSpec']['steps'][0]['command'] &&
           task['taskSpec']['steps'][0]['command'][0] &&
-          task['taskSpec']['steps'][0]['command'][0] === 'any-taskrun'
+          task['taskSpec']['steps'][0]['command'][0] === 'any-task'
         ) {
           let isNextTaskList = false;
           let isNextCondition = false;

diff --git a/sdk/FEATURES.md b/sdk/FEATURES.md
@@ -144,9 +144,11 @@ To see how the Python SDK provides this feature, refer to the examples below:
 
 When any one of the task dependencies completes successfully and the conditions meet, the dependent task will be started. Order of execution of the dependencies doesn’t matter, and the pipeline doesn't wait for all the task dependencies to complete before moving to the next step. Condition can be applied to enforce the task dependencies completes as expected. The condition expression should be the same format as is in Kubeflow ConditionOperator, and the result of containerOps can be used in expression. Notice the expression should only contain results from only one task because the purpose here is to check the simple condition for the task's output when a task complete. And also the operand in the expression should be int or string, other python types will be transferred to string automatically. 
 
-The exit status of any-sequencer is configurable through parameters, `statusPath` must be specified for any-sequencer to write status to, then use `skippingPolicy` and `errorPolicy`.
-- skippingPolicy --- determines for the AnySequencer reacts to no-dependency-condition-matching case. Values can be one of `skipOnNoMatch` or `errorOnNoMatch`, a status with value "Skipped" will be generated and the exit status will still be succeeded on `skipOnNoMatch`.
-- errorPolicy --- the standard field, either `failOnError` or `continueOnError`. On `continueOnError`, a status with value "Failed" will be generated but the exit status will still be succeeded. For `Fail_on_error` the AnySequencer should truly fail in the Tekton terms, as it does now.
+The Any Sequencer exits if all dependencies failed or skipped, or all conditions unmatched.
+
+The exit status of Any Sequencer is configurable through parameters, `statusPath` must be specified for Any Sequencer to write status to, then use `skippingPolicy` and `errorPolicy`.
+- skippingPolicy --- determines for the Any Sequencer reacts to no-dependency-condition-matching case. Values can be one of `skipOnNoMatch` or `errorOnNoMatch`, a status with value "Skipped" will be generated and the exit status will still be succeeded on `skipOnNoMatch`.
+- errorPolicy --- the standard field, either `failOnError` or `continueOnError`. On `continueOnError`, a status with value "Failed" will be generated but the exit status will still be succeeded. For `Fail_on_error` the Any Sequencer should truly fail in the Tekton terms, as it does now.
 
 Please follow the details of the implementation in the [design doc](https://docs.google.com/document/d/1oXOdiItI4GbEe_qzyBmMAqfLBjfYX1nM94WHY3EPa94/edit#heading=h.dt8bhna4spym).
 

diff --git a/sdk/python/kfp_tekton/tekton.py b/sdk/python/kfp_tekton/tekton.py
@@ -76,7 +76,7 @@ def __init__(self,
             name=name,
             image=image,
             file_outputs=file_outputs,
-            command="any-taskrun",
+            command="any-task",
             arguments=arguments,
         )
 

diff --git a/sdk/python/tests/compiler/testdata/any_sequencer.yaml b/sdk/python/tests/compiler/testdata/any_sequencer.yaml
@@ -165,7 +165,7 @@ spec:
           - -c
           - results_flipcoin_output == 'heads'
           command:
-          - any-taskrun
+          - any-task
           image: dspipelines/any-sequencer:latest
           name: main
       timeout: 0s

diff --git a/sdk/python/tests/compiler/testdata/any_sequencer_noninlined.yaml b/sdk/python/tests/compiler/testdata/any_sequencer_noninlined.yaml
@@ -165,7 +165,7 @@ spec:
           - -c
           - results_flipcoin_output == 'heads'
           command:
-          - any-taskrun
+          - any-task
           image: dspipelines/any-sequencer:latest
           name: main
       timeout: 0s

diff --git a/tekton-catalog/any-sequencer/Dockerfile b/tekton-catalog/any-sequencer/Dockerfile
@@ -2,5 +2,5 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal
 
 ARG bin_dir=_output/bin
 
-COPY ${bin_dir}/any-taskrun /usr/local/bin
+COPY ${bin_dir}/any-task /usr/local/bin
 
diff --git a/tekton-catalog/any-sequencer/Makefile b/tekton-catalog/any-sequencer/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2020 kubeflow.org
+# Copyright 2021 kubeflow.org
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,10 +24,10 @@ init:
 	mkdir -p ${BIN_DIR}
 
 local: init
-	go build -o=${BIN_DIR}/any-taskrun
+	go build -o=${BIN_DIR}/any-task
 
 build-linux: init
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o=${BIN_DIR}/any-taskrun
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o=${BIN_DIR}/any-task
 
 image: build-linux
 	docker build  . -t docker.io/dspipelines/any-sequencer:$(TAG)

diff --git a/tekton-catalog/any-sequencer/README.md b/tekton-catalog/any-sequencer/README.md
@@ -2,9 +2,9 @@
 
 `Any Sequencer`: When any one of the task dependencies complete successfully, the dependent task will be started. Order of execution of the dependencies doesn’t matter, e.g. if Job4 depends on Job1, Job2 and Job3, and when any one of the Job1, Job2 or Job3 complete successfully, Job4 will be started. Order doesn’t matter, and `Any Sequencer` doesn’t wait for all the job completions.
 
-This implements a `taskRun` status watcher task to watch the list of `taskRun` it depends on (using Kubernetes RBAC). If one or more is completed, it exits the watching task and continues moving forward with Pipeline run. This status watcher task can be implemented in the same way as our “condition” task to make it “dependable”. 
+This implements a `pipelineRun` status watcher task to watch the list of `taskRun` or `Run` it depends on (using Kubernetes RBAC). If one or more is completed, it exits the watching task and continues moving forward with Pipeline run. This status watcher task can be implemented in the same way as our “condition” task to make it “dependable”. 
 
-Note that the service account of `Any Sequencer` needs `get` permission to watch the status of the specified `taskRun`.
+Note that the service account of `Any Sequencer` needs `get` permission to watch the status of the `pipelineRun`.
 
 ### How to build binary