[TEP-0076]Support Array Results substitution

This is part of work in TEP-0076.
This commit provides the support to apply array results replacements.
Previous this commit we support emitting array results so users can
write array results to task level, but we cannot pass array results from
tasks within one pipeline. This commit adds the support for this.

pkg/apis/pipeline/v1beta1/result_types.go

@@ -13,6 +13,8 @@ limitations under the License.
 
 package v1beta1
 
+import "strings"
+
 // TaskResult used to describe the results of a task
 type TaskResult struct {
 	// Name the given name
@@ -60,3 +62,9 @@ const (
 
 // AllResultsTypes can be used for ResultsTypes validation.
 var AllResultsTypes = []ResultsType{ResultsTypeString, ResultsTypeArray, ResultsTypeObject}
+
+// ArrayReference returns the name of the parameter from array parameter reference
+// returns arrayParam from $(params.arrayParam[*])
+func ResultsArrayReference(a string) string {
+	return strings.TrimSuffix(strings.TrimPrefix(a, "$("), ")")
+}

pkg/apis/pipeline/v1beta1/resultref.go

@@ -35,13 +35,17 @@ const (
 	// ResultResultPart Constant used to define the "results" part of a pipeline result reference
 	ResultResultPart = "results"
 	// TODO(#2462) use one regex across all substitutions
-	variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*\)`
+	// variableSubstitutionFormat matches format like $result.resultname[int] and $result.resultname[*]
+	variableSubstitutionFormat = `\$\([_a-zA-Z0-9.-]+(\.[_a-zA-Z0-9.-]+)*(\[([0-9])*\*?\])?\)`
+	// excludeArrayIndexing will replace all `[int]` and `[*]` for parseExpression to extract result name
+	excludeArrayIndexing = `\[([0-9])*\*?\]`
 	// ResultNameFormat Constant used to define the the regex Result.Name should follow
 	ResultNameFormat = `^([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$`
 )
 
-var variableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
+var VariableSubstitutionRegex = regexp.MustCompile(variableSubstitutionFormat)
 var resultNameFormatRegex = regexp.MustCompile(ResultNameFormat)
+var excludeArrayIndexingRegex = regexp.MustCompile(excludeArrayIndexing)
 
 // NewResultRefs extracts all ResultReferences from a param or a pipeline result.
 // If the ResultReference can be extracted, they are returned. Expressions which are not
@@ -107,7 +111,7 @@ func GetVarSubstitutionExpressionsForPipelineResult(result PipelineResult) ([]st
 }
 
 func validateString(value string) []string {
-	expressions := variableSubstitutionRegex.FindAllString(value, -1)
+	expressions := VariableSubstitutionRegex.FindAllString(value, -1)
 	if expressions == nil {
 		return nil
 	}
@@ -127,6 +131,7 @@ func parseExpression(substitutionExpression string) (string, string, error) {
 	if len(subExpressions) != 4 || subExpressions[0] != ResultTaskPart || subExpressions[2] != ResultResultPart {
 		return "", "", fmt.Errorf("Must be of the form %q", resultExpressionFormat)
 	}
+	subExpressions[3] = excludeArrayIndexingRegex.ReplaceAllString(subExpressions[3], "")
 	return subExpressions[1], subExpressions[3], nil
 }
 

pkg/apis/pipeline/v1beta1/when_types.go

@@ -62,10 +62,14 @@ func (we *WhenExpression) applyReplacements(replacements map[string]string, arra
 	for _, val := range we.Values {
 		// arrayReplacements holds a list of array parameters with a pattern - params.arrayParam1
 		// array params are referenced using $(params.arrayParam1[*])
+		// array results are referenced using $(results.resultname[*])
 		// check if the param exist in the arrayReplacements to replace it with a list of values
+		fmt.Println(fmt.Sprintf("%s", ResultsArrayReference(val)))
 		if _, ok := arrayReplacements[fmt.Sprintf("%s.%s", ParamsPrefix, ArrayReference(val))]; ok {
 			replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
-		} else {
+		} else if _, ok := arrayReplacements[fmt.Sprintf("%s", ResultsArrayReference(val))]; ok{
+			replacedValues = append(replacedValues, substitution.ApplyArrayReplacements(val, replacements, arrayReplacements)...)
+		}else{
 			replacedValues = append(replacedValues, substitution.ApplyReplacements(val, replacements))
 		}
 	}

pkg/reconciler/pipeline/dag/dag_test.go

@@ -371,6 +371,7 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
 	c := v1beta1.PipelineTask{Name: "c"}
 	d := v1beta1.PipelineTask{Name: "d"}
 	e := v1beta1.PipelineTask{Name: "e"}
+	f := v1beta1.PipelineTask{Name: "f"}
 	xDependsOnA := v1beta1.PipelineTask{
 		Name: "x",
 		Params: []v1beta1.Param{{
@@ -393,27 +394,38 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
 			Value: *v1beta1.NewArrayOrString("$(tasks.d.results.resultD) $(tasks.e.results.resultE)"),
 		}},
 	}
+	wDependsOnF := v1beta1.PipelineTask{
+		Name: "w",
+		Params: []v1beta1.Param{{
+			Name:  "paramw",
+			Value: *v1beta1.NewArrayOrString("$(tasks.f.results.resultF[*])"),
+		}},
+	}
 
-	//   a  b   c  d   e
-	//   |   \ /    \ /
-	//   x    y      z
+	//   a  b   c  d   e  f
+	//   |   \ /    \ /   |
+	//   x    y      z    w
 	nodeA := &dag.Node{Task: a}
 	nodeB := &dag.Node{Task: b}
 	nodeC := &dag.Node{Task: c}
 	nodeD := &dag.Node{Task: d}
 	nodeE := &dag.Node{Task: e}
+	nodeF := &dag.Node{Task: f}
 	nodeX := &dag.Node{Task: xDependsOnA}
 	nodeY := &dag.Node{Task: yDependsOnBRunsAfterC}
 	nodeZ := &dag.Node{Task: zDependsOnDAndE}
+	nodeW := &dag.Node{Task: wDependsOnF}
 
 	nodeA.Next = []*dag.Node{nodeX}
 	nodeB.Next = []*dag.Node{nodeY}
 	nodeC.Next = []*dag.Node{nodeY}
 	nodeD.Next = []*dag.Node{nodeZ}
 	nodeE.Next = []*dag.Node{nodeZ}
+	nodeF.Next = []*dag.Node{nodeW}
 	nodeX.Prev = []*dag.Node{nodeA}
 	nodeY.Prev = []*dag.Node{nodeB, nodeC}
 	nodeZ.Prev = []*dag.Node{nodeD, nodeE}
+	nodeW.Prev = []*dag.Node{nodeF}
 
 	expectedDAG := &dag.Graph{
 		Nodes: map[string]*dag.Node{
@@ -422,15 +434,17 @@ func TestBuild_TaskParamsFromTaskResults(t *testing.T) {
 			"c": nodeC,
 			"d": nodeD,
 			"e": nodeE,
+			"f": nodeF,
 			"x": nodeX,
 			"y": nodeY,
 			"z": nodeZ,
+			"w": nodeW,
 		},
 	}
 	p := &v1beta1.Pipeline{
 		ObjectMeta: metav1.ObjectMeta{Name: "pipeline"},
 		Spec: v1beta1.PipelineSpec{
-			Tasks: []v1beta1.PipelineTask{a, b, c, d, e, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE},
+			Tasks: []v1beta1.PipelineTask{a, b, c, d, e, f, xDependsOnA, yDependsOnBRunsAfterC, zDependsOnDAndE, wDependsOnF},
 		},
 	}
 	tasks := v1beta1.PipelineTaskList(p.Spec.Tasks)

pkg/reconciler/pipelinerun/resources/apply.go

@@ -99,17 +99,18 @@ func ApplyPipelineTaskContexts(pt *v1beta1.PipelineTask) *v1beta1.PipelineTask {
 // ApplyTaskResults applies the ResolvedResultRef to each PipelineTask.Params and Pipeline.WhenExpressions in targets
 func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResultRefs) {
 	stringReplacements := resolvedResultRefs.getStringReplacements()
+	arrayReplacements := resolvedResultRefs.getArrayReplacements()
 	for _, resolvedPipelineRunTask := range targets {
 		// also make substitution for resolved condition checks
 		for _, resolvedConditionCheck := range resolvedPipelineRunTask.ResolvedConditionChecks {
 			pipelineTaskCondition := resolvedConditionCheck.PipelineTaskCondition.DeepCopy()
-			pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, nil)
+			pipelineTaskCondition.Params = replaceParamValues(pipelineTaskCondition.Params, stringReplacements, arrayReplacements)
 			resolvedConditionCheck.PipelineTaskCondition = pipelineTaskCondition
 		}
 		if resolvedPipelineRunTask.PipelineTask != nil {
 			pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy()
-			pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, nil)
-			pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, nil)
+			pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements)
+			pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
 			resolvedPipelineRunTask.PipelineTask = pipelineTask
 		}
 	}

pkg/reconciler/pipelinerun/resources/apply_test.go

@@ -401,6 +401,38 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
 				}},
 			},
 		}},
+	}, {
+		name: "Test array result substitution on minimal variable substitution expression - params",
+		resolvedResultRefs: ResolvedResultRefs{{
+			Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+			ResultReference: v1beta1.ResultRef{
+				PipelineTask: "aTask",
+				Result:       "a.Result",
+			},
+			FromTaskRun: "aTaskRun",
+		}},
+		targets: PipelineRunState{{
+			PipelineTask: &v1beta1.PipelineTask{
+				Name:    "bTask",
+				TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+				Params: []v1beta1.Param{{
+					Name: "bParam",
+					Value: v1beta1.ArrayOrString{Type: v1beta1.ParamTypeArray,
+						ArrayVal: []string{`$(tasks.aTask.results["a.Result"][*])`},
+					},
+				}},
+			},
+		}},
+		want: PipelineRunState{{
+			PipelineTask: &v1beta1.PipelineTask{
+				Name:    "bTask",
+				TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+				Params: []v1beta1.Param{{
+					Name:  "bParam",
+					Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+				}},
+			},
+		}},
 	}, {
 		name: "Test result substitution on minimal variable substitution expression - when expressions",
 		resolvedResultRefs: ResolvedResultRefs{{
@@ -433,6 +465,39 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) {
 				}},
 			},
 		}},
+	}, {
+		name: "Test array result substitution on minimal variable substitution expression - when expressions",
+		resolvedResultRefs: ResolvedResultRefs{{
+			Value: *v1beta1.NewArrayOrString("arrayResultValueOne", "arrayResultValueTwo"),
+			ResultReference: v1beta1.ResultRef{
+				PipelineTask: "aTask",
+				Result:       "aResult",
+			},
+			FromTaskRun: "aTaskRun",
+		}},
+		targets: PipelineRunState{{
+			PipelineTask: &v1beta1.PipelineTask{
+				Name:    "bTask",
+				TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+				WhenExpressions: []v1beta1.WhenExpression{{
+					// Note that Input doesn't support array replacement.
+					Input:    "anInput",
+					Operator: selection.In,
+					Values:   []string{"$(tasks.aTask.results.aResult[*])"},
+				}},
+			},
+		}},
+		want: PipelineRunState{{
+			PipelineTask: &v1beta1.PipelineTask{
+				Name:    "bTask",
+				TaskRef: &v1beta1.TaskRef{Name: "bTask"},
+				WhenExpressions: []v1beta1.WhenExpression{{
+					Input:    "anInput",
+					Operator: selection.In,
+					Values:   []string{"arrayResultValueOne", "arrayResultValueTwo"},
+				}},
+			},
+		}},
 	}} {
 		t.Run(tt.name, func(t *testing.T) {
 			ApplyTaskResults(tt.targets, tt.resolvedResultRefs)

pkg/reconciler/pipelinerun/resources/resultrefresolution.go

@@ -187,8 +187,22 @@ func findTaskResultForParam(taskRun *v1beta1.TaskRun, reference *v1beta1.ResultR
 func (rs ResolvedResultRefs) getStringReplacements() map[string]string {
 	replacements := map[string]string{}
 	for _, r := range rs {
-		for _, target := range r.getReplaceTarget() {
-			replacements[target] = r.Value.StringVal
+		if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeString) {
+			for _, target := range r.getReplaceTarget() {
+				replacements[target] = r.Value.StringVal
+			}
+		}
+	}
+	return replacements
+}
+
+func (rs ResolvedResultRefs) getArrayReplacements() map[string][]string {
+	replacements := map[string][]string{}
+	for _, r := range rs {
+		if r.Value.Type == v1beta1.ParamType(v1beta1.ResultsTypeArray) {
+			for _, target := range r.getArrayReplaceTarget() {
+				replacements[target] = r.Value.ArrayVal
+			}
 		}
 	}
 	return replacements
@@ -201,3 +215,11 @@ func (r *ResolvedResultRef) getReplaceTarget() []string {
 		fmt.Sprintf("%s.%s.%s['%s']", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
 	}
 }
+
+func (r *ResolvedResultRef) getArrayReplaceTarget() []string {
+	return []string{
+		fmt.Sprintf("%s.%s.%s.%s[*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+		fmt.Sprintf("%s.%s.%s[%q][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+		fmt.Sprintf("%s.%s.%s['%s'][*]", v1beta1.ResultTaskPart, r.ResultReference.PipelineTask, v1beta1.ResultResultPart, r.ResultReference.Result),
+	}
+}

pkg/reconciler/taskrun/validate_resources.go

@@ -78,6 +78,11 @@ func validateParams(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params
 	if extraParamsNames := extraParamsNames(ctx, neededParamsNames, providedParamsNames); len(extraParamsNames) != 0 {
 		return fmt.Errorf("didn't need these params but they were provided anyway: %s", extraParamsNames)
 	}
+	// This is needed to support array replacements in params.Users want to use $(results.resultname[*])
+	// to pass array result to array param, yet in yaml format this will be
+	// unmarshalled to string for ArrayOrString. So we need to check and correct the param type.
+	// Please check issue #4879 for more details and examples.
+	correctAoStype(params, neededParamsTypes)
 	if wrongTypeParamNames := wrongTypeParamsNames(params, matrix, neededParamsTypes); len(wrongTypeParamNames) != 0 {
 		return fmt.Errorf("param types don't match the user-specified type: %s", wrongTypeParamNames)
 	}
@@ -88,6 +93,16 @@ func validateParams(ctx context.Context, paramSpecs []v1beta1.ParamSpec, params
 	return nil
 }
 
+func correctAoStype(params []v1beta1.Param, neededParamsTypes map[string]v1beta1.ParamType){
+	for i :=0; i<len(params); i++{
+		if params[i].Value.Type == "string" && neededParamsTypes[params[i].Name] == "array" && v1beta1.VariableSubstitutionRegex.MatchString(params[i].Value.StringVal){
+			params[i].Value.Type = neededParamsTypes[params[i].Name]
+			params[i].Value.ArrayVal = []string{params[i].Value.StringVal}
+			params[i].Value.StringVal = ""
+		}
+	}
+}
+
 func neededParamsNamesAndTypes(paramSpecs []v1beta1.ParamSpec) ([]string, map[string]v1beta1.ParamType) {
 	var neededParamsNames []string
 	neededParamsTypes := make(map[string]v1beta1.ParamType)

pkg/reconciler/taskrun/validate_resources_test.go

@@ -142,6 +142,9 @@ func TestValidateResolvedTaskResources_ValidParams(t *testing.T) {
 					Name: "zoo",
 					Type: v1beta1.ParamTypeString,
 				}, {
+					Name: "arrayResultRef",
+					Type: v1beta1.ParamTypeArray,
+				},{
 					Name: "myobj",
 					Type: v1beta1.ParamTypeObject,
 					Properties: map[string]v1beta1.PropertySpec{
@@ -161,6 +164,9 @@ func TestValidateResolvedTaskResources_ValidParams(t *testing.T) {
 	}, {
 		Name:  "bar",
 		Value: *v1beta1.NewArrayOrString("somethinggood"),
+	}, {
+		Name:  "arrayResultRef",
+		Value: *v1beta1.NewArrayOrString("$(results.resultname)"),
 	}, {
 		Name: "myobj",
 		Value: *v1beta1.NewObject(map[string]string{
@@ -550,3 +556,19 @@ func TestValidateOverrides(t *testing.T) {
 		})
 	}
 }
+
+func TestCorrectAoStype(t *testing.T) {
+	params:=[]v1beta1.Param{{
+		Name:  "foo",
+		Value: *v1beta1.NewArrayOrString("$(results.bar[*])"),
+	}}
+
+	paramSpecs:=[]v1beta1.ParamSpec{
+		{
+			Name: "foo",
+			Type: v1beta1.ParamTypeArray,
+		},
+	}
+	_, neededParamsTypes := neededParamsNamesAndTypes(paramSpecs)
+	correctAoStype(params, neededParamsTypes)
+}