From 97eba028763894b4eebe03083fd46a3dda5f7372 Mon Sep 17 00:00:00 2001 From: Emma Munley <46881325+EmmaMunley@users.noreply.github.com> Date: Fri, 28 Apr 2023 17:44:57 -0400 Subject: [PATCH] Add support for consuming whole array results in matrix This commit adds support for referencing whole array results produced by another PipelineTask as a matrix parameter value. This feature request is necessary before promoting Matrix to beta. For more details please see issue #6110. This closes issue #6602. Co-authored-by: Yongxuan Zhang --- docs/matrix.md | 20 +- .../pipelinerun-with-matrix-and-results.yaml | 7 +- pkg/apis/pipeline/v1/matrix_types.go | 21 - pkg/apis/pipeline/v1/pipeline_types_test.go | 13 - pkg/apis/pipeline/v1/pipeline_validation.go | 1 - .../pipeline/v1/pipeline_validation_test.go | 10 + pkg/apis/pipeline/v1beta1/matrix_types.go | 19 - .../pipeline/v1beta1/pipeline_types_test.go | 13 - .../pipeline/v1beta1/pipeline_validation.go | 1 - .../v1beta1/pipeline_validation_test.go | 10 + .../pipelinerun/pipelinerun_test.go | 381 +++++++++++++++++- pkg/reconciler/pipelinerun/resources/apply.go | 3 +- .../resources/pipelinerunresolution.go | 20 + .../resources/pipelinerunresolution_test.go | 39 ++ .../example_customrun_matrix_results.yaml | 57 +++ 15 files changed, 529 insertions(+), 86 deletions(-) create mode 100644 test/custom-task-ctrls/wait-task-beta/example_customrun_matrix_results.yaml diff --git a/docs/matrix.md b/docs/matrix.md index fcffb8e7440..9727bc525c5 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -303,7 +303,7 @@ See the end-to-end example in [`PipelineRun` with `Matrix` and `Results`][pr-wit #### Results in Matrix.Params -`Matrix.Params` supports string replacements from `Results` of type String, Array or Object. +`Matrix.Params` supports whole array replacements and string replacements from `Results` of type String, Array or Object ```yaml tasks: @@ -314,16 +314,9 @@ tasks: matrix: params: - name: values - value: - - $(tasks.task-1.results.foo) # string replacement from string result - - $(tasks.task-2.results.bar[0]) # string replacement from array result - - $(tasks.task-3.results.rad.key) # string replacement from object result + value: $(tasks.task-4.results.whole-array[*]) ``` -For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results]. - -We plan to add support passing whole Array `Results` into the `Matrix` (#5925): - ```yaml tasks: ... @@ -333,9 +326,16 @@ tasks: matrix: params: - name: values - value: $(tasks.task-4.results.foo) # array + value: + - $(tasks.task-1.results.a-string-result) + - $(tasks.task-2.results.an-array-result[0]) + - $(tasks.task-3.results.an-object-result.key) ``` + +For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results]. + + #### Results in Matrix.Include.Params `Matrix.Include.Params` supports string replacements from `Results` of type String, Array or Object. diff --git a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml index 20408bb168f..6ebd50e0f8e 100644 --- a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml +++ b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml @@ -21,7 +21,7 @@ kind: PipelineRun metadata: generateName: matrixed-pr- spec: - serviceAccountName: 'default' + serviceAccountName: "default" pipelineSpec: tasks: - name: get-platforms @@ -56,10 +56,7 @@ spec: matrix: params: - name: platform - value: - - $(tasks.get-platforms.results.platforms[0]) - - $(tasks.get-platforms.results.platforms[1]) - - $(tasks.get-platforms.results.platforms[2]) + value: $(tasks.get-platforms.results.platforms[*]) - name: browser value: - $(tasks.get-browsers-and-url.results.browsers[0]) diff --git a/pkg/apis/pipeline/v1/matrix_types.go b/pkg/apis/pipeline/v1/matrix_types.go index 03fba3f29aa..f51a0ac4af4 100644 --- a/pkg/apis/pipeline/v1/matrix_types.go +++ b/pkg/apis/pipeline/v1/matrix_types.go @@ -348,24 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params []Param) (errs *a } return errs } - -// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references -// to entire arrays. This is temporary until whole array replacements for matrix parameters are supported. -// See issue #6056 for more details -func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) { - if m.HasParams() { - for i, param := range m.Params { - val := param.Value.StringVal - expressions, ok := GetVarSubstitutionExpressionsForParam(param) - if ok { - if LooksLikeContainsResultRefs(expressions) { - _, stringIdx := ParseResultName(val) - if stringIdx == "*" { - errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i)) - } - } - } - } - } - return errs -} diff --git a/pkg/apis/pipeline/v1/pipeline_types_test.go b/pkg/apis/pipeline/v1/pipeline_types_test.go index d0249b463ff..712e0bdd324 100644 --- a/pkg/apis/pipeline/v1/pipeline_types_test.go +++ b/pkg/apis/pipeline/v1/pipeline_types_test.go @@ -711,19 +711,6 @@ func TestPipelineTask_ValidateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}}, }}}, }, - }, { - name: "parameters in matrix contain whole array results references", - pt: &PipelineTask{ - Name: "task", - Matrix: &Matrix{ - Params: Params{{ - Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, - }}}, - }, - wantErrs: &apis.FieldError{ - Message: "matrix parameters cannot contain whole array result references", - Paths: []string{"matrix.params[0]"}, - }, }, { name: "count of combinations of parameters in the matrix exceeds the maximum", pt: &PipelineTask{ diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go index 4a426ebf0c3..72ede789240 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1/pipeline_validation.go @@ -161,7 +161,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr // when the enable-api-fields feature gate is anything but "alpha". errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) - errs = errs.Also(pt.Matrix.validateNoWholeArrayResults()) errs = errs.Also(pt.Matrix.validateUniqueParams()) } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go index a039914dba9..ebbf14be7bc 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1/pipeline_validation_test.go @@ -3103,6 +3103,16 @@ func Test_validateMatrix(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}}, }}}, }}, + }, { + name: "parameters in matrix contain whole array results references", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, + }}}, + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1beta1/matrix_types.go b/pkg/apis/pipeline/v1beta1/matrix_types.go index ac44ff6891f..19042e11e4b 100644 --- a/pkg/apis/pipeline/v1beta1/matrix_types.go +++ b/pkg/apis/pipeline/v1beta1/matrix_types.go @@ -17,7 +17,6 @@ import ( "context" "fmt" "sort" - "strings" "github.com/tektoncd/pipeline/pkg/apis/config" "golang.org/x/exp/maps" @@ -349,21 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params Params) (errs *ap } return errs } - -// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references -// to entire arrays. This is temporary until whole array replacements for matrix paraemeters are supported. -// See issue #6056 for more details -func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) { - if m.HasParams() { - for i, param := range m.Params { - val := param.Value.StringVal - expressions, ok := GetVarSubstitutionExpressionsForParam(param) - if ok { - if LooksLikeContainsResultRefs(expressions) && strings.Contains(val, "[*]") { - errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i)) - } - } - } - } - return errs -} diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go index f137ae152a6..42bf0a8e996 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go @@ -684,19 +684,6 @@ func TestPipelineTask_validateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}}, }}}, }, - }, { - name: "parameters in matrix contain whole array results references", - pt: &PipelineTask{ - Name: "task", - Matrix: &Matrix{ - Params: Params{{ - Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, - }}}, - }, - wantErrs: &apis.FieldError{ - Message: "matrix parameters cannot contain whole array result references", - Paths: []string{"matrix.params[0]"}, - }, }, { name: "count of combinations of parameters in the matrix exceeds the maximum", pt: &PipelineTask{ diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go index 33f27701af7..8cbdccd6bc5 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation.go @@ -172,7 +172,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr // when the enable-api-fields feature gate is anything but "alpha". errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) - errs = errs.Also(pt.Matrix.validateNoWholeArrayResults()) errs = errs.Also(pt.Matrix.validateUniqueParams()) } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go index b269844caec..791a563b739 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go @@ -3146,6 +3146,16 @@ func Test_validateMatrix(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}}, }}}, }}, + }, { + name: "parameters in matrix contain whole array results references", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, + }}}, + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index 1ca8eb3106d..68becce66d3 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -9817,7 +9817,7 @@ spec: } } -func TestReconciler_PipelineTaskMatrixResultsWithArrayIndexing(t *testing.T) { +func TestReconciler_PipelineTaskMatrixResultsWithArrays(t *testing.T) { names.TestingSeed() task := parse.MustParseV1beta1Task(t, ` metadata: @@ -10142,6 +10142,152 @@ status: EnableProvenanceInStatus: true ResultExtractionMethod: "termination-message" MaxResultSize: 4096 +`), + }, { + name: "whole array result replacements in matrix.params", + pName: "p-dag-3", + p: parse.MustParseV1beta1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + - name: echo-platforms + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + taskRef: + name: mytask +`, "p-dag-3")), + tr: mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-with-result", "foo", + "pr", "p-dag-3", "pt-with-result", false), + ` +spec: + serviceAccountName: test-sa + taskRef: + name: taskwithresults +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + taskResults: + - name: platforms + value: + - linux + - mac + - windows +`), + expectedTaskRuns: []*v1beta1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-0", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: linux + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-1", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: mac + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-2", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: windows + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + }, + expectedPipelineRun: parse.MustParseV1beta1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag-3 +spec: + serviceAccountName: test-sa + pipelineRef: + name: p-dag-3 +status: + pipelineSpec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + kind: Task + - name: echo-platforms + taskRef: + name: mytask + kind: Task + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-pt-with-result + pipelineTaskName: pt-with-result + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-0 + pipelineTaskName: echo-platforms + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-1 + pipelineTaskName: echo-platforms + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-2 + pipelineTaskName: echo-platforms `), }} for _, tt := range tests { @@ -11191,6 +11337,239 @@ spec: } } +func TestReconciler_CustomPipelineTaskMatrixResultsWholeArray(t *testing.T) { + names.TestingSeed() + + taskwithresults := parse.MustParseV1beta1Task(t, ` +metadata: + name: taskwithresults + namespace: foo +spec: + results: + - name: platforms + type: array + steps: + - name: produce-a-list-of-platforms + image: bash:latest + script: | + #!/usr/bin/env bash + echo -n "[\"linux\",\"mac\",\"windows\"]" | tee $(results.platforms.path) +`) + expectedCustomRuns := []*v1beta1.CustomRun{ + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-0", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: linux + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-1", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: mac + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-2", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: windows + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + } + + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + + tests := []struct { + name string + memberOf string + p *v1beta1.Pipeline + tr *v1beta1.TaskRun + expectedPipelineRun *v1beta1.PipelineRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParseV1beta1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + - name: pt-matrix-custom-task + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + taskRef: + apiVersion: example.dev/v0 + kind: Example +`, "p-dag")), + + tr: mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-with-result", "foo", + "pr", "p-dag", "pt-with-result", false), + ` +spec: + serviceAccountName: test-sa + taskRef: + name: taskwithresults +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + taskResults: + - name: platforms + value: + - linux + - mac + - windows +`), + expectedPipelineRun: parse.MustParseV1beta1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag +spec: + serviceAccountName: test-sa + pipelineRef: + name: p-dag +status: + pipelineSpec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + kind: Task + - name: pt-matrix-custom-task + taskRef: + apiVersion: example.dev/v0 + kind: Example + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-pt-with-result + pipelineTaskName: pt-with-result + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-0 + pipelineTaskName: pt-matrix-custom-task + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-1 + pipelineTaskName: pt-matrix-custom-task + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-2 + pipelineTaskName: pt-matrix-custom-task +`), + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParseV1beta1PipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1beta1.PipelineRun{pr}, + Pipelines: []*v1beta1.Pipeline{tt.p}, + Tasks: []*v1beta1.Task{taskwithresults}, + ConfigMaps: cms, + } + if tt.tr != nil { + d.TaskRuns = []*v1beta1.TaskRun{tt.tr} + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + + pipelineRun, clients := prt.reconcileRun("foo", "pr", []string{}, false) + customRuns, err := clients.Pipeline.TektonV1beta1().CustomRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failure to list customRuns's %s", err) + } + + if len(customRuns.Items) != 3 { + t.Fatalf("Expected 3 customRuns got %d", len(customRuns.Items)) + } + + for i := range customRuns.Items { + expectedCustomRun := expectedCustomRuns[i] + if d := cmp.Diff(expectedCustomRun, &customRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see CustomRun %v created. Diff %s", expectedCustomRun.Name, diff.PrintWantGot(d)) + } + } + + if d := cmp.Diff(tt.expectedPipelineRun, pipelineRun, ignoreResourceVersion, ignoreTypeMeta, ignoreLastTransitionTime, ignoreStartTime, ignoreFinallyStartTime, cmpopts.EquateEmpty()); d != "" { + t.Errorf("expected PipelineRun was not created. Diff %s", diff.PrintWantGot(d)) + } + }) + } +} + func TestReconcile_SetDefaults(t *testing.T) { names.TestingSeed() diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index 100f161116b..4d8a383e1ba 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -176,8 +176,7 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy() pipelineTask.Params = pipelineTask.Params.ReplaceVariables(stringReplacements, arrayReplacements, objectReplacements) if pipelineTask.IsMatrixed() { - // Only string replacements from string, array or object results are supported - // We plan to support array replacements from array results soon (#5925) + // String replacements from string, array or object results or array replacements from array results are supported pipelineTask.Matrix.Params = pipelineTask.Matrix.Params.ReplaceVariables(stringReplacements, nil, nil) for i := range pipelineTask.Matrix.Include { // matrix include parameters can only be type string diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index e97e325ba5c..d15a96dad2d 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "regexp" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" @@ -747,6 +748,25 @@ func (t *ResolvedPipelineTask) hasResultReferences() bool { return false } +// containsArrayResultReferences returns true if the Matrix contains references to array +// results either through array indexing [0] or reference to the whole array[*] +func containsArrayResultReferences(m *v1beta1.Matrix) bool { + params := m.GetAllParams() + for _, param := range params { + expressions, ok := v1beta1.GetVarSubstitutionExpressionsForParam(param) + if ok && v1beta1.LooksLikeContainsResultRefs(expressions) { + for _, expression := range expressions { + var isNum = regexp.MustCompile(`^[0-9]+$`) + _, stringIdx := v1beta1.ParseResultName(expression) + if stringIdx == "*" || isNum.MatchString(stringIdx) { + return true + } + } + } + } + return false +} + func isCustomRunCancelledByPipelineRunTimeout(ro v1beta1.RunObject) bool { cr := ro.(*v1beta1.CustomRun) return cr.Spec.StatusMessage == v1beta1.CustomRunCancelledByPipelineTimeoutMsg diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go index 698b7644b43..1e91418f3a2 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go @@ -3555,6 +3555,16 @@ func TestResolvePipelineRunTask_WithMatrix(t *testing.T) { Name: "browsers", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, }}, + }, + }, { + Name: "pipelinetask-with-whole-array-results", + TaskRef: &v1beta1.TaskRef{ + Name: "my-task-with-results", + }, + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "foo", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, + }}, }}} rtr := &resources.ResolvedTask{ @@ -3592,6 +3602,15 @@ func TestResolvePipelineRunTask_WithMatrix(t *testing.T) { PipelineTask: &pts[1], ResolvedTask: rtr, }, + }, { + name: "task with matrix - whole array results", + pt: pts[2], + want: &ResolvedPipelineTask{ + TaskRunNames: taskRunsNames, + TaskRuns: nil, + PipelineTask: &pts[2], + ResolvedTask: rtr, + }, }} { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() @@ -3666,6 +3685,17 @@ func TestResolvePipelineRunTask_WithMatrixedCustomTask(t *testing.T) { Name: "browsers", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, }}}, + }, { + Name: "pipelinetask-custom-task-with-whole-array-results", + TaskRef: &v1beta1.TaskRef{ + APIVersion: "example.dev/v0", + Kind: "Example", + Name: "my-task", + }, + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "foo", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}}, + }}, }} getTask := func(ctx context.Context, name string) (*v1beta1.Task, *v1beta1.RefSource, error) { @@ -3709,6 +3739,15 @@ func TestResolvePipelineRunTask_WithMatrixedCustomTask(t *testing.T) { RunObjects: nil, PipelineTask: &pts[1], }, + }, { + name: "custom task with matrix - whole array results", + pt: pts[2], + want: &ResolvedPipelineTask{ + CustomTask: true, + RunObjectNames: nil, + RunObjects: nil, + PipelineTask: &pts[2], + }, }} { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() diff --git a/test/custom-task-ctrls/wait-task-beta/example_customrun_matrix_results.yaml b/test/custom-task-ctrls/wait-task-beta/example_customrun_matrix_results.yaml new file mode 100644 index 00000000000..0f43f79d3ab --- /dev/null +++ b/test/custom-task-ctrls/wait-task-beta/example_customrun_matrix_results.yaml @@ -0,0 +1,57 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: echo-duration + annotations: + description: | + A task that echos duration +spec: + params: + - name: duration + steps: + - name: echo + image: alpine + script: | + echo "$(params.duration)" +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: produce-duration +spec: + results: + - name: durations + type: array + steps: + - name: produce-a-list-of-durations + image: bash:latest + script: | + #!/usr/bin/env bash + echo -n "[\"10s\",\"2s\",\"5s\"]" | tee $(results.durations.path) +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + generateName: matrixed-pr- +spec: + pipelineSpec: + tasks: + - name: pt-with-result + taskRef: + name: produce-duration + - name: pt-matrix-echo-duration + matrix: + params: + - name: duration + value: $(tasks.pt-with-result.results.durations[*]) + taskRef: + name: echo-duration + - name: pt-matrix-custom-task + matrix: + params: + - name: duration + value: $(tasks.pt-with-result.results.durations[*]) + taskRef: + apiVersion: wait.testing.tekton.dev/v1beta1 + kind: Wait