From bace60965a8688444043573f9f1a2f54bba269d8 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 24 Apr 2023 09:41:55 +0200 Subject: [PATCH 1/4] Introduce new 'ImageNamesAsSelector' field in the Devfile parser args The goal of this field is to allow tools to provide information (currently registry and tag) allowing to replace matching image names. See [1] for more details. [1] https://github.com/devfile/api/issues/985 Signed-off-by: Armel Soro --- pkg/devfile/parser/parse.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index e22fa0eb..2be87cb9 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -25,17 +25,15 @@ import ( "net/url" "os" "path" + "reflect" "strings" "github.com/devfile/api/v2/pkg/attributes" - registryLibrary "github.com/devfile/registry-support/registry-library/library" - - "reflect" - devfileCtx "github.com/devfile/library/v2/pkg/devfile/parser/context" "github.com/devfile/library/v2/pkg/devfile/parser/data" "github.com/devfile/library/v2/pkg/devfile/parser/data/v2/common" "github.com/devfile/library/v2/pkg/util" + registryLibrary "github.com/devfile/registry-support/registry-library/library" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" @@ -166,6 +164,29 @@ type ParserArgs struct { // SetBooleanDefaults sets the boolean properties to their default values after a devfile been parsed. // The value is true by default. Clients can set this to false if they want to set the boolean properties themselves SetBooleanDefaults *bool + // ImageNamesAsSelector sets the information that will be used to handle image names as selectors when parsing the Devfile. + // Not setting this field or setting it to nil disables the logic of handling image names as selectors. + ImageNamesAsSelector *ImageSelectorArgs +} + +// ImageSelectorArgs defines the structure to leverage for using image names as selectors after parsing the Devfile. +// The fields defined here will be used together to compute the final image names that will be built and pushed, +// and replaced in all matching Image, Container or Kubernetes/OpenShift components. +// +// For Kubernetes/OpenShift components, replacement is done only in core Kubernetes resources +// (CronJob, DaemonSet, Deployment, Job, Pod, ReplicaSet, ReplicationController, StatefulSet) that are *inlined* in those components. +// Resources referenced via URIs will not be resolved. So you may want to also set ConvertKubernetesContentInUri to true in the parser args. +// +// For example, if Registry is set to "/" and Tag is set to "some-dynamic-unique-tag", +// all container and Kubernetes/OpenShift components matching a relative image name (say "my-image-name") of an Image component +// will be replaced in the resulting Devfile by: "//-my-image-name:some-dynamic-unique-tag". +type ImageSelectorArgs struct { + // Registry is the registry base path under which images matching selectors will be built and pushed to. + // Example: / + Registry string + // Tag represents a tag identifier under which images matching selectors will be built and pushed to. + // This should ideally be set to a unique identifier for each run of the caller tool. + Tag string } // ParseDevfile func populates the devfile data, parses and validates the devfile integrity. From 983e4e242c6bc72bcff7fc6c3337c7b9de8f2b40 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 24 Apr 2023 09:55:07 +0200 Subject: [PATCH 2/4] Add function performing replacements in matching Image, Container and Kubernetes components This relies on the Docker Distribution library to parse image references in order to detect if they are absolute or relative. See [1] for more details about the proposal. [1] https://github.com/devfile/api/issues/985#issuecomment-1404939472 Signed-off-by: Armel Soro --- go.mod | 1 + pkg/devfile/imageNameSelector.go | 304 +++++++++++ pkg/devfile/imageNameSelector_test.go | 719 ++++++++++++++++++++++++++ 3 files changed, 1024 insertions(+) create mode 100644 pkg/devfile/imageNameSelector.go create mode 100644 pkg/devfile/imageNameSelector_test.go diff --git a/go.mod b/go.mod index 4bf4f5e0..762e2141 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/devfile/api/v2 v2.2.1-alpha.0.20230413012049-a6c32fca0dbd github.com/devfile/registry-support/registry-library v0.0.0-20221018213054-47b3ffaeadba + github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 github.com/fatih/color v1.7.0 github.com/fsnotify/fsnotify v1.6.0 github.com/go-git/go-git/v5 v5.4.2 diff --git a/pkg/devfile/imageNameSelector.go b/pkg/devfile/imageNameSelector.go new file mode 100644 index 00000000..5dce1554 --- /dev/null +++ b/pkg/devfile/imageNameSelector.go @@ -0,0 +1,304 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devfile + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + + v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + "github.com/devfile/library/v2/pkg/devfile/parser" + "github.com/devfile/library/v2/pkg/devfile/parser/data/v2/common" + "github.com/distribution/distribution/v3/reference" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" +) + +var k8sSerializer = json.NewSerializerWithOptions( + json.DefaultMetaFactory, + scheme.Scheme, + scheme.Scheme, + json.SerializerOptions{ + Yaml: true, + Pretty: true, + }) + +// replaceImageNames parses all Image components in the specified Devfile object and, +// for each relative image name, replaces the value in all matching Image, Container and Kubernetes/Openshift components. +// +// An image is said to be relative if it has a canonical name different from its actual name. +// For example, image names like 'nodejs-devtools', 'nodejs-devtools:some-tag', 'nodejs-devtools@digest', or even 'some_name_different_from_localhost/nodejs-devtools' are all relative because +// their canonical form (as returned by the Distribution library) will be prefixed with 'docker.io/library/'. +// On the other hand, image names like 'docker.io/library/nodejs-devtools', 'localhost/nodejs-devtools@digest' or 'quay.io/nodejs-devtools:some-tag' are absolute. +// +// A component is said to be matching if the base name of the image used in this component is the same as the base name of the image component, regardless of its tag, digest or registry. +// For example, if the Devfile has an Image component with an image named 'nodejs-devtools' and 2 Container components using an image named 'nodejs-devtools:some-tag' and another absolute image named +// 'quay.io/nodejs-devtools@digest', both image names in the two Container components will be replaced by a value described below (because the base names of those images are 'nodejs-devtools', which +// match the base name of the relative image name of the Image Component). +// But `nodejs-devtools2` or 'ghcr.io/some-user/nodejs-devtools3' do not match the 'nodejs-devtools' image name and won't be replaced. +// +// For Kubernetes and OpenShift components, this function assumes that the actual resource manifests are inlined in the components, +// in order to perform any replacements for matching image names. +// At the moment, this function only supports replacements in Kubernetes native resource types (Pod, CronJob, Job, DaemonSet; Deployment, ReplicaSet, ReplicationController, StatefulSet). +// +// Absolute images and non-matching image references are left unchanged. +// +// And the replacement is done by using the following format: "/-:", +// where both and are set by the tool itself (either via auto-detection or via user input). +func replaceImageNames(d *parser.DevfileObj, registry string, imageTag string) (err error) { + var imageComponents []v1.Component + imageComponents, err = d.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.ImageComponentType}, + }) + if err != nil { + return err + } + + var isAbs bool + var imageRef reference.Named + for _, comp := range imageComponents { + imageName := comp.Image.ImageName + isAbs, imageRef, err = parseImageReference(imageName) + if err != nil { + return err + } + if isAbs { + continue + } + baseImageName := getImageSimpleName(imageRef) + + replacement := baseImageName + if d.GetMetadataName() != "" { + replacement = fmt.Sprintf("%s-%s", d.GetMetadataName(), replacement) + } + if registry != "" { + replacement = fmt.Sprintf("%s/%s", strings.TrimSuffix(registry, "/"), replacement) + } + if imageTag != "" { + replacement += fmt.Sprintf(":%s", imageTag) + } + + // Replace so that the image can be built and pushed to the registry specified by the tool. + comp.Image.ImageName = replacement + + // Replace in matching container components + err = handleContainerComponents(d, baseImageName, replacement) + if err != nil { + return err + } + + // Replace in matching Kubernetes and OpenShift components + err = handleKubernetesLikeComponents(d, baseImageName, replacement) + if err != nil { + return err + } + } + + return nil +} + +// parseImageReference uses the Docker reference library to detect if the image name is absolute or not +// and returns a struct from which we can extract the domain, tag and digest if needed. +func parseImageReference(imageName string) (isAbsolute bool, imageRef reference.Named, err error) { + imageRef, err = reference.ParseNormalizedNamed(imageName) + if err != nil { + return false, nil, err + } + + // Non-canonical image references are not absolute. + // For example, "nodejs-devtools" will be parsed as "docker.io/library/nodejs-devtools" + isAbsolute = imageRef.String() == imageName + + return isAbsolute, imageRef, nil +} + +func getImageSimpleName(img reference.Named) string { + p := reference.Path(img) + i := strings.LastIndex(p, "/") + result := p + if i >= 0 { + result = strings.TrimPrefix(p[i:], "/") + } + return result +} + +func hasMatch(baseImageName, compImage string) (bool, error) { + _, imageRef, err := parseImageReference(compImage) + if err != nil { + return false, err + } + return getImageSimpleName(imageRef) == baseImageName, nil +} + +func handleContainerComponents(d *parser.DevfileObj, baseImageName, replacement string) (err error) { + var containerComponents []v1.Component + containerComponents, err = d.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.ContainerComponentType}, + }) + if err != nil { + return err + } + + for _, comp := range containerComponents { + var match bool + match, err = hasMatch(baseImageName, comp.Container.Image) + if err != nil { + return err + } + if !match { + continue + } + comp.Container.Image = replacement + } + return nil +} + +func handleKubernetesLikeComponents(d *parser.DevfileObj, baseImageName, replacement string) error { + var allK8sOcComponents []v1.Component + + k8sComponents, err := d.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.KubernetesComponentType}, + }) + if err != nil { + return err + } + allK8sOcComponents = append(allK8sOcComponents, k8sComponents...) + + ocComponents, err := d.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.OpenshiftComponentType}, + }) + if err != nil { + return err + } + allK8sOcComponents = append(allK8sOcComponents, ocComponents...) + + updateImageInPodSpecIfNeeded := func(obj runtime.Object, ps *corev1.PodSpec) (string, error) { + handleContainer := func(c *corev1.Container) (match bool, err error) { + match, err = hasMatch(baseImageName, c.Image) + if err != nil { + return false, err + } + if !match { + return false, nil + } + c.Image = replacement + return true, nil + } + for i := range ps.Containers { + if _, err = handleContainer(&ps.Containers[i]); err != nil { + return "", err + } + } + for i := range ps.InitContainers { + if _, err = handleContainer(&ps.InitContainers[i]); err != nil { + return "", err + } + } + for i := range ps.EphemeralContainers { + if _, err = handleContainer((*corev1.Container)(&ps.EphemeralContainers[i].EphemeralContainerCommon)); err != nil { + return "", err + } + } + + //Encode obj back into a YAML string + var s strings.Builder + err = k8sSerializer.Encode(obj, &s) + if err != nil { + return "", err + } + + return s.String(), nil + } + + handleK8sContent := func(content string) (newContent string, err error) { + multidocReader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewBufferString(content))) + var yamlAsStringList []string + var buf []byte + var obj runtime.Object + for { + buf, err = multidocReader.Read() + if err != nil { + if err == io.EOF { + break + } + return "", err + } + + obj, _, err = k8sSerializer.Decode(buf, nil, nil) + if err != nil { + // Use raw string as it is, as it might be a Custom Resource with a Kind that is not known + // by the K8s decoder. + yamlAsStringList = append(yamlAsStringList, strings.TrimSpace(string(buf))) + continue + } + + newYaml := string(buf) + switch r := obj.(type) { + case *batchv1.CronJob: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.JobTemplate.Spec.Template.Spec) + case *appsv1.DaemonSet: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + case *appsv1.Deployment: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + case *batchv1.Job: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + case *corev1.Pod: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec) + case *appsv1.ReplicaSet: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + case *corev1.ReplicationController: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + case *appsv1.StatefulSet: + newYaml, err = updateImageInPodSpecIfNeeded(r, &r.Spec.Template.Spec) + } + + if err != nil { + return "", err + } + + yamlAsStringList = append(yamlAsStringList, strings.TrimSpace(newYaml)) + } + + return strings.Join(yamlAsStringList, "\n---\n"), nil + } + + var newContent string + for _, comp := range allK8sOcComponents { + if comp.Kubernetes != nil { + newContent, err = handleK8sContent(comp.Kubernetes.Inlined) + if err != nil { + return err + } + comp.Kubernetes.Inlined = newContent + } else { + newContent, err = handleK8sContent(comp.Openshift.Inlined) + if err != nil { + return err + } + comp.Openshift.Inlined = newContent + } + } + + return nil +} diff --git a/pkg/devfile/imageNameSelector_test.go b/pkg/devfile/imageNameSelector_test.go new file mode 100644 index 00000000..ca4dfec8 --- /dev/null +++ b/pkg/devfile/imageNameSelector_test.go @@ -0,0 +1,719 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package devfile + +import ( + "fmt" + "strings" + "testing" + + v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + "github.com/devfile/library/v2/pkg/devfile/parser" + "github.com/devfile/library/v2/pkg/devfile/parser/data" + "github.com/devfile/library/v2/pkg/devfile/parser/data/v2/common" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func Test_replaceImageNames(t *testing.T) { + const ( + devfileName = "my-component-app" + targetRegistry = "localhost:5000/my-org/my-user" + targetImageTag = "1.2.3-some-build-id" + ) + + const ( + absoluteImageNoTag = "quay.io/some-org/absolute-image-no-tag" + absoluteImageTag = "ghcr.io/some-user/absolute-image-tag:11.22.33-dev" + absoluteImageDigest = "docker.io/library/absolute-image-digest@sha256:26c68657ccce2cb0a31b330cb0be2b5e108d467f641c62e13ab40cbec258c68d" + + relativeImageNoTag = "relative-image-no-tag:tag" + relativeImageTag = "relative-image-tag:latest" + relativeImageTag2 = "org/user/relative-image-tag:2" + relativeImageDigest = "library/relative-image-digest@sha256:36c68657ccce2cb0a31b330cb0be2b5e108d467f641c62e13ab40cbec258c68d" + + notMatchingRelativeImageNoTag = "relative-image-no-tag2" + notMatchingRelativeImageTag = "relative-image-tag2:latest" + notMatchingRelativeImageTag2 = "relative-image-tag2:2" + notMatchingRelativeImageDigest = "relative-image-digest2@sha256:36c68657ccce2cb0a31b330cb0be2b5e108d467f641c62e13ab40cbec258c68d" + ) + + getImageNameReplacement := func(img string) string { + return fmt.Sprintf("%s/%s-%s:%s", targetRegistry, devfileName, img, targetImageTag) + } + + type args struct { + devfileObjProvider func() (*parser.DevfileObj, error) + } + tests := []struct { + name string + args args + wantErr bool + wantImageComponents []v1.Component + wantContainerComponents []v1.Component + wantK8sOcpComponents []v1.Component + }{ + { + name: "relative image names not matching any container or K8s/Openshift components", + args: args{ + devfileObjProvider: func() (*parser.DevfileObj, error) { + dData, err := data.NewDevfileData(string(data.APISchemaVersion220)) + if err != nil { + return nil, err + } + err = dData.AddComponents([]v1.Component{ + buildImageComponent("i-absolute-image-no-tag", absoluteImageNoTag), + buildImageComponent("i-absolute-image-tag", absoluteImageTag), + buildImageComponent("i-absolute-image-digest", absoluteImageDigest), + buildImageComponent("i-relative-image-no-tag", relativeImageNoTag), + buildImageComponent("i-relative-image-tag", relativeImageTag), + buildImageComponent("i-relative-image-tag-2", relativeImageTag2), + buildImageComponent("i-relative-image-digest", relativeImageDigest), + + buildContainerComponent("c-absolute-image-no-tag", absoluteImageNoTag), + buildContainerComponent("c-absolute-image-tag", absoluteImageTag), + buildContainerComponent("c-absolute-image-digest", absoluteImageDigest), + buildContainerComponent("c-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag), + buildContainerComponent("c-not-matching-relative-image-tag", notMatchingRelativeImageTag), + buildContainerComponent("c-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2), + buildContainerComponent("c-not-matching-relative-image-digest", notMatchingRelativeImageDigest), + + buildInlinedKubernetesComponent("k-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedKubernetesComponent("k-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedKubernetesComponent("k-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedKubernetesComponent("k-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedKubernetesComponent("k-not-matching-relative-image-tag", notMatchingRelativeImageTag, notMatchingRelativeImageTag), + buildInlinedKubernetesComponent("k-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2, notMatchingRelativeImageTag2), + buildInlinedKubernetesComponent("k-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + + buildInlinedOpenshiftComponent("o-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedOpenshiftComponent("o-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedOpenshiftComponent("o-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-tag", notMatchingRelativeImageTag, notMatchingRelativeImageTag), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2, notMatchingRelativeImageTag2), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + }) + if err != nil { + return nil, err + } + metadata := dData.GetMetadata() + metadata.Name = devfileName + dData.SetMetadata(metadata) + d := parser.DevfileObj{Data: dData} + if err != nil { + return nil, err + } + return &d, nil + }, + }, + wantImageComponents: []v1.Component{ + //Relative image names should still be replaced + buildImageComponent("i-absolute-image-no-tag", absoluteImageNoTag), + buildImageComponent("i-absolute-image-tag", absoluteImageTag), + buildImageComponent("i-absolute-image-digest", absoluteImageDigest), + + buildImageComponent("i-relative-image-no-tag", getImageNameReplacement("relative-image-no-tag")), + buildImageComponent("i-relative-image-tag", getImageNameReplacement("relative-image-tag")), + buildImageComponent("i-relative-image-tag-2", getImageNameReplacement("relative-image-tag")), + buildImageComponent("i-relative-image-digest", getImageNameReplacement("relative-image-digest")), + }, + wantContainerComponents: []v1.Component{ + // Should not change as none was matching + buildContainerComponent("c-absolute-image-no-tag", absoluteImageNoTag), + buildContainerComponent("c-absolute-image-tag", absoluteImageTag), + buildContainerComponent("c-absolute-image-digest", absoluteImageDigest), + buildContainerComponent("c-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag), + buildContainerComponent("c-not-matching-relative-image-tag", notMatchingRelativeImageTag), + buildContainerComponent("c-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2), + buildContainerComponent("c-not-matching-relative-image-digest", notMatchingRelativeImageDigest), + }, + wantK8sOcpComponents: []v1.Component{ + // Should not change as none was matching + buildInlinedKubernetesComponent("k-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedKubernetesComponent("k-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedKubernetesComponent("k-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedKubernetesComponent("k-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedKubernetesComponent("k-not-matching-relative-image-tag", notMatchingRelativeImageTag, notMatchingRelativeImageTag), + buildInlinedKubernetesComponent("k-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2, notMatchingRelativeImageTag2), + buildInlinedKubernetesComponent("k-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + + buildInlinedOpenshiftComponent("o-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedOpenshiftComponent("o-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedOpenshiftComponent("o-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-tag", notMatchingRelativeImageTag, notMatchingRelativeImageTag), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-tag-2", notMatchingRelativeImageTag2, notMatchingRelativeImageTag2), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + }, + }, + + { + name: "should update images for matching container or K8s/Openshift components", + args: args{ + devfileObjProvider: func() (*parser.DevfileObj, error) { + dData, err := data.NewDevfileData(string(data.APISchemaVersion220)) + if err != nil { + return nil, err + } + err = dData.AddComponents([]v1.Component{ + buildImageComponent("i-absolute-image-no-tag", absoluteImageNoTag), + buildImageComponent("i-absolute-image-tag", absoluteImageTag), + buildImageComponent("i-absolute-image-digest", absoluteImageDigest), + buildImageComponent("i-relative-image-no-tag", relativeImageNoTag), + buildImageComponent("i-relative-image-tag", relativeImageTag), + buildImageComponent("i-relative-image-tag-2", relativeImageTag2), + buildImageComponent("i-relative-image-digest", relativeImageDigest), + + buildContainerComponent("c-absolute-image-no-tag", absoluteImageNoTag), + buildContainerComponent("c-absolute-image-tag", absoluteImageTag), + buildContainerComponent("c-absolute-image-digest", absoluteImageDigest), + buildContainerComponent("c-matching-relative-image-no-tag", relativeImageNoTag), + buildContainerComponent("c-matching-relative-image-tag", relativeImageTag), + buildContainerComponent("c-matching-relative-image-tag-2", relativeImageTag2), + buildContainerComponent("c-matching-relative-image-digest", relativeImageDigest), + + buildInlinedKubernetesComponent("k-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedKubernetesComponent("k-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedKubernetesComponent("k-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedKubernetesComponent("k-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedKubernetesComponent("k-matching-relative-image-no-tag", relativeImageNoTag, relativeImageNoTag), + buildInlinedKubernetesComponent("k-matching-relative-image-tag", relativeImageTag, relativeImageTag), + buildInlinedKubernetesComponent("k-matching-relative-image-tag-2", relativeImageTag2, relativeImageTag2), + buildInlinedKubernetesComponent("k-matching-relative-image-digest", relativeImageDigest, relativeImageDigest), + + buildInlinedOpenshiftComponent("o-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedOpenshiftComponent("o-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedOpenshiftComponent("o-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedOpenshiftComponent("o-matching-relative-image-no-tag", relativeImageNoTag, relativeImageNoTag), + buildInlinedOpenshiftComponent("o-matching-relative-image-tag", relativeImageTag, relativeImageTag), + buildInlinedOpenshiftComponent("o-matching-relative-image-tag-2", relativeImageTag2, relativeImageTag2), + buildInlinedOpenshiftComponent("o-matching-relative-image-digest", relativeImageDigest, relativeImageDigest), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + }) + if err != nil { + return nil, err + } + metadata := dData.GetMetadata() + metadata.Name = devfileName + dData.SetMetadata(metadata) + d := parser.DevfileObj{Data: dData} + if err != nil { + return nil, err + } + return &d, nil + }, + }, + wantImageComponents: []v1.Component{ + buildImageComponent("i-absolute-image-no-tag", absoluteImageNoTag), + buildImageComponent("i-absolute-image-tag", absoluteImageTag), + buildImageComponent("i-absolute-image-digest", absoluteImageDigest), + + buildImageComponent("i-relative-image-no-tag", getImageNameReplacement("relative-image-no-tag")), + buildImageComponent("i-relative-image-tag", getImageNameReplacement("relative-image-tag")), + buildImageComponent("i-relative-image-tag-2", getImageNameReplacement("relative-image-tag")), + buildImageComponent("i-relative-image-digest", getImageNameReplacement("relative-image-digest")), + }, + wantContainerComponents: []v1.Component{ + buildContainerComponent("c-absolute-image-no-tag", absoluteImageNoTag), + buildContainerComponent("c-absolute-image-tag", absoluteImageTag), + buildContainerComponent("c-absolute-image-digest", absoluteImageDigest), + buildContainerComponent("c-matching-relative-image-no-tag", getImageNameReplacement("relative-image-no-tag")), + buildContainerComponent("c-matching-relative-image-tag", getImageNameReplacement("relative-image-tag")), + buildContainerComponent("c-matching-relative-image-tag-2", getImageNameReplacement("relative-image-tag")), + buildContainerComponent("c-matching-relative-image-digest", getImageNameReplacement("relative-image-digest")), + }, + wantK8sOcpComponents: []v1.Component{ + buildInlinedKubernetesComponent("k-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedKubernetesComponent("k-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedKubernetesComponent("k-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedKubernetesComponent("k-not-matching-relative-image-no-tag", notMatchingRelativeImageNoTag, notMatchingRelativeImageNoTag), + buildInlinedKubernetesComponent("k-matching-relative-image-no-tag", getImageNameReplacement("relative-image-no-tag"), relativeImageNoTag), + buildInlinedKubernetesComponent("k-matching-relative-image-tag", getImageNameReplacement("relative-image-tag"), relativeImageTag), + buildInlinedKubernetesComponent("k-matching-relative-image-tag-2", getImageNameReplacement("relative-image-tag"), relativeImageTag2), + buildInlinedKubernetesComponent("k-matching-relative-image-digest", getImageNameReplacement("relative-image-digest"), relativeImageDigest), + + buildInlinedOpenshiftComponent("o-absolute-image-no-tag", absoluteImageNoTag, absoluteImageNoTag), + buildInlinedOpenshiftComponent("o-absolute-image-tag", absoluteImageNoTag, absoluteImageTag), + buildInlinedOpenshiftComponent("o-absolute-image-digest", absoluteImageNoTag, absoluteImageDigest), + buildInlinedOpenshiftComponent("o-matching-relative-image-no-tag", getImageNameReplacement("relative-image-no-tag"), relativeImageNoTag), + buildInlinedOpenshiftComponent("o-matching-relative-image-tag", getImageNameReplacement("relative-image-tag"), relativeImageTag), + buildInlinedOpenshiftComponent("o-matching-relative-image-tag-2", getImageNameReplacement("relative-image-tag"), relativeImageTag2), + buildInlinedOpenshiftComponent("o-matching-relative-image-digest", getImageNameReplacement("relative-image-digest"), relativeImageDigest), + buildInlinedOpenshiftComponent("o-not-matching-relative-image-digest", notMatchingRelativeImageDigest, notMatchingRelativeImageDigest), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + devfileObj, err := tt.args.devfileObjProvider() + if err != nil { + t.Errorf("unexpected error while building DevfileObj: %v", err) + return + } + err = replaceImageNames(devfileObj, targetRegistry, targetImageTag) + if tt.wantErr != (err != nil) { + t.Errorf("replaceImageNames() unexpected error: %v, wantErr: %v", err, tt.wantErr) + } + + lessFn := cmpopts.SortSlices(func(c1, c2 v1.Component) bool { + return c1.Name < c2.Name + }) + + var imageComponents []v1.Component + imageComponents, err = devfileObj.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.ImageComponentType}, + }) + if err != nil { + t.Errorf("replaceImageNames() unexpected error while getting image components: %v", err) + return + } + if diff := cmp.Diff(tt.wantImageComponents, imageComponents, cmpopts.EquateEmpty(), lessFn); diff != "" { + t.Errorf("replaceImageNames() mismatch with image components (-want +got):\n%s", diff) + } + + var containerComponents []v1.Component + containerComponents, err = devfileObj.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.ContainerComponentType}, + }) + if err != nil { + t.Errorf("replaceImageNames() unexpected error while getting container components: %v", err) + return + } + if diff := cmp.Diff(tt.wantContainerComponents, containerComponents, cmpopts.EquateEmpty(), lessFn); diff != "" { + t.Errorf("replaceImageNames() mismatch with container components (-want +got):\n%s", diff) + } + + var allk8sOcComponents []v1.Component + var k8sComponents []v1.Component + k8sComponents, err = devfileObj.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.KubernetesComponentType}, + }) + if err != nil { + t.Errorf("replaceImageNames() unexpected error while getting Kubernetes components: %v", err) + return + } + allk8sOcComponents = append(allk8sOcComponents, k8sComponents...) + var ocComponents []v1.Component + ocComponents, err = devfileObj.Data.GetComponents(common.DevfileOptions{ + ComponentOptions: common.ComponentOptions{ComponentType: v1.OpenshiftComponentType}, + }) + if err != nil { + t.Errorf("replaceImageNames() unexpected error while getting Openshift components: %v", err) + return + } + allk8sOcComponents = append(allk8sOcComponents, ocComponents...) + if diff := cmp.Diff(tt.wantK8sOcpComponents, allk8sOcComponents, cmpopts.EquateEmpty(), lessFn); diff != "" { + t.Errorf("replaceImageNames() mismatch with Kubernetes and OpenShift components (-want +got):\n%s", diff) + } + }) + } +} + +func buildImageComponent(cmpName, imageName string) v1.Component { + return v1.Component{ + Name: cmpName, + ComponentUnion: v1.ComponentUnion{ + Image: &v1.ImageComponent{ + Image: v1.Image{ + ImageName: imageName, + }, + }, + }, + } +} + +func buildContainerComponent(cmpName, image string) v1.Component { + return v1.Component{ + Name: cmpName, + ComponentUnion: v1.ComponentUnion{ + Container: &v1.ContainerComponent{ + Container: v1.Container{ + Image: image, + }, + }, + }, + } +} + +func buildInlinedKubernetesComponent(cmpName, imageName, crImageName string) v1.Component { + return v1.Component{ + Name: cmpName, + ComponentUnion: v1.ComponentUnion{ + Kubernetes: &v1.KubernetesComponent{ + K8sLikeComponent: v1.K8sLikeComponent{ + K8sLikeComponentLocation: v1.K8sLikeComponentLocation{ + Inlined: buildInlinedK8sResource(imageName, crImageName), + }, + }, + }, + }, + } +} + +func buildInlinedOpenshiftComponent(cmpName, imageName, crImageName string) v1.Component { + return v1.Component{ + Name: cmpName, + ComponentUnion: v1.ComponentUnion{ + Openshift: &v1.OpenshiftComponent{ + K8sLikeComponent: v1.K8sLikeComponent{ + K8sLikeComponentLocation: v1.K8sLikeComponentLocation{ + Inlined: buildInlinedK8sResource(imageName, crImageName), + }, + }, + }, + }, + } +} + +func buildInlinedK8sResource(imageName, crImageName string) string { + return strings.Join([]string{ + buildCustomResource(), + buildCustomResourceWithImageName(crImageName), + buildPodManifest(imageName), + buildDaemonSetManifest(imageName), + buildDeploymentManifest(imageName), + buildJobManifest(imageName), + buildCronJobManifest(imageName), + buildReplicaSetManifest(imageName), + buildReplicationControllerManifest(imageName), + buildStatefulSetManifest(imageName), + }, "\n---\n") +} + +func buildPodManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: my-pod +spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-other-cont + resources: {} + ephemeralContainers: + - image: %[1]s + name: my-ephemeral-cont1 + resources: {} + - image: my-ephemeral-container-image + name: my-ephemeral-cont2 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} +status: {} +`, image)) +} + +func buildDaemonSetManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app: my-app + name: my-daemonset +spec: + selector: + matchLabels: + name: my-app + template: + metadata: + creationTimestamp: null + labels: + creationTimestamp: "" + name: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +`, image)) +} + +func buildDeploymentManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app: my-app + name: my-deployment +spec: + replicas: 10 + selector: + matchLabels: + app: my-app + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + app: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} +status: {} +`, image)) +} + +func buildJobManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: batch/v1 +kind: Job +metadata: + creationTimestamp: null + name: my-job +spec: + template: + metadata: + creationTimestamp: null + name: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} +status: {} +`, image)) +} + +func buildCronJobManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: batch/v1 +kind: CronJob +metadata: + creationTimestamp: null + name: my-cron-job +spec: + jobTemplate: + metadata: + creationTimestamp: null + spec: + template: + metadata: + creationTimestamp: null + name: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} + schedule: '*/1 * * * *' +status: {} +`, image)) +} + +func buildReplicaSetManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: apps/v1 +kind: ReplicaSet +metadata: + creationTimestamp: null + labels: + app: my-app + name: my-replicaset +spec: + replicas: 3 + selector: + matchLabels: + app: my-app + template: + metadata: + creationTimestamp: null + labels: + app: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} +status: + replicas: 0 +`, image)) +} + +func buildReplicationControllerManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: v1 +kind: ReplicationController +metadata: + creationTimestamp: null + name: my-replicationcontroller +spec: + replicas: 3 + selector: + app: my-app + template: + metadata: + creationTimestamp: null + labels: + app: my-app + name: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} +status: + replicas: 0 +`, image)) +} + +func buildStatefulSetManifest(image string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: apps/v1 +kind: StatefulSet +metadata: + creationTimestamp: null + name: my-statefulset +spec: + replicas: 3 + selector: + matchLabels: + app: my-app + serviceName: my-app + template: + metadata: + creationTimestamp: null + labels: + app: my-app + spec: + containers: + - image: %[1]s + name: my-main-cont1 + resources: {} + - image: my-other-image + name: my-main-cont1 + resources: {} + initContainers: + - image: %[1]s + name: my-init-cont1 + resources: {} + - image: my-init-container-image + name: my-init-cont2 + resources: {} + updateStrategy: {} +status: + availableReplicas: 0 + replicas: 0 +`, image)) +} + +func buildCustomResourceWithImageName(imageName string) string { + return strings.TrimSpace(fmt.Sprintf(` +apiVersion: "stable.example.com/v1" +kind: CronTab +metadata: + name: my-v1-crontab-cr +spec: + cronSpec: "* * * * */5" + image: %s + someRandomField: 42 +`, imageName)) +} + +func buildCustomResource() string { + return strings.TrimSpace(` +apiVersion: "stable.example.com/v2" +kind: NewCronTab +metadata: + name: my-v2-crontab-cr +spec: + at: "every 1 hour" + image: my-awesome-cron-image + randomField: 77 +`) +} From 2013a9db32d6b25d1a0043e6c0c25e1cd78144a1 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 24 Apr 2023 10:28:26 +0200 Subject: [PATCH 3/4] Perform image name replacement after variable substitution in the Devfile Users might actually be using variables for image names. Signed-off-by: Armel Soro --- pkg/devfile/parse.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/devfile/parse.go b/pkg/devfile/parse.go index 1a2b0aa3..ada87de2 100644 --- a/pkg/devfile/parse.go +++ b/pkg/devfile/parse.go @@ -109,6 +109,15 @@ func ParseDevfileAndValidate(args parser.ParserArgs) (d parser.DevfileObj, varWa varWarning = variables.ValidateAndReplaceGlobalVariable(d.Data.GetDevfileWorkspaceSpec()) } + // Use image names as selectors after variable substitution, + // as users might be using variables for image names. + if args.ImageNamesAsSelector != nil && args.ImageNamesAsSelector.Registry != "" { + err = replaceImageNames(&d, args.ImageNamesAsSelector.Registry, args.ImageNamesAsSelector.Tag) + if err != nil { + return d, varWarning, err + } + } + // generic validation on devfile content err = validate.ValidateDevfileData(d.Data) if err != nil { From d32bcd660fbfb4483ccf9c82763b020d9cd45023 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Wed, 10 May 2023 15:02:50 +0200 Subject: [PATCH 4/4] Make ParserArgs.ImageNamesAsSelector.Registry mandatory It does not make sense to set a non-nil ParserArgs.ImageNamesAsSelector with no Registry in it Signed-off-by: Armel Soro --- pkg/devfile/parser/parse.go | 7 ++++++- pkg/devfile/parser/parse_test.go | 30 ++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index 2be87cb9..6f25a3ab 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -181,7 +181,8 @@ type ParserArgs struct { // all container and Kubernetes/OpenShift components matching a relative image name (say "my-image-name") of an Image component // will be replaced in the resulting Devfile by: "//-my-image-name:some-dynamic-unique-tag". type ImageSelectorArgs struct { - // Registry is the registry base path under which images matching selectors will be built and pushed to. + // Registry is the registry base path under which images matching selectors will be built and pushed to. Required. + // // Example: / Registry string // Tag represents a tag identifier under which images matching selectors will be built and pushed to. @@ -192,6 +193,10 @@ type ImageSelectorArgs struct { // ParseDevfile func populates the devfile data, parses and validates the devfile integrity. // Creates devfile context and runtime objects func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { + if args.ImageNamesAsSelector != nil && strings.TrimSpace(args.ImageNamesAsSelector.Registry) == "" { + return DevfileObj{}, errors.New("registry is mandatory when setting ImageNamesAsSelector in the parser args") + } + if args.Data != nil { d.Ctx, err = devfileCtx.NewByteContentDevfileCtx(args.Data) if err != nil { diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index 0971f8d9..4820a648 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -19,8 +19,6 @@ import ( "bytes" "context" "fmt" - v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" - "github.com/devfile/library/v2/pkg/git" "io/ioutil" "net" "net/http" @@ -32,6 +30,9 @@ import ( "strings" "testing" + v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + "github.com/devfile/library/v2/pkg/git" + "github.com/devfile/api/v2/pkg/attributes" devfilepkg "github.com/devfile/api/v2/pkg/devfile" devfileCtx "github.com/devfile/library/v2/pkg/devfile/parser/context" @@ -3338,6 +3339,7 @@ commands: tests := []struct { name string parserArgs ParserArgs + wantErr bool wantBoolPropsSet []setFields }{ { @@ -3380,12 +3382,32 @@ commands: {compProp: compBooleanProp{prop: "Secure", name: "kubernetes-deploy", value: nil, compType: v1.KubernetesComponentType}}, //unset property should be nil }, }, + { + name: "error if ImageNamesAsSelector is non-nil but ImageNamesAsSelector.Registry is empty", + parserArgs: ParserArgs{ + Data: []byte(mainDevfile), + ConvertKubernetesContentInUri: &isFalse, //this is a workaround for a known parsing issue that requires support for downloading the deploy.yaml https://github.com/devfile/api/issues/1073 + ImageNamesAsSelector: &ImageSelectorArgs{}, + }, + wantErr: true, + }, + { + name: "error if ImageNamesAsSelector is non-nil but ImageNamesAsSelector.Registry is blank", + parserArgs: ParserArgs{ + Data: []byte(mainDevfile), + ConvertKubernetesContentInUri: &isFalse, //this is a workaround for a known parsing issue that requires support for downloading the deploy.yaml https://github.com/devfile/api/issues/1073 + ImageNamesAsSelector: &ImageSelectorArgs{ + Registry: " \t \n", + }, + }, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d, err := ParseDevfile(tt.parserArgs) - if err != nil { - t.Errorf("Problems parsing devfile") + if (err != nil) != tt.wantErr { + t.Errorf("unexpected err when parsing devfile: error=%v, wantErr =%v", err, tt.wantErr) } for i, _ := range tt.wantBoolPropsSet { wantProps := tt.wantBoolPropsSet[i]