Merge pull request opendatahub-io#311 from HumairAK/stable

Add V1.1.2 to stable.
red-hat-data-services · Aug 31, 2023 · 21d05a5 · 21d05a5
2 parents 5f0c332 + ee499fc
commit 21d05a5
Show file tree

Hide file tree

Showing 16 changed files with 132 additions and 47 deletions.
diff --git a/.github/workflows/image-check.yaml b/.github/workflows/image-check.yaml
@@ -0,0 +1,30 @@
+name: Image-check
+on:
+  push:
+    branches:
+      - '**'
+    tags-ignore:
+      - 'v*'
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.19.9'
+      - name: Check hard-coded image names in template files
+        run: |
+          lines=$(grep -r "image: " ./config/internal/ | egrep -v "{{[a-zA-Z.]+}}" | grep -v sample | wc -l)
+          if [ "$lines" -gt "0" ]; then exit 1; else exit 0; fi
+      - name: Check Operator resources
+        run: |
+          lines=$(grep -r "image: " ./config/manager/ | grep -oP -v "\\$\([[:upper:]_]+\)" | wc -l)
+          if [ "$lines" -gt "0" ]; then exit 1; else exit 0; fi
+      - name: Check non-sha image parameters
+        run: |
+          lines=$(egrep -v "IMAGES_[A-Z]+=[a-z0-9./-]+@sha256"  ./config/base/params.env | wc -l)
+          if [ "$lines" -gt "0" ]; then exit 1; else exit 0; fi
diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -215,9 +215,8 @@ type ExternalStorage struct {
 	Bucket              string `json:"bucket"`
 	Scheme              string `json:"scheme"`
 	*S3CredentialSecret `json:"s3CredentialsSecret"`
-	// +kubebuilder:default:=true
 	// +kubebuilder:validation:Optional
-	Secure bool `json:"secure"`
+	Secure *bool `json:"secure"`
 	// +kubebuilder:validation:Optional
 	Port string `json:"port"`
 }

diff --git a/config/base/params.env b/config/base/params.env
@@ -1,12 +1,12 @@
-IMAGES_APISERVER=quay.io/opendatahub/ds-pipelines-api-server@sha256:e310024243fa47dc30f89b6b53f633db79c69c2ae7202e483f3bc37bfd95d526
-IMAGES_ARTIFACT=quay.io/opendatahub/ds-pipelines-artifact-manager@sha256:bf13a1d4117e3fadc63ce03eaba3230f5286195ac88c3951696d53448808e36e
-IMAGES_PERSISTENTAGENT=quay.io/opendatahub/ds-pipelines-persistenceagent@sha256:5b98125e99bfa1ddb6197ddf2b520e0a8807b551a24ef96a0ccf2cb0257ab700
-IMAGES_SCHEDULEDWORKFLOW=quay.io/opendatahub/ds-pipelines-scheduledworkflow@sha256:31262aaf1abadfe0f707224a9b7b0b9b666539d10cd76804f79804b1aa2763cd
-IMAGES_CACHE=registry.access.redhat.com/ubi8/ubi-minimal@sha256:e52fc1de73dc2879516431ff1865e0fb61b1a32f57b6f914bdcddb13c62f84e6
-IMAGES_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro@sha256:443db9a646aaf9374f95d266ba0c8656a52d70d0ffcc386a782cea28fa32e55d
-IMAGES_MARIADB=registry.redhat.io/rhel8/mariadb-103@sha256:cafc7364494fb7206c373a1235fd5da74399c19b5c34d87dd02aa07e8f343fa2
-IMAGES_DSPO=quay.io/opendatahub/data-science-pipelines-operator@sha256:2a7a4afcd5deca392b297bc996b27c58158f2428491671d29213dab0ae9c2256
-IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:d0f2f1ef0bdc3aa1a70794ac8ac779271b634af83e939029ac5224ec0c815d7a
+IMAGES_APISERVER=quay.io/opendatahub/ds-pipelines-api-server@sha256:96d89e18106e914ef140874b903996dd4a2cbc731ad5e50d9446e60c4109dfcf
+IMAGES_ARTIFACT=quay.io/opendatahub/ds-pipelines-artifact-manager@sha256:32ddade7c930decb425f6667c23b585064eb4f4239cde407f2b596f7f739c9c3
+IMAGES_PERSISTENTAGENT=quay.io/opendatahub/ds-pipelines-persistenceagent@sha256:07cec6e87cbccb1ba96eae034662c92fb6ed915a897a93fa01698add19ab3308
+IMAGES_SCHEDULEDWORKFLOW=quay.io/opendatahub/ds-pipelines-scheduledworkflow@sha256:61c7f3d0b0b6275253c3b0fe7eb880c07aad3b6fa2f0747a38b86a0be2d4cc03
 IMAGES_MLMDENVOY=quay.io/opendatahub/ds-pipelines-metadata-envoy@sha256:851386f25bec1051a472e87eb98b3b8016f80e1d2e05a4f5d0c4323cb1c99563
 IMAGES_MLMDGRPC=quay.io/opendatahub/ds-pipelines-metadata-grpc@sha256:f2ff89ac664916789e690f8939b5fb0881e6662211a9c40712779236b862735d
-IMAGES_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer@sha256:aea738408d1a4334671b165ad54196836530e7602764be066924c2afd02abf41
+IMAGES_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer@sha256:5b98bf1c5bf33e6458d29beae1de69619c9e0a011576f1d8be5546ea43f572f2
+IMAGES_DSPO=quay.io/opendatahub/data-science-pipelines-operator@sha256:f87948f6f6c4f32c8ad4c922a111ed492f82f2ce56d7a27e864eaad01fb10bcc
+IMAGES_CACHE=registry.access.redhat.com/ubi8/ubi-minimal@sha256:7394c071ed74ace08cfd51f881c94067fa7a570e7f7e4a0ef0aff1b4f6a2a949
+IMAGES_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro@sha256:98f8ddc69b6210001351a5fd07993b3a758bc6af3702319493f7a5582dd65a9a
+IMAGES_MARIADB=registry.redhat.io/rhel8/mariadb-103@sha256:d0eea30ae4fc8c5bb06d0e4d61d92fba9c0ae40b8023f72702301b70a7537faa
+IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:d0f2f1ef0bdc3aa1a70794ac8ac779271b634af83e939029ac5224ec0c815d7a
diff --git a/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml
@@ -457,7 +457,6 @@ spec:
                       scheme:
                         type: string
                       secure:
-                        default: true
                         type: boolean
                     required:
                     - bucket

diff --git a/config/internal/apiserver/artifact_script.yaml.tmpl b/config/internal/apiserver/artifact_script.yaml.tmpl
@@ -9,10 +9,10 @@ data:
         if [ -f "$workspace_dest/$artifact_name" ]; then
             echo sending to: ${workspace_dest}/${artifact_name}
             tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint {{.ObjectStorageConnection.Endpoint}} cp $1.tgz s3://{{.ObjectStorageConnection.Bucket}}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         elif [ -f "$2" ]; then
             tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint {{.ObjectStorageConnection.Endpoint}} cp $1.tgz s3://{{.ObjectStorageConnection.Bucket}}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         else
             echo "$2 file does not exist. Skip artifact tracking for $1"
         fi

diff --git a/config/internal/common/mlmd-envoy-dashboard-access-policy.yaml.tmpl b/config/internal/common/mlmd-envoy-dashboard-access-policy.yaml.tmpl
@@ -0,0 +1,24 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: ds-pipelines-envoy-{{ .Name }}
+  namespace: {{ .Namespace }}
+spec:
+  podSelector:
+    matchLabels:
+      app: ds-pipeline-metadata-envoy-{{ .Name }}
+      component: data-science-pipelines
+  ingress:
+    - ports:
+        - protocol: TCP
+          port: 9090
+      from:
+        - podSelector:
+            matchLabels:
+              app: odh-dashboard
+          namespaceSelector: {}
+        - podSelector:
+            matchLabels:
+              component: data-science-pipelines
+  policyTypes:
+    - Ingress
diff --git a/config/internal/common/policy.yaml.tmpl b/config/internal/common/policy.yaml.tmpl
@@ -45,6 +45,18 @@ spec:
             matchLabels:
               app: ds-pipeline-scheduledworkflow-{{.Name}}
               component: data-science-pipelines
+        - podSelector:
+            matchLabels:
+              app: ds-pipeline-metadata-envoy-{{.Name}}
+              component: data-science-pipelines
+        - podSelector:
+            matchLabels:
+              app: ds-pipeline-metadata-grpc-{{.Name}}
+              component: data-science-pipelines
+        - podSelector:
+            matchLabels:
+              app: ds-pipeline-metadata-writer-{{.Name}}
+              component: data-science-pipelines
       ports:
         - protocol: TCP
           port: 8888

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -154,6 +154,16 @@ rules:
   - seldondeployments
   verbs:
   - '*'
+- apiGroups:
+  - mcad.ibm.com
+  resources:
+  - appwrappers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
 - apiGroups:
   - monitoring.coreos.com
   resources:
@@ -178,6 +188,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ray.io
+  resources:
+  - rayclusters
+  - rayjobs
+  - rayservices
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
 - apiGroups:
   - rbac.authorization.k8s.io
   resources:
@@ -229,25 +251,3 @@ rules:
   - '*'
   verbs:
   - '*'
-- apiGroups:
-    - mcad.ibm.com
-  resources:
-    - appwrappers
-  verbs:
-    - create
-    - get
-    - list
-    - patch
-    - delete
-- apiGroups:
-    - ray.io
-  resources:
-    - rayclusters
-    - rayjobs
-    - rayservices
-  verbs:
-    - create
-    - get
-    - list
-    - patch
-    - delete
diff --git a/controllers/common.go b/controllers/common.go
@@ -21,6 +21,7 @@ import (
 
 var commonTemplates = []string{
 	"common/policy.yaml.tmpl",
+	"common/mlmd-envoy-dashboard-access-policy.yaml.tmpl",
 }
 
 const commonCusterRolebindingTemplate = "common/clusterrolebinding.yaml.tmpl"

diff --git a/controllers/dspipeline_controller.go b/controllers/dspipeline_controller.go
@@ -142,6 +142,8 @@ func (r *DSPAReconciler) buildCondition(conditionType string, dspa *dspav1alpha1
 //+kubebuilder:rbac:groups=machinelearning.seldon.io,resources=seldondeployments,verbs=*
 //+kubebuilder:rbac:groups=tekton.dev,resources=*,verbs=*
 //+kubebuilder:rbac:groups=custom.tekton.dev,resources=pipelineloops,verbs=*
+//+kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers,verbs=create;get;list;patch;delete
+//+kubebuilder:rbac:groups=ray.io,resources=rayclusters;rayjobs;rayservices,verbs=create;get;list;patch;delete
 //+kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create
 //+kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create
 //+kubebuilder:rbac:groups=image.openshift.io,resources=imagestreamtags,verbs=get

diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -27,6 +27,7 @@ import (
 	mf "github.com/manifestival/manifestival"
 	dspa "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1"
 	"github.com/opendatahub-io/data-science-pipelines-operator/controllers/config"
+	"github.com/opendatahub-io/data-science-pipelines-operator/controllers/util"
 	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -63,10 +64,10 @@ type DBConnection struct {
 type ObjectStorageConnection struct {
 	Bucket            string
 	CredentialsSecret *dspa.S3CredentialSecret
-	Secure            bool
 	Host              string
 	Port              string
 	Scheme            string
+	Secure            *bool
 	Endpoint          string // scheme://host:port
 	AccessKeyID       string
 	SecretAccessKey   string
@@ -235,7 +236,17 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 		p.ObjectStorageConnection.Bucket = dsp.Spec.ObjectStorage.ExternalStorage.Bucket
 		p.ObjectStorageConnection.Host = dsp.Spec.ObjectStorage.ExternalStorage.Host
 		p.ObjectStorageConnection.Scheme = dsp.Spec.ObjectStorage.ExternalStorage.Scheme
-		p.ObjectStorageConnection.Secure = dsp.Spec.ObjectStorage.ExternalStorage.Secure
+
+		if dsp.Spec.ObjectStorage.ExternalStorage.Secure == nil {
+			if p.ObjectStorageConnection.Scheme == "https" {
+				p.ObjectStorageConnection.Secure = util.BoolPointer(true)
+			} else {
+				p.ObjectStorageConnection.Secure = util.BoolPointer(false)
+			}
+		} else {
+			p.ObjectStorageConnection.Secure = dsp.Spec.ObjectStorage.ExternalStorage.Secure
+		}
+
 		// Port can be empty, which is fine.
 		p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port
 		customCreds = dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret
@@ -265,6 +276,8 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 		)
 		p.ObjectStorageConnection.Port = config.MinioPort
 		p.ObjectStorageConnection.Scheme = config.MinioScheme
+		p.ObjectStorageConnection.Secure = util.BoolPointer(false)
+
 		if p.Minio.S3CredentialSecret != nil {
 			customCreds = p.Minio.S3CredentialSecret
 		}
@@ -343,6 +356,7 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc
 	}
 
 	return nil
+
 }
 
 func (p *DSPAParams) SetupMLMD(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error {

diff --git a/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml
@@ -9,10 +9,10 @@ data:
         if [ -f "$workspace_dest/$artifact_name" ]; then
             echo sending to: ${workspace_dest}/${artifact_name}
             tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp0.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         elif [ -f "$2" ]; then
             tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp0.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         else
             echo "$2 file does not exist. Skip artifact tracking for $1"
         fi

diff --git a/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml
@@ -9,10 +9,10 @@ data:
         if [ -f "$workspace_dest/$artifact_name" ]; then
             echo sending to: ${workspace_dest}/${artifact_name}
             tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         elif [ -f "$2" ]; then
             tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         else
             echo "$2 file does not exist. Skip artifact tracking for $1"
         fi

diff --git a/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml
@@ -9,10 +9,10 @@ data:
         if [ -f "$workspace_dest/$artifact_name" ]; then
             echo sending to: ${workspace_dest}/${artifact_name}
             tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp4.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         elif [ -f "$2" ]; then
             tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp4.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         else
             echo "$2 file does not exist. Skip artifact tracking for $1"
         fi

diff --git a/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml
@@ -9,10 +9,10 @@ data:
         if [ -f "$workspace_dest/$artifact_name" ]; then
             echo sending to: ${workspace_dest}/${artifact_name}
             tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp5.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         elif [ -f "$2" ]; then
             tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
-            aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+            aws s3 --endpoint http://minio-testdsp5.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
         else
             echo "$2 file does not exist. Skip artifact tracking for $1"
         fi

diff --git a/controllers/util/util.go b/controllers/util/util.go
@@ -40,3 +40,7 @@ func GetDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.Depl
 	}
 	return nil
 }
+
+func BoolPointer(b bool) *bool {
+	return &b
+}