[Frontend] minio client in API server can be configured with environm…

…ent variables (#1324)

* kfp frontend API service can configure minio client params thru env vars

* minio endpoint is composed from host and namespace to support k8s yaml

* Added kustomize patch for pipeline-ui deploy

frontend/server/server.ts

@@ -29,20 +29,44 @@ import { Stream } from 'stream';
 
 const BASEPATH = '/pipeline';
 
-// The minio endpoint, port, access and secret keys are hardcoded to the same
-// values used in the deployment.
+/** All configurable environment variables can be found here. */
+const {
+  /** minio client use these to retrieve minio objects/artifacts */
+  MINIO_ACCESS_KEY = 'minio',
+  MINIO_SECRET_KEY = 'minio123',
+  MINIO_PORT = '9000',
+  MINIO_HOST = 'minio-service',
+  MINIO_NAMESPACE = 'kubeflow',
+  MINIO_SSL = 'false',
+  /** minio client use these to retrieve s3 objects/artifacts */
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  /** API service will listen to this host */
+  ML_PIPELINE_SERVICE_HOST = 'localhost',
+  /** API service will listen to this port */
+  ML_PIPELINE_SERVICE_PORT = '3001'
+} = process.env
+
+/** construct minio endpoint from host and namespace (optional) */
+const MINIO_ENDPOINT = MINIO_NAMESPACE && MINIO_NAMESPACE.length > 0 ? `${MINIO_HOST}.${MINIO_NAMESPACE}` : MINIO_HOST
+
+/** converts string to bool */
+const _as_bool = (value: string) => ['true', '1'].indexOf(value.toLowerCase()) >= 0
+
+/** minio client for minio storage */
 const minioClient = new MinioClient({
-  accessKey: 'minio',
-  endPoint: 'minio-service.kubeflow',
-  port: 9000,
-  secretKey: 'minio123',
-  useSSL: false,
+  accessKey: MINIO_ACCESS_KEY,
+  endPoint: MINIO_ENDPOINT,
+  port: parseInt(MINIO_PORT, 10),
+  secretKey: MINIO_SECRET_KEY,
+  useSSL: _as_bool(MINIO_SSL),
 } as any);
 
+/** minio client for s3 objects */
 const s3Client = new MinioClient({
   endPoint: 's3.amazonaws.com',
-  accessKey: process.env.AWS_ACCESS_KEY_ID,
-  secretKey: process.env.AWS_SECRET_ACCESS_KEY,
+  accessKey: AWS_ACCESS_KEY_ID,
+  secretKey: AWS_SECRET_ACCESS_KEY,
 } as any);
 
 const app = express() as Application;
@@ -71,9 +95,7 @@ const buildDate =
 const commitHash =
   fs.existsSync(commitHashPath) ? fs.readFileSync(commitHashPath, 'utf-8').trim() : '';
 const port = process.argv[3] || 3000;
-const apiServerHost = process.env.ML_PIPELINE_SERVICE_HOST || 'localhost';
-const apiServerPort = process.env.ML_PIPELINE_SERVICE_PORT || '3001';
-const apiServerAddress = `http://${apiServerHost}:${apiServerPort}`;
+const apiServerAddress = `http://${ML_PIPELINE_SERVICE_HOST}:${ML_PIPELINE_SERVICE_PORT}`;
 
 const v1beta1Prefix = 'apis/v1beta1';
 

manifests/namespaced-install.yaml

@@ -607,7 +607,7 @@ spec:
       - env:
         - name: NAMESPACE
           value: kubeflow
-        image: gcr.io/ml-pipeline-staging/persistenceagent:22be84b13c9d3c94d803d78685f00b0b5607258f
+        image: gcr.io/ml-pipeline/persistenceagent:0.1.20
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-persistenceagent
       serviceAccountName: ml-pipeline-persistenceagent
@@ -632,7 +632,7 @@ spec:
       - env:
         - name: NAMESPACE
           value: kubeflow
-        image: gcr.io/ml-pipeline-staging/scheduledworkflow:22be84b13c9d3c94d803d78685f00b0b5607258f
+        image: gcr.io/ml-pipeline/scheduledworkflow:0.1.20
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-scheduledworkflow
       serviceAccountName: ml-pipeline-scheduledworkflow
@@ -654,7 +654,10 @@ spec:
         app: ml-pipeline-ui
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/frontend:0.1.15
+      - env:
+        - name: MINIO_NAMESPACE
+          value: kubeflow
+        image: gcr.io/ml-pipeline/frontend:0.1.20
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-ui
         ports:
@@ -678,7 +681,7 @@ spec:
         app: ml-pipeline-viewer-crd
     spec:
       containers:
-      - image: gcr.io/ml-pipeline/viewer-crd-controller:0.1.15
+      - image: gcr.io/ml-pipeline/viewer-crd-controller:0.1.20
         imagePullPolicy: Always
         name: ml-pipeline-viewer-crd
       serviceAccountName: ml-pipeline-viewer-crd-service-account
@@ -705,7 +708,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: gcr.io/ml-pipeline/api-server:0.1.15
+        image: gcr.io/ml-pipeline/api-server:0.1.20
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-api-server
         ports:

manifests/namespaced-install/kustomization.yaml

@@ -13,6 +13,7 @@ patchesStrategicMerge:
 - workflow-controller-configmap.yaml
 - ml-pipeline-persistenceagent-deployment-patch.yaml
 - ml-pipeline-scheduledworkflow-deployment-patch.yaml
+- ml-pipeline-ui-deployment-patch.yaml
 
 vars:
   - name: NAMESPACE

manifests/namespaced-install/ml-pipeline-ui-deployment-patch.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline-ui
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-ui
+          env:
+          - name: MINIO_NAMESPACE
+            value: $(NAMESPACE)