From 0555c1156c41c1af39c32ca36493f937b8707e9c Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 18:22:14 +0800
Subject: [PATCH 01/12] try generate MKP binary for each submit

---
 .cloudbuild.yaml | 23 +++++++++++++++++++++++
 VERSION          |  1 +
 2 files changed, 24 insertions(+)
 create mode 100644 VERSION

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index 0f4403ddae3..05302df7a67 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -182,6 +182,29 @@ steps:
          '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
   id:   'buildMetadataEnvoy'
 
+# Tag for Hosted - SemVersion to Major.Minor parsing
+- id: "parseMajorMinorVersion"
+name: gcr.io/cloud-builders/docker
+entrypoint: /bin/bash
+args:
+  - -ceux
+  - |
+    # Parse major minor version and save to a file for reusing in other steps.
+    cat /workspace/VERSION | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver
+
+# Tag for Hosted - Tag to hosted folder with MKP friendly name
+- id:   'tagForHosted'
+waitFor: [parseMajorMinorVersion', 'buildFrontend']
+name: 'gcr.io/cloud-builders/docker'
+entrypoint: bash
+args:
+- -ceux
+- |
+  docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/VERSION)
+  docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/mm.ver)
+  docker push gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/VERSION)
+  docker push gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/mm.ver)
+
 images:
 # Images for the pipeline system itself
 - 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
diff --git a/VERSION b/VERSION
new file mode 100644
index 00000000000..28af839c0a9
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+0.2.5
\ No newline at end of file

From 94b1f92775a69de84f2cb83f903a2ec191cc375d Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 18:50:29 +0800
Subject: [PATCH 02/12] try run

---
 .cloudbuild.yaml          |  9 ++++-----
 release/tag_for_hosted.sh | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 5 deletions(-)
 create mode 100755 release/tag_for_hosted.sh

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index 05302df7a67..b237652b34c 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -194,16 +194,15 @@ args:
 
 # Tag for Hosted - Tag to hosted folder with MKP friendly name
 - id:   'tagForHosted'
-waitFor: [parseMajorMinorVersion', 'buildFrontend']
+waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildViewerCrdController',
+          'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+          'buildMarketplaceDeployer']
 name: 'gcr.io/cloud-builders/docker'
 entrypoint: bash
 args:
 - -ceux
 - |
-  docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/VERSION)
-  docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/mm.ver)
-  docker push gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/VERSION)
-  docker push gcr.io/$PROJECT_ID/hosted/frontend:$(cat /workspace/mm.ver)
+  /workspace/release/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
 
 images:
 # Images for the pipeline system itself
diff --git a/release/tag_for_hosted.sh b/release/tag_for_hosted.sh
new file mode 100755
index 00000000000..fa8122dee44
--- /dev/null
+++ b/release/tag_for_hosted.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT_ID=$1
+COMMIT_SHA=$2
+SEM_VER=$3
+MM_VER=$4
+
+docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/frontend:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/frontend:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/apiserver:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/apiserver:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/apiserver:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/apiserver:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
\ No newline at end of file

From 119f6761fa1f0e56fd710d663363f636c1c58359 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 18:54:01 +0800
Subject: [PATCH 03/12] fix format

---
 .cloudbuild.yaml | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index b237652b34c..95cde63ba87 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -184,25 +184,25 @@ steps:
 
 # Tag for Hosted - SemVersion to Major.Minor parsing
 - id: "parseMajorMinorVersion"
-name: gcr.io/cloud-builders/docker
-entrypoint: /bin/bash
-args:
-  - -ceux
-  - |
-    # Parse major minor version and save to a file for reusing in other steps.
-    cat /workspace/VERSION | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver
+  name: gcr.io/cloud-builders/docker
+  entrypoint: /bin/bash
+  args:
+    - -ceux
+    - |
+      # Parse major minor version and save to a file for reusing in other steps.
+      cat /workspace/VERSION | sed -e "s#[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)#\1.\2#" > /workspace/mm.ver
 
 # Tag for Hosted - Tag to hosted folder with MKP friendly name
 - id:   'tagForHosted'
-waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildViewerCrdController',
-          'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
-          'buildMarketplaceDeployer']
-name: 'gcr.io/cloud-builders/docker'
-entrypoint: bash
-args:
-- -ceux
-- |
-  /workspace/release/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
+  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildViewerCrdController',
+            'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+            'buildMarketplaceDeployer']
+  name: 'gcr.io/cloud-builders/docker'
+  entrypoint: bash
+  args:
+  - -ceux
+  - |
+    /workspace/release/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
 
 images:
 # Images for the pipeline system itself

From 2c558994a846c94c1bed79b0d0170dbc541332ad Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 18:54:48 +0800
Subject: [PATCH 04/12] fix format

---
 release/tag_for_hosted.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release/tag_for_hosted.sh b/release/tag_for_hosted.sh
index fa8122dee44..f6a9a397c83 100755
--- a/release/tag_for_hosted.sh
+++ b/release/tag_for_hosted.sh
@@ -32,4 +32,5 @@ docker push gcr.io/$PROJECT_ID/hosted/apiserver:$MM_VER
 docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
 docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
 docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
-docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
\ No newline at end of file
+docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
+

From 53c2394bf46484b9902a9caf2975216fd16fe6f0 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 18:56:18 +0800
Subject: [PATCH 05/12] fix format

---
 .cloudbuild.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index 95cde63ba87..bbecb56ab83 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -184,6 +184,7 @@ steps:
 
 # Tag for Hosted - SemVersion to Major.Minor parsing
 - id: "parseMajorMinorVersion"
+  waitFor: ["-"]
   name: gcr.io/cloud-builders/docker
   entrypoint: /bin/bash
   args:

From 99ac282882b0baeb48ddfb97e2ee9eeff8e9cd79 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Thu, 27 Feb 2020 23:19:33 +0800
Subject: [PATCH 06/12] it works, gcloud builds submit --config
 test/cloudbuild/mkp_verify.yaml --project ml-pipeline-test

---
 test/cloudbuild/mkp_verify.yaml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 test/cloudbuild/mkp_verify.yaml

diff --git a/test/cloudbuild/mkp_verify.yaml b/test/cloudbuild/mkp_verify.yaml
new file mode 100644
index 00000000000..cd614c8c846
--- /dev/null
+++ b/test/cloudbuild/mkp_verify.yaml
@@ -0,0 +1,20 @@
+steps:
+- id:   'pullDeployer'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/google/pipelines-test/deployer:0.2']  
+- id: "verify"
+  waitFor: ['pullDeployer']
+  name: "gcr.io/cloud-marketplace-tools/k8s/dev:latest"
+  entrypoint: '/bin/bash'
+  args:
+    - -ceux
+    - |
+      gcloud container clusters get-credentials hosted-verify-test --zone us-central1-c --project ml-pipeline-test
+      kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
+      /scripts/verify --deployer="gcr.io/ml-pipeline/google/pipelines-test/deployer:0.2" --wait_timeout="1800"
+     
+timeout: '3600s'
+options:
+  machineType: N1_HIGHCPU_8
+
+# kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user <GSA>
\ No newline at end of file

From 06de8f1aa33b8cf40fa90a9dc9ce0ef072811162 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Fri, 28 Feb 2020 22:21:20 +0800
Subject: [PATCH 07/12] test commit trigger

---
 test/cloudbuild/mkp_verify.yaml | 10 ++++---
 test/presubmit-tests-mkp.sh     | 47 +++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 3 deletions(-)
 create mode 100644 test/presubmit-tests-mkp.sh

diff --git a/test/cloudbuild/mkp_verify.yaml b/test/cloudbuild/mkp_verify.yaml
index cd614c8c846..3d61860ea4e 100644
--- a/test/cloudbuild/mkp_verify.yaml
+++ b/test/cloudbuild/mkp_verify.yaml
@@ -1,7 +1,7 @@
 steps:
 - id:   'pullDeployer'
   name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/google/pipelines-test/deployer:0.2']  
+  args: ['pull', 'gcr.io/ml-pipeline/google/pipelines-test/deployer:${_MM_VERSION}']  
 - id: "verify"
   waitFor: ['pullDeployer']
   name: "gcr.io/cloud-marketplace-tools/k8s/dev:latest"
@@ -11,10 +11,14 @@ steps:
     - |
       gcloud container clusters get-credentials hosted-verify-test --zone us-central1-c --project ml-pipeline-test
       kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
-      /scripts/verify --deployer="gcr.io/ml-pipeline/google/pipelines-test/deployer:0.2" --wait_timeout="1800"
+      /scripts/verify --deployer="gcr.io/ml-pipeline/google/pipelines-test/deployer:${_MM_VERSION}" --wait_timeout="1800"
      
 timeout: '3600s'
 options:
   machineType: N1_HIGHCPU_8
 
-# kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user <GSA>
\ No newline at end of file
+substitutions:
+  _MM_VERSION: ''
+
+# kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user <GSA>
+# 
\ No newline at end of file
diff --git a/test/presubmit-tests-mkp.sh b/test/presubmit-tests-mkp.sh
new file mode 100644
index 00000000000..d5f0733d34d
--- /dev/null
+++ b/test/presubmit-tests-mkp.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# This file is forked from presubmit-test-with-pipoeline-deployment.sh
+# as the test can't be run inside Argo.
+
+# This is merged commit's SHA.
+COMMIT_SHA="$(git rev-parse HEAD)"
+GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/presubmit/${COMMIT_SHA}
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+# Configure `time` command output format.
+TIMEFORMAT="[test-timing] It took %lR."
+
+echo "presubmit test starts"
+if [ -n "$PULL_PULL_SHA" ]; then
+    echo "PR commit is ${PULL_PULL_SHA}"
+fi
+time source "${DIR}/test-prep.sh"
+echo "test env prepared"
+
+# We don't wait for image building here, because cluster can be deployed in
+# parallel so that we save a few minutes of test time.
+time source "${DIR}/build-images.sh"
+echo "KFP images cloudbuild jobs submitted"
+
+time source "${DIR}/deploy-cluster.sh"
+echo "cluster deployed"
+
+time source "${DIR}/check-build-image-status.sh"
+echo "KFP images built"
+

From 6183ee4b4aa9c47ae6f19c90d7f8f1f47ce1fa71 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Fri, 28 Feb 2020 23:16:55 +0800
Subject: [PATCH 08/12] backup codes

---
 test/cloudbuild/mkp_verify.yaml |  1 +
 test/presubmit-tests-mkp.sh     | 65 +++++++++++++++++++++++++++------
 2 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/test/cloudbuild/mkp_verify.yaml b/test/cloudbuild/mkp_verify.yaml
index 3d61860ea4e..98a359b3011 100644
--- a/test/cloudbuild/mkp_verify.yaml
+++ b/test/cloudbuild/mkp_verify.yaml
@@ -20,5 +20,6 @@ options:
 substitutions:
   _MM_VERSION: ''
 
+# TODO: create new cluster and handover name here. use existing cluster will hit image cache
 # kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user <GSA>
 # 
\ No newline at end of file
diff --git a/test/presubmit-tests-mkp.sh b/test/presubmit-tests-mkp.sh
index d5f0733d34d..8ce3ce36c5e 100644
--- a/test/presubmit-tests-mkp.sh
+++ b/test/presubmit-tests-mkp.sh
@@ -16,9 +16,6 @@
 
 set -ex
 
-# This file is forked from presubmit-test-with-pipoeline-deployment.sh
-# as the test can't be run inside Argo.
-
 # This is merged commit's SHA.
 COMMIT_SHA="$(git rev-parse HEAD)"
 GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/presubmit/${COMMIT_SHA}
@@ -34,14 +31,60 @@ fi
 time source "${DIR}/test-prep.sh"
 echo "test env prepared"
 
-# We don't wait for image building here, because cluster can be deployed in
-# parallel so that we save a few minutes of test time.
-time source "${DIR}/build-images.sh"
-echo "KFP images cloudbuild jobs submitted"
+# TODO: extract this part to be a dedicated file for reusage.
+# =========================================================================
+## Wait for the cloudbuild job to be started (triggerd by PR commit)
+CLOUDBUILD_TIMEOUT_SECONDS=3600
+PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT=$(expr ${CLOUDBUILD_TIMEOUT_SECONDS} / 20 )
+CLOUDBUILD_STARTED=False
+
+for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
+do
+  output=`gcloud builds list --project="$CLOUDBUILD_PROJECT" --filter="sourceProvenance.resolvedRepoSource.commitSha:${COMMIT_SHA}"`
+  if [[ ${output} != "" ]]; then
+    CLOUDBUILD_STARTED=True
+    break
+  fi
+  sleep 20
+done
+
+if [[ ${CLOUDBUILD_STARTED} == False ]];then
+  echo "CloudBuild job for the commit not be triggered before timeout, exiting..."
+  exit 1
+fi
+
+## Wait for the cloudbuild job to complete, if success, all images ready
+CLOUDBUILD_FINISHED=TIMEOUT
+for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
+do
+  output=`gcloud builds list --project="$CLOUDBUILD_PROJECT" --filter="sourceProvenance.resolvedRepoSource.commitSha:${COMMIT_SHA}"`
+  if [[ ${output} == *"SUCCESS"* ]]; then
+    CLOUDBUILD_FINISHED=SUCCESS
+    break
+  elif [[ ${output} == *"FAILURE"* ]]; then
+    CLOUDBUILD_FINISHED=FAILURE
+    break
+  fi
+  sleep 20
+done
+
+if [[ ${CLOUDBUILD_FINISHED} == FAILURE ]];then
+  echo "Cloud build failure, cannot proceed. exiting..."
+  exit 1
+elif [[ ${CLOUDBUILD_FINISHED} == TIMEOUT ]];then
+  echo "Cloud build timeout, cannot proceed. exiting..."
+  exit 1
+fi
+
+# =========================================================================
+## Create Cluster (delete in clean up)
 
-time source "${DIR}/deploy-cluster.sh"
-echo "cluster deployed"
+# =========================================================================
+## All images ready, trigger the MKP verification. Here we can't use Argo and have to use CloudBuild again
+## Run CB job in sync mode to easily get test result.
 
-time source "${DIR}/check-build-image-status.sh"
-echo "KFP images built"
+## TODO
+## 
+# gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="0.2" --project=ml-pipeline-test
 
+## ??? how test cluster get deleted?
\ No newline at end of file

From d70f8a614b8ba1331f351846dd45e5ce4ae9f977 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Tue, 10 Mar 2020 17:09:54 +0800
Subject: [PATCH 09/12] test

---
 .cloudbuild.yaml                              |   7 +-
 release/tag_for_hosted.sh                     |  36 -----
 test/cloudbuild/mkp_verify.yaml               |   8 +-
 test/deploy-pipeline-mkp-cli.sh               |  91 +------------
 ...stsubmit-tests-with-pipeline-deployment.sh |  36 ++---
 test/presubmit-tests-mkp.sh                   |  77 +----------
 test/tag_for_hosted.sh                        | 123 ++++++++++++++++++
 7 files changed, 155 insertions(+), 223 deletions(-)
 delete mode 100755 release/tag_for_hosted.sh
 create mode 100755 test/tag_for_hosted.sh

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index 4a692689009..69614bce20c 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -207,15 +207,16 @@ steps:
 
 # Tag for Hosted - Tag to hosted folder with MKP friendly name
 - id:   'tagForHosted'
-  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildViewerCrdController',
-            'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',            
+            'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+            'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
             'buildMarketplaceDeployer']
   name: 'gcr.io/cloud-builders/docker'
   entrypoint: bash
   args:
   - -ceux
   - |
-    /workspace/release/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
+    /workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
 
 images:
 # Images for the pipeline system itself
diff --git a/release/tag_for_hosted.sh b/release/tag_for_hosted.sh
deleted file mode 100755
index f6a9a397c83..00000000000
--- a/release/tag_for_hosted.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PROJECT_ID=$1
-COMMIT_SHA=$2
-SEM_VER=$3
-MM_VER=$4
-
-docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$SEM_VER
-docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/frontend:$MM_VER
-docker push gcr.io/$PROJECT_ID/hosted/frontend:$SEM_VER
-docker push gcr.io/$PROJECT_ID/hosted/frontend:$MM_VER
-
-docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/apiserver:$SEM_VER
-docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/apiserver:$MM_VER
-docker push gcr.io/$PROJECT_ID/hosted/apiserver:$SEM_VER
-docker push gcr.io/$PROJECT_ID/hosted/apiserver:$MM_VER
-
-docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
-docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
-docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$SEM_VER
-docker push gcr.io/$PROJECT_ID/hosted/viewercrd:$MM_VER
-
diff --git a/test/cloudbuild/mkp_verify.yaml b/test/cloudbuild/mkp_verify.yaml
index 98a359b3011..8ea80e391a4 100644
--- a/test/cloudbuild/mkp_verify.yaml
+++ b/test/cloudbuild/mkp_verify.yaml
@@ -1,7 +1,7 @@
 steps:
 - id:   'pullDeployer'
   name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/google/pipelines-test/deployer:${_MM_VERSION}']  
+  args: ['pull', 'gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:${_MM_VERSION}']  
 - id: "verify"
   waitFor: ['pullDeployer']
   name: "gcr.io/cloud-marketplace-tools/k8s/dev:latest"
@@ -11,7 +11,7 @@ steps:
     - |
       gcloud container clusters get-credentials hosted-verify-test --zone us-central1-c --project ml-pipeline-test
       kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
-      /scripts/verify --deployer="gcr.io/ml-pipeline/google/pipelines-test/deployer:${_MM_VERSION}" --wait_timeout="1800"
+      /scripts/verify --deployer="gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:${_MM_VERSION}" --wait_timeout="1800"
      
 timeout: '3600s'
 options:
@@ -19,7 +19,3 @@ options:
 
 substitutions:
   _MM_VERSION: ''
-
-# TODO: create new cluster and handover name here. use existing cluster will hit image cache
-# kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user <GSA>
-# 
\ No newline at end of file
diff --git a/test/deploy-pipeline-mkp-cli.sh b/test/deploy-pipeline-mkp-cli.sh
index 4b85f969b03..7732b465bcf 100644
--- a/test/deploy-pipeline-mkp-cli.sh
+++ b/test/deploy-pipeline-mkp-cli.sh
@@ -14,91 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
 
-echo "=================START MKP DEPLOY================"
+VERSION=$1
+COMMIT_SHA=$2
 
-GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest}
-
-# Configure gcloud as a Docker credential helper
-gcloud auth configure-docker
-
-# Delete argo first because KFP comes with argo too
-kubectl delete namespace argo --wait || echo "No argo installed"
-
-# Install the application resource definition
-kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
-
-# Grant user ability for using Role-Based Access Control
-kubectl create clusterrolebinding cluster-admin-binding \
-  --clusterrole cluster-admin \
-  --user $(gcloud config get-value account)
-
-export APP_INSTANCE_NAME=kubeflow-pipelines-test
-
-# Install mpdev
-BIN_FILE="$HOME/bin/mpdev"
-KFP_MANIFEST_DIR=${DIR}/../manifests/gcp_marketplace
-
-if ! which mpdev; then
-  echo "Install mpdev"
-  sudo docker pull gcr.io/cloud-marketplace-tools/k8s/dev
-  mkdir -p $HOME/bin/
-  touch $BIN_FILE
-  export PATH=$HOME/bin:$PATH
-  MKP_TOOLS_IMAGE=`grep -i "FROM" ${KFP_MANIFEST_DIR}/deployer/Dockerfile | sed 's/FROM //'`
-  sudo docker run ${MKP_TOOLS_IMAGE} cat /scripts/dev > "$BIN_FILE"
-  chmod +x "$BIN_FILE"
-fi
-
-pushd ${KFP_MANIFEST_DIR}
-# Update the version value on schema.yaml and application.yaml
-sed -ri 's/publishedVersion:.*/publishedVersion: '"$GCR_IMAGE_TAG"'/' schema.yaml
-sed -ri 's/version:.*/version: '"$GCR_IMAGE_TAG"'/' ./chart/kubeflow-pipelines/templates/application.yaml
-
-# Build new deployer
-export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/')
-export APP_NAME=$COMMIT_SHA
-export DEPLOYER_NAME=$REGISTRY/$APP_NAME/deployer
-
-sudo docker build --tag $DEPLOYER_NAME -f deployer/Dockerfile .
-
-sudo docker push $DEPLOYER_NAME:$GCR_IMAGE_TAG
-
-# Copy rest images and rename current images
-export MKP_VERSION=$GCR_IMAGE_TAG
-export GCR_FOLDER=$REGISTRY/$APP_NAME
-
-echo MKP_VERSION:$MKP_VERSION, GCR_FOLDER:$GCR_FOLDER
-
-export ARGO_VERSION=v2.3.0-license-compliance
-export CLOUDSQL_PROXY_VERSION=1.14
-export MLMD_SERVER_VERSION=0.21.1
-export MLMD_ENVOY_VERSION=initial
-export MINIO_VERSION=RELEASE.2019-08-14T20-37-41Z-license-compliance
-export MYSQL_VERSION=5.6
-
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/argoexec:$ARGO_VERSION $GCR_FOLDER/argoexecutor:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/workflow-controller:$ARGO_VERSION $GCR_FOLDER/argoworkflowcontroller:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/cloudsql-docker/gce-proxy:$CLOUDSQL_PROXY_VERSION $GCR_FOLDER/cloudsqlproxy:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/tfx-oss-public/ml_metadata_store_server:$MLMD_SERVER_VERSION $GCR_FOLDER/metadataserver:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/envoy:$MLMD_ENVOY_VERSION $GCR_FOLDER/metadataenvoy:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/minio:$MINIO_VERSION $GCR_FOLDER/minio:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/mysql:$MYSQL_VERSION $GCR_FOLDER/mysql:$MKP_VERSION
-
-gcloud container images add-tag --quiet $GCR_FOLDER/api-server:$MKP_VERSION $GCR_FOLDER/apiserver:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/inverse-proxy-agent:$MKP_VERSION $GCR_FOLDER/proxyagent:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/viewer-crd-controller:$MKP_VERSION $GCR_FOLDER/viewercrd:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/visualization-server:$MKP_VERSION $GCR_FOLDER/visualizationserver:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/deployer:$GCR_IMAGE_TAG $GCR_FOLDER/pipelines-test/deployer:$GCR_IMAGE_TAG
-
-# Run install script
-mpdev scripts/install  --deployer=$DEPLOYER_NAME:$GCR_IMAGE_TAG   --parameters='{"name": "'$APP_INSTANCE_NAME'", "namespace": "'$NAMESPACE'"}'
-
-echo "Status of pods after mpdev install"
-kubectl get pods -n ${NAMESPACE}
-
-popd
-
-# Waiting for the KFP resources are ready. TODO: verification of KFP resources
-sleep 60
+# sync trigger to avoid wait
+gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test
+return $?
\ No newline at end of file
diff --git a/test/postsubmit-tests-with-pipeline-deployment.sh b/test/postsubmit-tests-with-pipeline-deployment.sh
index 966e53860ac..974680ee379 100755
--- a/test/postsubmit-tests-with-pipeline-deployment.sh
+++ b/test/postsubmit-tests-with-pipeline-deployment.sh
@@ -137,23 +137,23 @@ GCR_IMAGE_TAG=${PULL_BASE_SHA}
 if [ ${KFP_DEPLOYMENT} == standalone ]; then
   time source "${DIR}/deploy-pipeline-lite.sh"
   echo "KFP standalone deployed"
+  # Submit the argo job and check the results
+  echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
+  ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
+  -p image-build-context-gcs-uri="$remote_code_archive_uri" \
+  -p commit-sha="${PULL_BASE_SHA}" \
+  -p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
+  -p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
+  -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
+  -n ${NAMESPACE} \
+  --serviceaccount test-runner \
+  -o name
+  `
+  echo "argo workflow submitted successfully"
+  source "${DIR}/check-argo-status.sh"
+  echo "test workflow completed"
 else
-  time source "${DIR}/deploy-pipeline-mkp-cli.sh"
-  echo "KFP mkp deployed"
+  SEM_VERSION="$(cat ${DIR}/../VERSION)"
+  source "${DIR}/deploy-pipeline-mkp-cli.sh" $SEM_VERSION $COMMIT_SHA
+  exit $?
 fi
-
-# Submit the argo job and check the results
-echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
-ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
--p image-build-context-gcs-uri="$remote_code_archive_uri" \
--p commit-sha="${PULL_BASE_SHA}" \
--p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
--p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
--p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
--n ${NAMESPACE} \
---serviceaccount test-runner \
--o name
-`
-echo "argo workflow submitted successfully"
-source "${DIR}/check-argo-status.sh"
-echo "test workflow completed"
diff --git a/test/presubmit-tests-mkp.sh b/test/presubmit-tests-mkp.sh
index 8ce3ce36c5e..567ddf443f4 100644
--- a/test/presubmit-tests-mkp.sh
+++ b/test/presubmit-tests-mkp.sh
@@ -14,77 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-# This is merged commit's SHA.
-COMMIT_SHA="$(git rev-parse HEAD)"
-GCR_IMAGE_BASE_DIR=gcr.io/${PROJECT}/presubmit/${COMMIT_SHA}
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
-
-# Configure `time` command output format.
-TIMEFORMAT="[test-timing] It took %lR."
-
-echo "presubmit test starts"
-if [ -n "$PULL_PULL_SHA" ]; then
-    echo "PR commit is ${PULL_PULL_SHA}"
-fi
-time source "${DIR}/test-prep.sh"
-echo "test env prepared"
-
-# TODO: extract this part to be a dedicated file for reusage.
-# =========================================================================
-## Wait for the cloudbuild job to be started (triggerd by PR commit)
-CLOUDBUILD_TIMEOUT_SECONDS=3600
-PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT=$(expr ${CLOUDBUILD_TIMEOUT_SECONDS} / 20 )
-CLOUDBUILD_STARTED=False
-
-for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
-do
-  output=`gcloud builds list --project="$CLOUDBUILD_PROJECT" --filter="sourceProvenance.resolvedRepoSource.commitSha:${COMMIT_SHA}"`
-  if [[ ${output} != "" ]]; then
-    CLOUDBUILD_STARTED=True
-    break
-  fi
-  sleep 20
-done
-
-if [[ ${CLOUDBUILD_STARTED} == False ]];then
-  echo "CloudBuild job for the commit not be triggered before timeout, exiting..."
-  exit 1
-fi
-
-## Wait for the cloudbuild job to complete, if success, all images ready
-CLOUDBUILD_FINISHED=TIMEOUT
-for i in $(seq 1 ${PULL_CLOUDBUILD_STATUS_MAX_ATTEMPT})
-do
-  output=`gcloud builds list --project="$CLOUDBUILD_PROJECT" --filter="sourceProvenance.resolvedRepoSource.commitSha:${COMMIT_SHA}"`
-  if [[ ${output} == *"SUCCESS"* ]]; then
-    CLOUDBUILD_FINISHED=SUCCESS
-    break
-  elif [[ ${output} == *"FAILURE"* ]]; then
-    CLOUDBUILD_FINISHED=FAILURE
-    break
-  fi
-  sleep 20
-done
-
-if [[ ${CLOUDBUILD_FINISHED} == FAILURE ]];then
-  echo "Cloud build failure, cannot proceed. exiting..."
-  exit 1
-elif [[ ${CLOUDBUILD_FINISHED} == TIMEOUT ]];then
-  echo "Cloud build timeout, cannot proceed. exiting..."
-  exit 1
-fi
-
-# =========================================================================
-## Create Cluster (delete in clean up)
-
-# =========================================================================
-## All images ready, trigger the MKP verification. Here we can't use Argo and have to use CloudBuild again
-## Run CB job in sync mode to easily get test result.
-
-## TODO
-## 
-# gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="0.2" --project=ml-pipeline-test
-
-## ??? how test cluster get deleted?
\ No newline at end of file
+# It would slow down presubmit as build full set images take longer time.
+# We enable post-submit for Hosted and then back here.
+echo "Placeholder for presubmit test with Hosted/MKP"
\ No newline at end of file
diff --git a/test/tag_for_hosted.sh b/test/tag_for_hosted.sh
new file mode 100755
index 00000000000..5dd569c19ee
--- /dev/null
+++ b/test/tag_for_hosted.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It's called in ".cloudbuild.yaml" for each commits to master
+# It's used for Hosted deployment testing but also can be reused for other usages.
+#
+# Later we need to unify the image names and path, after that, we may don't need this file.
+# All will be handled in Cloud Build yaml file.
+PROJECT_ID=$1
+COMMIT_SHA=$2
+SEM_VER=$3
+MM_VER=$4
+
+docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/frontend:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/frontend:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/frontend:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/frontend:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/apiserver:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/apiserver:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/apiserver:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/apiserver:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/scheduledworkflow:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/scheduledworkflow:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/scheduledworkflow:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/scheduledworkflow:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/viewercrd:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/viewercrd:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/viewercrd:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/viewercrd:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/persistenceagent:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/persistenceagent:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/persistenceagent:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/persistenceagent:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/proxyagent:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/inverse-proxy-agent:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/proxyagent:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/proxyagent:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/proxyagent:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/visualizationserver:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/visualization-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/visualizationserver:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/visualizationserver:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/visualizationserver:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadatawriter:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/metadata-writer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadatawriter:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadatawriter:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadatawriter:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cacheserver:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/cache-server:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cacheserver:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cacheserver:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cacheserver:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cachedeployer:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/cache-deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cachedeployer:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cachedeployer:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cachedeployer:$MM_VER
+
+docker tag gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataenvoy:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/metadata-envoy:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataenvoy:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataenvoy:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataenvoy:$MM_VER
+
+# Deployer required by MKP
+docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:$MM_VER
+
+# Primary binary required by MKP
+docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:$SEM_VER
+docker tag gcr.io/$PROJECT_ID/deployer:$COMMIT_SHA gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:$MM_VER
+
+# 3rd-party images
+docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$SEM_VER
+docker tag gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/metadataserver:$MM_VER
+
+docker tag gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/minio:$SEM_VER
+docker tag gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/minio:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/minio:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/minio:$MM_VER
+
+docker tag gcr.io/ml-pipeline/mysql:5.6 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/mysql:$SEM_VER
+docker tag gcr.io/ml-pipeline/mysql:5.6 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/mysql:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/mysql:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/mysql:$MM_VER
+
+docker tag gcr.io/cloudsql-docker/gce-proxy:1.14 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$SEM_VER
+docker tag gcr.io/cloudsql-docker/gce-proxy:1.14 gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/cloudsqlproxy:$MM_VER
+
+docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
+docker tag gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoexecutor:$MM_VER
+
+docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
+docker tag gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$SEM_VER
+docker push gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/argoworkflowcontroller:$MM_VER

From a793153272cefd1db9b2a277eaa53a7f5715e166 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Fri, 20 Mar 2020 15:41:52 +0800
Subject: [PATCH 10/12] fix

---
 .cloudbuild.yaml                | 26 +++++++++++++++++++++++++-
 test/deploy-pipeline-mkp-cli.sh |  4 ++--
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
index 69614bce20c..c54f7f188b3 100644
--- a/.cloudbuild.yaml
+++ b/.cloudbuild.yaml
@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Test before submit:
+# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test
+
 steps:
 
 # Build the Python SDK
@@ -194,6 +197,26 @@ steps:
          '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
   id:   'buildMetadataEnvoy'
 
+# Pull third_party images
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1']
+  id:   'pullMetadataServer'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
+  id:   'pullMinio'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/mysql:5.6']
+  id:   'pullMysql'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
+  id:   'pullCloudsqlProxy'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance']
+  id:   'pullArgoExecutor'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance']
+  id:   'pullArgoWorkflowController'
+
 # Tag for Hosted - SemVersion to Major.Minor parsing
 - id: "parseMajorMinorVersion"
   waitFor: ["-"]
@@ -210,7 +233,8 @@ steps:
   waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',            
             'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
             'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
-            'buildMarketplaceDeployer']
+            'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
+            'pullArgoExecutor', 'pullArgoWorkflowController']
   name: 'gcr.io/cloud-builders/docker'
   entrypoint: bash
   args:
diff --git a/test/deploy-pipeline-mkp-cli.sh b/test/deploy-pipeline-mkp-cli.sh
index 7732b465bcf..67d0fb7e41f 100644
--- a/test/deploy-pipeline-mkp-cli.sh
+++ b/test/deploy-pipeline-mkp-cli.sh
@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
 
 VERSION=$1
 COMMIT_SHA=$2
 
 # sync trigger to avoid wait
-gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test
-return $?
\ No newline at end of file
+gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test
\ No newline at end of file

From dcaa1fed1bbec2e7205322e05875490d2f6cb36b Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Fri, 20 Mar 2020 17:07:10 +0800
Subject: [PATCH 11/12] pass manual test before submit

---
 test/cloudbuild/mkp_verify.yaml | 6 +++---
 test/deploy-pipeline-mkp-cli.sh | 6 +++++-
 2 files changed, 8 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 test/deploy-pipeline-mkp-cli.sh

diff --git a/test/cloudbuild/mkp_verify.yaml b/test/cloudbuild/mkp_verify.yaml
index 8ea80e391a4..d263a038957 100644
--- a/test/cloudbuild/mkp_verify.yaml
+++ b/test/cloudbuild/mkp_verify.yaml
@@ -1,7 +1,7 @@
 steps:
 - id:   'pullDeployer'
   name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:${_MM_VERSION}']  
+  args: ['pull', 'gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:${_DEPLOYER_VERSION}']  
 - id: "verify"
   waitFor: ['pullDeployer']
   name: "gcr.io/cloud-marketplace-tools/k8s/dev:latest"
@@ -11,11 +11,11 @@ steps:
     - |
       gcloud container clusters get-credentials hosted-verify-test --zone us-central1-c --project ml-pipeline-test
       kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
-      /scripts/verify --deployer="gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA:${_MM_VERSION}" --wait_timeout="1800"
+      /scripts/verify --deployer="gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:${_DEPLOYER_VERSION}" --wait_timeout="1800"
      
 timeout: '3600s'
 options:
   machineType: N1_HIGHCPU_8
 
 substitutions:
-  _MM_VERSION: ''
+  _DEPLOYER_VERSION: ''
diff --git a/test/deploy-pipeline-mkp-cli.sh b/test/deploy-pipeline-mkp-cli.sh
old mode 100644
new mode 100755
index 67d0fb7e41f..e3337afa0c6
--- a/test/deploy-pipeline-mkp-cli.sh
+++ b/test/deploy-pipeline-mkp-cli.sh
@@ -14,10 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+# Test before submit:
+# test/deploy-pipline-mkp-cli.sh 0.2 $(git rev-parse HEAD)
+
 set -ex
 
 VERSION=$1
 COMMIT_SHA=$2
 
 # sync trigger to avoid wait
-gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_MM_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test
\ No newline at end of file
+gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_DEPLOYER_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test
\ No newline at end of file

From ec5c9ffbcde42c4eaa60e8b1cb1ea9775de7ba08 Mon Sep 17 00:00:00 2001
From: Renmin Gu <renming@google.com>
Date: Mon, 23 Mar 2020 09:52:04 +0800
Subject: [PATCH 12/12] 0.3.0

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 28af839c0a9..9325c3ccda9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.2.5
\ No newline at end of file
+0.3.0
\ No newline at end of file