Post-submit test for Hosted/MKP (mpdev verify)

.cloudbuild.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Test before submit:
+# gcloud builds submit --config=.cloudbuild.yaml --substitutions=COMMIT_SHA="$(git rev-parse HEAD)" --project=ml-pipeline-test
+
 steps:
 
 # Build the Python SDK
@@ -194,6 +197,26 @@ steps:
          '/workspace/third_party/metadata_envoy/Dockerfile', '/workspace']
   id:   'buildMetadataEnvoy'
 
+# Pull third_party images
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.21.1']
+  id:   'pullMetadataServer'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance']
+  id:   'pullMinio'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/mysql:5.6']
+  id:   'pullMysql'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.14']
+  id:   'pullCloudsqlProxy'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v2.3.0-license-compliance']
+  id:   'pullArgoExecutor'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v2.3.0-license-compliance']
+  id:   'pullArgoWorkflowController'
+
 # Tag for Hosted - SemVersion to Major.Minor parsing
 - id: "parseMajorMinorVersion"
   waitFor: ["-"]
@@ -207,15 +230,17 @@ steps:
 
 # Tag for Hosted - Tag to hosted folder with MKP friendly name
 - id:   'tagForHosted'
-  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildViewerCrdController',
-            'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
-            'buildMarketplaceDeployer']
+  waitFor: ['parseMajorMinorVersion', 'buildFrontend', 'buildApiServer', 'buildScheduledWorkflow',            
+            'buildViewerCrdController', 'buildPersistenceAgent', 'buildInverseProxyAgent', 'buildVisualizationServer',
+            'buildMetadataWriter', 'buildCacheServer', 'buildCacheDeployer', 'buildMetadataEnvoy',
+            'buildMarketplaceDeployer', 'pullMetadataServer', 'pullMinio', 'pullMysql', 'pullCloudsqlProxy',
+            'pullArgoExecutor', 'pullArgoWorkflowController']
   name: 'gcr.io/cloud-builders/docker'
   entrypoint: bash
   args:
   - -ceux
   - |
-    /workspace/release/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
+    /workspace/test/tag_for_hosted.sh $PROJECT_ID $COMMIT_SHA $(cat /workspace/VERSION) $(cat /workspace/mm.ver)
 
 images:
 # Images for the pipeline system itself

VERSION

@@ -1 +1 @@
-0.2.5
+0.3.0

test/cloudbuild/mkp_verify.yaml

@@ -0,0 +1,21 @@
+steps:
+- id:   'pullDeployer'
+  name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:${_DEPLOYER_VERSION}']  
+- id: "verify"
+  waitFor: ['pullDeployer']
+  name: "gcr.io/cloud-marketplace-tools/k8s/dev:latest"
+  entrypoint: '/bin/bash'
+  args:
+    - -ceux
+    - |
+      gcloud container clusters get-credentials hosted-verify-test --zone us-central1-c --project ml-pipeline-test
+      kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
+      /scripts/verify --deployer="gcr.io/$PROJECT_ID/hosted/$COMMIT_SHA/deployer:${_DEPLOYER_VERSION}" --wait_timeout="1800"
+
+timeout: '3600s'
+options:
+  machineType: N1_HIGHCPU_8
+
+substitutions:
+  _DEPLOYER_VERSION: ''

test/deploy-pipeline-mkp-cli.sh

@@ -14,91 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-echo "=================START MKP DEPLOY================"
-
-GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest}
-
-# Configure gcloud as a Docker credential helper
-gcloud auth configure-docker
-
-# Delete argo first because KFP comes with argo too
-kubectl delete namespace argo --wait || echo "No argo installed"
-
-# Install the application resource definition
-kubectl apply -f "https://raw.githubusercontent.com/GoogleCloudPlatform/marketplace-k8s-app-tools/master/crd/app-crd.yaml"
-
-# Grant user ability for using Role-Based Access Control
-kubectl create clusterrolebinding cluster-admin-binding \
-  --clusterrole cluster-admin \
-  --user $(gcloud config get-value account)
-
-export APP_INSTANCE_NAME=kubeflow-pipelines-test
-
-# Install mpdev
-BIN_FILE="$HOME/bin/mpdev"
-KFP_MANIFEST_DIR=${DIR}/../manifests/gcp_marketplace
-
-if ! which mpdev; then
-  echo "Install mpdev"
-  sudo docker pull gcr.io/cloud-marketplace-tools/k8s/dev
-  mkdir -p $HOME/bin/
-  touch $BIN_FILE
-  export PATH=$HOME/bin:$PATH
-  MKP_TOOLS_IMAGE=`grep -i "FROM" ${KFP_MANIFEST_DIR}/deployer/Dockerfile | sed 's/FROM //'`
-  sudo docker run ${MKP_TOOLS_IMAGE} cat /scripts/dev > "$BIN_FILE"
-  chmod +x "$BIN_FILE"
-fi
 
-pushd ${KFP_MANIFEST_DIR}
-# Update the version value on schema.yaml and application.yaml
-sed -ri 's/publishedVersion:.*/publishedVersion: '"$GCR_IMAGE_TAG"'/' schema.yaml
-sed -ri 's/version:.*/version: '"$GCR_IMAGE_TAG"'/' ./chart/kubeflow-pipelines/templates/application.yaml
+# Test before submit:
+# test/deploy-pipline-mkp-cli.sh 0.2 $(git rev-parse HEAD)
 
-# Build new deployer
-export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/')
-export APP_NAME=$COMMIT_SHA
-export DEPLOYER_NAME=$REGISTRY/$APP_NAME/deployer
-
-sudo docker build --tag $DEPLOYER_NAME -f deployer/Dockerfile .
-
-sudo docker push $DEPLOYER_NAME:$GCR_IMAGE_TAG
-
-# Copy rest images and rename current images
-export MKP_VERSION=$GCR_IMAGE_TAG
-export GCR_FOLDER=$REGISTRY/$APP_NAME
-
-echo MKP_VERSION:$MKP_VERSION, GCR_FOLDER:$GCR_FOLDER
-
-export ARGO_VERSION=v2.3.0-license-compliance
-export CLOUDSQL_PROXY_VERSION=1.14
-export MLMD_SERVER_VERSION=0.21.1
-export MLMD_ENVOY_VERSION=initial
-export MINIO_VERSION=RELEASE.2019-08-14T20-37-41Z-license-compliance
-export MYSQL_VERSION=5.6
-
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/argoexec:$ARGO_VERSION $GCR_FOLDER/argoexecutor:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/workflow-controller:$ARGO_VERSION $GCR_FOLDER/argoworkflowcontroller:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/cloudsql-docker/gce-proxy:$CLOUDSQL_PROXY_VERSION $GCR_FOLDER/cloudsqlproxy:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/tfx-oss-public/ml_metadata_store_server:$MLMD_SERVER_VERSION $GCR_FOLDER/metadataserver:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/envoy:$MLMD_ENVOY_VERSION $GCR_FOLDER/metadataenvoy:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/minio:$MINIO_VERSION $GCR_FOLDER/minio:$MKP_VERSION
-gcloud container images add-tag --quiet gcr.io/ml-pipeline/mysql:$MYSQL_VERSION $GCR_FOLDER/mysql:$MKP_VERSION
-
-gcloud container images add-tag --quiet $GCR_FOLDER/api-server:$MKP_VERSION $GCR_FOLDER/apiserver:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/inverse-proxy-agent:$MKP_VERSION $GCR_FOLDER/proxyagent:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/viewer-crd-controller:$MKP_VERSION $GCR_FOLDER/viewercrd:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/visualization-server:$MKP_VERSION $GCR_FOLDER/visualizationserver:$MKP_VERSION
-gcloud container images add-tag --quiet $GCR_FOLDER/deployer:$GCR_IMAGE_TAG $GCR_FOLDER/pipelines-test/deployer:$GCR_IMAGE_TAG
-
-# Run install script
-mpdev scripts/install  --deployer=$DEPLOYER_NAME:$GCR_IMAGE_TAG   --parameters='{"name": "'$APP_INSTANCE_NAME'", "namespace": "'$NAMESPACE'"}'
-
-echo "Status of pods after mpdev install"
-kubectl get pods -n ${NAMESPACE}
+set -ex
 
-popd
+VERSION=$1
+COMMIT_SHA=$2
 
-# Waiting for the KFP resources are ready. TODO: verification of KFP resources
-sleep 60
+# sync trigger to avoid wait
+gcloud builds submit --config=test/cloudbuild/mkp_verify.yaml --substitutions=_DEPLOYER_VERSION="$VERSION",COMMIT_SHA="$COMMIT_SHA" --project=ml-pipeline-test

test/postsubmit-tests-with-pipeline-deployment.sh

@@ -137,23 +137,23 @@ GCR_IMAGE_TAG=${PULL_BASE_SHA}
 if [ ${KFP_DEPLOYMENT} == standalone ]; then
   time source "${DIR}/deploy-pipeline-lite.sh"
   echo "KFP standalone deployed"
+  # Submit the argo job and check the results
+  echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
+  ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
+  -p image-build-context-gcs-uri="$remote_code_archive_uri" \
+  -p commit-sha="${PULL_BASE_SHA}" \
+  -p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
+  -p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
+  -p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
+  -n ${NAMESPACE} \
+  --serviceaccount test-runner \
+  -o name
+  `
+  echo "argo workflow submitted successfully"
+  source "${DIR}/check-argo-status.sh"
+  echo "test workflow completed"
 else
-  time source "${DIR}/deploy-pipeline-mkp-cli.sh"
-  echo "KFP mkp deployed"
+  SEM_VERSION="$(cat ${DIR}/../VERSION)"
+  source "${DIR}/deploy-pipeline-mkp-cli.sh" $SEM_VERSION $COMMIT_SHA
+  exit $?
 fi
-
-# Submit the argo job and check the results
-echo "submitting argo workflow for commit ${PULL_BASE_SHA}..."
-ARGO_WORKFLOW=`argo submit ${DIR}/${WORKFLOW_FILE} \
--p image-build-context-gcs-uri="$remote_code_archive_uri" \
--p commit-sha="${PULL_BASE_SHA}" \
--p component-image-prefix="${GCR_IMAGE_BASE_DIR}/" \
--p target-image-prefix="${TARGET_IMAGE_BASE_DIR}/" \
--p test-results-gcs-dir="${TEST_RESULTS_GCS_DIR}" \
--n ${NAMESPACE} \
---serviceaccount test-runner \
--o name
-`
-echo "argo workflow submitted successfully"
-source "${DIR}/check-argo-status.sh"
-echo "test workflow completed"

test/presubmit-tests-mkp.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# It would slow down presubmit as build full set images take longer time.
+# We enable post-submit for Hosted and then back here.
+echo "Placeholder for presubmit test with Hosted/MKP"