.cloudbuild.yaml

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

steps:

# Build the Python SDK
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh /workspace/kfp.tar.gz']
  id:   'preparePythonSDK'
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz']
  id:   'copyPythonSDK'
  waitFor: ['preparePythonSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz']
  id:   'copyPythonSDKToLatest'
  waitFor: ['preparePythonSDK']

# Build the Python Component SDK
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools;cd /workspace/component_sdk/python;python setup.py sdist --format=gztar;cp dist/*.tar.gz /workspace/kfp-component.tar.gz']
  id:   'preparePythonComponentSDK'
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp-component.tar.gz']
  id:   'copyPythonComponentSDK'
  waitFor: ['preparePythonComponentSDK']
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '/workspace/kfp-component.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp-component.tar.gz']
  id:   'copyPythonComponentSDKToLatest'
  waitFor: ['preparePythonComponentSDK']

# Build the pipeline system images
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'sed -i -e "s/ARG DATE/ENV DATE \"$(date -u)\"/" /workspace/frontend/Dockerfile']
  id:   'prepareFrontend'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA',
         '--build-arg', 'COMMIT_HASH=$COMMIT_SHA', '-f',
         '/workspace/frontend/Dockerfile', '/workspace']
  id:   'buildFrontend'
  waitFor: ['prepareFrontend']
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA',
         '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile', '/workspace']
  id:   'buildApiServer'
  waitFor: ['copyPythonSDK']
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.scheduledworkflow', '/workspace']
  id:   'buildScheduledWorkflow'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.viewercontroller', '/workspace']
  id:   'buildViewerCrdController'
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA', '-f',
         '/workspace/backend/Dockerfile.persistenceagent', '/workspace']
  id:   'buildPersistenceAgent'

# Build the Dataflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataflow/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildPredictComponent'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataflow/tfdv && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTFDVComponent'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataflow/tft && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTFTComponent'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataflow/tfma && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildTMAComponent'

# Build the Kubeflow-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  args: ['build', '-t', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA',
         '/workspace/components/kubeflow/deployer']
  id: 'buildDeployer'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/launcher && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildLauncher'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/kubeflow/dnntrainer && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA -l ml-pipeline-kubeflow-tf-trainer-gpu -b 1.6.0-gpu']
  id: 'buildGpuTrainer'

# Build the Dataproc-based pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/analyze && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocAnalyze'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/create_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocCreateCluster'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/delete_cluster && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocDeleteCluster'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/predict && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocPredict'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/transform && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTransform'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/dataproc/train && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildDataprocTrain'

# Build the ResNet-CMLE sample pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/resnet-cmle/containers/deploy && ./build.sh $PROJECT_ID $COMMIT_SHA']
  id: 'buildResnetCMLEDeployComponent'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/resnet-cmle/containers/preprocess && ./build.sh $PROJECT_ID $COMMIT_SHA']
  id: 'buildResnetCMLEPreprocessComponent'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/resnet-cmle/containers/train && ./build.sh $PROJECT_ID $COMMIT_SHA']
  id: 'buildResnetCMLETrainComponent'

# Build the Generic GCP component image
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/gcp/container/ && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildGcpGenericComponent'

# Build the local pipeline component images
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/confusion_matrix && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildConfusionMatrix'
- name: 'gcr.io/cloud-builders/docker'
  entrypoint: '/bin/bash'
  args: ['-c', 'cd /workspace/components/local/roc && ./build_image.sh -p $PROJECT_ID -t $COMMIT_SHA']
  id: 'buildROC'

# Build the tagged samples
- name: 'debian'
  entrypoint: '/bin/bash'
  args: ['-c', 'sed -i -e "s/gcr.io\/ml-pipeline\/\([^\/:]\+\):\([a-zA-Z0-9_.-]\)\+/gcr.io\/$PROJECT_ID\/\\1:$COMMIT_SHA/g" samples/*/*.py']
  id:   'prepareVersionedSamples'
- name: 'gcr.io/cloud-builders/gsutil'
  args: ['cp', '-r', '/workspace/samples/', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/samples']
  id:   'copyVersionedSamples'
  waitFor: ['prepareVersionedSamples']

images:
# Images for the pipeline system itself
- 'gcr.io/$PROJECT_ID/frontend:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/api-server:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/persistenceagent:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/viewer-crd-controller:$COMMIT_SHA'

# Images for the Dataflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfdv:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tft:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfma:$COMMIT_SHA'

# Images for the Kubeflow-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf:$COMMIT_SHA'

# Images for the Dataproc-based pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA'

# Images for the ResNet CMLE sample  pipeline components
- 'gcr.io/$PROJECT_ID/resnet-deploy:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/resnet-preprocess:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/resnet-train:$COMMIT_SHA'

# Images for the GCP generic pipeline components
- 'gcr.io/$PROJECT_ID/ml-pipeline-gcp:$COMMIT_SHA'

# Images for the local components
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'
- 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'

timeout: '3600s'
options:
 diskSizeGb: 300
 machineType: 'N1_HIGHCPU_8'