diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 3667f45d01b..1fb614af519 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -116,229 +116,6 @@ steps: id: 'copyPythonSDKToLatest' waitFor: ['copyPythonSDKLocal'] -# Pull and retag the images for the Dataflow-based pipeline components -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA'] - id: 'pullPredict' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:$TAG_NAME'] - id: 'tagPredictVersionNumber' - waitFor: ['pullPredict'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA'] - id: 'tagPredictCommitSHA' - waitFor: ['pullPredict'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfdv:$COMMIT_SHA'] - id: 'pullTFDV' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfdv:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:$TAG_NAME'] - id: 'tagTFDVVersionNumber' - waitFor: ['pullTFDV'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfdv:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:$COMMIT_SHA'] - id: 'tagTFDVCommitSHA' - waitFor: ['pullTFDV'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tft:$COMMIT_SHA'] - id: 'pullTFT' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tft:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:$TAG_NAME'] - id: 'tagTFTVersionNumber' - waitFor: ['pullTFT'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tft:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:$COMMIT_SHA'] - id: 'tagTFTCommitSHA' - waitFor: ['pullTFT'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfma:$COMMIT_SHA'] - id: 'pullTFMA' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfma:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:$TAG_NAME'] - id: 'tagTFMAVersionNumber' - waitFor: ['pullTFMA'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tfma:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:$COMMIT_SHA'] - id: 'tagTFMACommitSHA' - waitFor: ['pullTFMA'] - -# Pull and retag the images for the Kubeflow-based pipeline components -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'] - id: 'pullDeployer' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:$TAG_NAME'] - id: 'tagDeployerVersionNumber' - waitFor: ['pullDeployer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-deployer:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:$COMMIT_SHA'] - id: 'tagDeployerCommitSHA' - waitFor: ['pullDeployer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'] - id: 'pullTrainer' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:$TAG_NAME'] - id: 'tagTrainerVersionNumber' - waitFor: ['pullTrainer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA'] - id: 'tagTrainerCommitSHA' - waitFor: ['pullTrainer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'] - id: 'pullGpuTrainer' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:$TAG_NAME'] - id: 'tagGpuTrainerVersionNumber' - waitFor: ['pullGpuTrainer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer-gpu:$COMMIT_SHA'] - id: 'tagGpuTrainerCommitSHA' - waitFor: ['pullGpuTrainer'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf:$COMMIT_SHA'] - id: 'pullLauncher' -- name: 'gcr.io/cloud-builders/docker' - entrypoint: '/bin/bash' - args: ['-c', 'printf "FROM gcr.io/$PROJECT_ID/ml-pipeline-kubeflow-tf:$COMMIT_SHA\nENV TRAINER_IMAGE_NAME gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA" > Dockerfile; docker build -t ml-pipeline-kubeflow-tf .'] - id: 'buildLauncher' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'ml-pipeline-kubeflow-tf', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:$TAG_NAME'] - id: 'tagLauncherVersionNumber' - waitFor: ['buildLauncher'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'ml-pipeline-kubeflow-tf', 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:$COMMIT_SHA'] - id: 'tagLauncherCommitSHA' - waitFor: ['buildLauncher'] - -# Pull and retag the images for the Dataproc-based pipeline components -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA'] - id: 'pullDataprocAnalyze' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:$TAG_NAME'] - id: 'tagDataprocAnalyzeVersionNumber' - waitFor: ['pullDataprocAnalyze'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-analyze:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:$COMMIT_SHA'] - id: 'tagDataprocAnalyzeCommitSHA' - waitFor: ['pullDataprocAnalyze'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA'] - id: 'pullDataprocCreateCluster' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:$TAG_NAME'] - id: 'tagDataprocCreateClusterVersionNumber' - waitFor: ['pullDataprocCreateCluster'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA'] - id: 'tagDataprocCreateClusterCommitSHA' - waitFor: ['pullDataprocCreateCluster'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA'] - id: 'pullDataprocDeleteCluster' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:$TAG_NAME'] - id: 'tagDataprocDeleteClusterVersionNumber' - waitFor: ['pullDataprocDeleteCluster'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA'] - id: 'tagDataprocDeleteClusterCommitSHA' - waitFor: ['pullDataprocDeleteCluster'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA'] - id: 'pullDataprocPredict' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:$TAG_NAME'] - id: 'tagDataprocPredictVersionNumber' - waitFor: ['pullDataprocPredict'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-predict:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:$COMMIT_SHA'] - id: 'tagDataprocPredictCommitSHA' - waitFor: ['pullDataprocPredict'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA'] - id: 'pullDataprocTransform' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:$TAG_NAME'] - id: 'tagDataprocTransformVersionNumber' - waitFor: ['pullDataprocTransform'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-transform:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:$COMMIT_SHA'] - id: 'tagDataprocTransformCommitSHA' - waitFor: ['pullDataprocTransform'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA'] - id: 'pullDataprocTrain' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-train:$TAG_NAME'] - id: 'tagDataprocTrainVersionNumber' - waitFor: ['pullDataprocTrain'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-dataproc-train:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-dataproc-train:$COMMIT_SHA'] - id: 'tagDataprocTrainCommitSHA' - waitFor: ['pullDataprocTrain'] - -# Pull and retag the images for the ResNet CMLE sample pipeline components -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/resnet-deploy:$COMMIT_SHA'] - id: 'pullResNetDeploy' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-deploy:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-deploy:$TAG_NAME'] - id: 'tagResNetDeployVersionNumber' - waitFor: ['pullResNetDeploy'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-deploy:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-deploy:$COMMIT_SHA'] - id: 'tagResNetDeployCommitSHA' - waitFor: ['pullResNetDeploy'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/resnet-preprocess:$COMMIT_SHA'] - id: 'pullResNetPreprocess' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-preprocess:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-preprocess:$TAG_NAME'] - id: 'tagResNetPreprocessVersionNumber' - waitFor: ['pullResNetPreprocess'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-preprocess:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-preprocess:$COMMIT_SHA'] - id: 'tagResNetPreprocessCommitSHA' - waitFor: ['pullResNetPreprocess'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/resnet-train:$COMMIT_SHA'] - id: 'pullResNetTrain' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-train:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-train:$TAG_NAME'] - id: 'tagResNetTrainVersionNumber' - waitFor: ['pullResNetTrain'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/resnet-train:$COMMIT_SHA', 'gcr.io/ml-pipeline/resnet-train:$COMMIT_SHA'] - id: 'tagResNetTrainCommitSHA' - waitFor: ['pullResNetTrain'] - -# Pull and retag the images for the local pipeline components -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'] - id: 'pullLocalConfusionMatrix' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:$TAG_NAME'] - id: 'tagLocalConfusionMatrixVersionNumber' - waitFor: ['pullLocalConfusionMatrix'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-local-confusion-matrix:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:$COMMIT_SHA'] - id: 'tagLocalConfusionMatrixCommitSHA' - waitFor: ['pullLocalConfusionMatrix'] -- name: 'gcr.io/cloud-builders/docker' - args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA'] - id: 'pullLocalROC' -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-local-roc:$TAG_NAME'] - id: 'tagLocalROCVersionNumber' - waitFor: ['pullLocalROC'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', 'gcr.io/$PROJECT_ID/ml-pipeline-local-roc:$COMMIT_SHA', 'gcr.io/ml-pipeline/ml-pipeline-local-roc:$COMMIT_SHA'] - id: 'tagLocalROCCommitSHA' - waitFor: ['pullLocalROC'] - # Build the tagged samples - name: 'debian' entrypoint: '/bin/bash' @@ -366,45 +143,4 @@ images: - 'gcr.io/ml-pipeline/bootstrapper:$TAG_NAME' - 'gcr.io/ml-pipeline/bootstrapper:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:$COMMIT_SHA' - -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:$COMMIT_SHA' - -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-train:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-dataproc-train:$COMMIT_SHA' - -- 'gcr.io/ml-pipeline/resnet-deploy:$TAG_NAME' -- 'gcr.io/ml-pipeline/resnet-deploy:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/resnet-preprocess:$TAG_NAME' -- 'gcr.io/ml-pipeline/resnet-preprocess:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/resnet-train:$TAG_NAME' -- 'gcr.io/ml-pipeline/resnet-train:$COMMIT_SHA' - -- 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:$COMMIT_SHA' -- 'gcr.io/ml-pipeline/ml-pipeline-local-roc:$TAG_NAME' -- 'gcr.io/ml-pipeline/ml-pipeline-local-roc:$COMMIT_SHA' - timeout: '1200s' diff --git a/bootstrapper.yaml b/bootstrapper.yaml index bd621e58640..10c7b70459c 100644 --- a/bootstrapper.yaml +++ b/bootstrapper.yaml @@ -65,7 +65,7 @@ spec: spec: containers: - name: deploy - image: gcr.io/ml-pipeline/bootstrapper:0.1.4 #TODO-release: update the release tag for the next release + image: gcr.io/ml-pipeline/bootstrapper:0.1.4 imagePullPolicy: 'Always' # Additional parameter available: args: [ diff --git a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py index 25ea5caea09..63b62e19670 100644 --- a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py +++ b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py @@ -17,7 +17,7 @@ def kubeflow_tfjob_launcher_op(container_image, command, number_of_workers: int, number_of_parameter_servers: int, tfjob_timeout_minutes: int, output_dir=None, step_name='TFJob-launcher'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:0.1.4',#TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--workers', number_of_workers, '--pss', number_of_parameter_servers, diff --git a/components/kubeflow/launcher/src/train.template.yaml b/components/kubeflow/launcher/src/train.template.yaml index 7be145daccd..d0877911fb5 100644 --- a/components/kubeflow/launcher/src/train.template.yaml +++ b/components/kubeflow/launcher/src/train.template.yaml @@ -26,7 +26,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4 #TODO-release: update the release tag for the next release + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789 command: - python - -m @@ -38,7 +38,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4 #TODO-release: update the release tag for the next release + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789 command: - python - -m @@ -50,7 +50,7 @@ spec: spec: containers: - name: tensorflow - image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4 #TODO-release: update the release tag for the next release + image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789 command: - python - -m diff --git a/components/release.sh b/components/release.sh new file mode 100755 index 00000000000..f94ebdf60b2 --- /dev/null +++ b/components/release.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script automated the process to release the component images. +# To run it, find a good release candidate commit SHA from ml-pipeline-staging project, +# and provide a full github COMMIT SHA to the script. E.g. +# ./release.sh 2118baf752d3d30a8e43141165e13573b20d85b8 +# The script copies the images from staging to prod, and update the local code. +# You can then send a PR using your local branch. + +set -xe + +images=( + "ml-pipeline-dataflow-tf-predict" + "ml-pipeline-dataflow-tfdv" + "ml-pipeline-dataflow-tft" + "ml-pipeline-dataflow-tfma" + "ml-pipeline-kubeflow-deployer" + "ml-pipeline-kubeflow-tf-trainer" + "ml-pipeline-kubeflow-tf" + "ml-pipeline-dataproc-analyze" + "ml-pipeline-dataproc-create-cluster" + "ml-pipeline-dataproc-delete-cluster" + "ml-pipeline-dataproc-predict" + "ml-pipeline-dataproc-transform" + "ml-pipeline-dataproc-train" + "resnet-deploy" + "resnet-preprocess" + "resnet-train" + "ml-pipeline-local-confusion-matrix" + "ml-pipeline-local-roc" +) + +COMMIT_SHA=$1 +FROM_GCR_PREFIX='gcr.io/ml-pipeline-staging/' +TO_GCR_PREFIX='gcr.io/ml-pipeline/' +PARENT_PATH=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) + +for image in "${images[@]}" +do + TARGET_IMAGE_BASE=${TO_GCR_PREFIX}${image} + TARGET_IMAGE=${TARGET_IMAGE_BASE}:${COMMIT_SHA} + + # Move image from staging to prod GCR + gcloud container images add-tag --quiet \ + ${FROM_GCR_PREFIX}${image}:${COMMIT_SHA} ${TARGET_IMAGE} + + # Update the code + find "${PARENT_PATH}/../samples" -type f | while read file; do sed -i -e "s|${TARGET_IMAGE_BASE}:\([a-zA-Z0-9_.-]\)\+|${TARGET_IMAGE}|g" "$file"; done + find "${PARENT_PATH}" -type f | while read file; do sed -i -e "s|${TARGET_IMAGE_BASE}:\([a-zA-Z0-9_.-]\)\+|${TARGET_IMAGE}|g" "$file"; done +done diff --git a/samples/kubeflow-tf/kubeflow-training-classification.py b/samples/kubeflow-tf/kubeflow-training-classification.py index d08f81fe59c..d776d10ec6e 100755 --- a/samples/kubeflow-tf/kubeflow-training-classification.py +++ b/samples/kubeflow-tf/kubeflow-training-classification.py @@ -21,7 +21,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--train', train_data, '--eval', evaluation_data, @@ -38,7 +38,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--transformed-data-dir', transformed_data_dir, '--schema', schema, @@ -55,7 +55,7 @@ def kubeflow_tf_training_op(transformed_data_dir, schema: 'GcsUri[text/json]', l def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--data', evaluation_data, '--schema', schema, @@ -71,7 +71,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] def confusion_matrix_op(predictions, output, step_name='confusionmatrix'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--predictions', predictions, '--output', output, diff --git a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb index 8c45205d2ad..a8409edc615 100644 --- a/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb +++ b/samples/notebooks/KubeFlow Pipeline Using TFX OSS Components.ipynb @@ -43,12 +43,12 @@ "EVAL_DATA = 'gs://ml-pipeline-playground/tfx/taxi-cab-classification/eval.csv'\n", "HIDDEN_LAYER_SIZE = '1500'\n", "STEPS = 3000\n", - "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:0.1.4'#TODO-release: update the release tag for the next release\n", - "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:0.1.4'#TODO-release: update the release tag for the next release\n", - "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:0.1.4'#TODO-release: update the release tag for the next release\n", - "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:0.1.4'#TODO-release: update the release tag for the next release\n", - "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4'#TODO-release: update the release tag for the next release\n", - "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:0.1.4'#TODO-release: update the release tag for the next release\n", + "DATAFLOW_TFDV_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", + "DATAFLOW_TFT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", + "DATAFLOW_TFMA_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", + "DATAFLOW_TF_PREDICT_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", + "KUBEFLOW_TF_TRAINER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", + "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789'\n", "DEV_DEPLOYER_MODEL = 'notebook_tfx_devtaxi.beta'\n", "PROD_DEPLOYER_MODEL = 'notebook_tfx_prodtaxi.beta'" ] diff --git a/samples/tfx/taxi-cab-classification-pipeline.py b/samples/tfx/taxi-cab-classification-pipeline.py index 291a48e3aff..23253cbf890 100755 --- a/samples/tfx/taxi-cab-classification-pipeline.py +++ b/samples/tfx/taxi-cab-classification-pipeline.py @@ -21,7 +21,7 @@ def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'GcsUri', column_names: 'GcsUri[text/json]', key_columns, project: 'GcpProject', mode, validation_output: 'GcsUri[Directory]', step_name='validation'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfdv:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--csv-data-for-inference', inference_data, '--csv-data-to-validate', validation_data, @@ -40,7 +40,7 @@ def dataflow_tf_data_validation_op(inference_data: 'GcsUri', validation_data: 'G def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', preprocess_mode, preprocess_module: 'GcsUri[text/code/python]', transform_output: 'GcsUri[Directory]', step_name='preprocess'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tft:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--train', train_data, '--eval', evaluation_data, @@ -57,7 +57,7 @@ def dataflow_tf_transform_op(train_data: 'GcsUri', evaluation_data: 'GcsUri', sc def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate: float, hidden_layer_size: int, steps: int, target: str, preprocess_module: 'GcsUri[text/code/python]', training_output: 'GcsUri[Directory]', step_name='training'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--transformed-data-dir', transformed_data_dir, '--schema', schema, @@ -74,7 +74,7 @@ def tf_train_op(transformed_data_dir, schema: 'GcsUri[text/json]', learning_rate def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', project: 'GcpProject', analyze_mode, analyze_slice_column, analysis_output: 'GcsUri', step_name='analysis'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--model', model, '--eval', evaluation_data, @@ -91,7 +91,7 @@ def dataflow_tf_model_analyze_op(model: 'TensorFlow model', evaluation_data: 'Gc def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json]', target: str, model: 'TensorFlow model', predict_mode, project: 'GcpProject', prediction_output: 'GcsUri', step_name='prediction'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--data', evaluation_data, '--schema', schema, @@ -108,7 +108,7 @@ def dataflow_tf_predict_op(evaluation_data: 'GcsUri', schema: 'GcsUri[text/json] def confusion_matrix_op(predictions: 'GcsUri', output: 'GcsUri', step_name='confusion_matrix'): return dsl.ContainerOp( name=step_name, - image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--output', '%s/{{workflow.name}}/confusionmatrix' % output, '--predictions', predictions, @@ -119,7 +119,7 @@ def confusion_matrix_op(predictions: 'GcsUri', output: 'GcsUri', step_name='conf def roc_op(predictions: 'GcsUri', output: 'GcsUri', step_name='roc'): return dsl.ContainerOp( name=step_name, - image='gcr.io/ml-pipeline/ml-pipeline-local-roc:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-local-roc:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--output', '%s/{{workflow.name}}/roc' % output, '--predictions', predictions, @@ -130,7 +130,7 @@ def roc_op(predictions: 'GcsUri', output: 'GcsUri', step_name='roc'): def kubeflow_deploy_op(model: 'TensorFlow model', tf_server_name, step_name='deploy'): return dsl.ContainerOp( name = step_name, - image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:0.1.4', #TODO-release: update the release tag for the next release + image = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments = [ '--model-path', model, '--server-name', tf_server_name diff --git a/samples/xgboost-spark/xgboost-training-cm.py b/samples/xgboost-spark/xgboost-training-cm.py index 292d141faed..0e2b541ccea 100755 --- a/samples/xgboost-spark/xgboost-training-cm.py +++ b/samples/xgboost-spark/xgboost-training-cm.py @@ -26,7 +26,7 @@ class CreateClusterOp(dsl.ContainerOp): def __init__(self, name, project, region, staging): super(CreateClusterOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-create-cluster:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -41,7 +41,7 @@ class DeleteClusterOp(dsl.ContainerOp): def __init__(self, name, project, region): super(DeleteClusterOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-delete-cluster:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -55,7 +55,7 @@ class AnalyzeOp(dsl.ContainerOp): def __init__(self, name, project, region, cluster_name, schema, train_data, output): super(AnalyzeOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-analyze:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -73,7 +73,7 @@ def __init__(self, name, project, region, cluster_name, train_data, eval_data, target, analysis, output): super(TransformOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-transform:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -98,7 +98,7 @@ def __init__(self, name, project, region, cluster_name, train_data, eval_data, super(TrainerOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-train:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -121,7 +121,7 @@ class PredictOp(dsl.ContainerOp): def __init__(self, name, project, region, cluster_name, data, model, target, analysis, output): super(PredictOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-dataproc-predict:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--project', project, '--region', region, @@ -141,7 +141,7 @@ class ConfusionMatrixOp(dsl.ContainerOp): def __init__(self, name, predictions, output): super(ConfusionMatrixOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:0.1.4', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--output', output, '--predictions', predictions @@ -153,7 +153,7 @@ class RocOp(dsl.ContainerOp): def __init__(self, name, predictions, trueclass, output): super(RocOp, self).__init__( name=name, - image='gcr.io/ml-pipeline/ml-pipeline-local-roc:dev', #TODO-release: update the release tag for the next release + image='gcr.io/ml-pipeline/ml-pipeline-local-roc:85c6413a2e13da4b8f198aeac1abc2f3a74fe789', arguments=[ '--output', output, '--predictions', predictions, diff --git a/uninstaller.yaml b/uninstaller.yaml index 72354c7df3b..e9d82feb74e 100644 --- a/uninstaller.yaml +++ b/uninstaller.yaml @@ -24,7 +24,7 @@ spec: spec: containers: - name: uninstaller - image: gcr.io/ml-pipeline/bootstrapper:0.1.4 #TODO-release: update the release tag for the next release + image: gcr.io/ml-pipeline/bootstrapper:0.1.4 imagePullPolicy: 'Always' # Additional parameter available: args: [