From a77255843ad5fd46637bb7bcbfbc62e4e900e7a5 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 5 Nov 2018 14:02:42 -0800 Subject: [PATCH 1/4] publish python sdk in the cloud build --- .cloudbuild.yaml | 13 +++++++++++++ .release.cloudbuild.yaml | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index 8a736da028d..f83474c5b6f 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -58,6 +58,19 @@ steps: id: 'copyBootstrapperYAMLToLatest' waitFor: ['prepareBootstrapperYAML'] +# Build the Python SDK +- name: 'debian' + entrypoint: '/bin/bash' + args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk;get http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh ./kfp.tar.gz'] + id: 'preparePythonSDK' +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/sdk/python/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] + id: 'copyPythonSDK' + waitFor: ['preparePythonSDK'] +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/sdk/python/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz'] + id: 'copyPythonSDKToLatest' + waitFor: ['preparePythonSDK'] # Copy the uninstaller config - name: 'debian' diff --git a/.release.cloudbuild.yaml b/.release.cloudbuild.yaml index 9c266ea0423..6aa3c38870c 100644 --- a/.release.cloudbuild.yaml +++ b/.release.cloudbuild.yaml @@ -103,6 +103,19 @@ steps: id: 'copyUninstallerYAMLToLatest' waitFor: ['prepareUninstallerYAML'] +# Copy the Python SDK +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz', '/workspace/'] + id: 'copyPythonSDKLocal' +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/kfp.tar.gz', 'gs://ml-pipeline/release/$TAG_NAME/kfp.tar.gz'] + id: 'copyPythonSDK' + waitFor: ['copyPythonSDKLocal'] +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/kfp.tar.gz', 'gs://ml-pipeline/release/latest/kfp.tar.gz'] + id: 'copyPythonSDKToLatest' + waitFor: ['copyPythonSDKLocal'] + # Pull and retag the images for the Dataflow-based pipeline components - name: 'gcr.io/cloud-builders/docker' args: ['pull', 'gcr.io/$PROJECT_ID/ml-pipeline-dataflow-tf-predict:$COMMIT_SHA'] From 560725d67bd354b4af4da62a74bf5246a1930be7 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 5 Nov 2018 15:31:34 -0800 Subject: [PATCH 2/4] update cloudbuild --- .cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index f83474c5b6f..239383ef625 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -61,7 +61,7 @@ steps: # Build the Python SDK - name: 'debian' entrypoint: '/bin/bash' - args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk;get http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh ./kfp.tar.gz'] + args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh ./kfp.tar.gz'] id: 'preparePythonSDK' - name: 'gcr.io/cloud-builders/gsutil' args: ['cp', '/workspace/sdk/python/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] From 421cfe43f743f5154c5c27c1521082b19abcd732 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 5 Nov 2018 16:43:10 -0800 Subject: [PATCH 3/4] adjust output path --- .cloudbuild.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index 239383ef625..16d8beb42a5 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -61,14 +61,14 @@ steps: # Build the Python SDK - name: 'debian' entrypoint: '/bin/bash' - args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh ./kfp.tar.gz'] + args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh /workspace/kfp.tar.gz'] id: 'preparePythonSDK' - name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/sdk/python/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] + args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] id: 'copyPythonSDK' waitFor: ['preparePythonSDK'] - name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/sdk/python/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz'] + args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz'] id: 'copyPythonSDKToLatest' waitFor: ['preparePythonSDK'] From 2ed0dbbde719dfeab9c6862fca58aa191392ee96 Mon Sep 17 00:00:00 2001 From: Ning Gao Date: Mon, 5 Nov 2018 17:42:26 -0800 Subject: [PATCH 4/4] add dependency for buildapiserver --- .cloudbuild.yaml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml index 16d8beb42a5..1421ad9f23e 100644 --- a/.cloudbuild.yaml +++ b/.cloudbuild.yaml @@ -13,6 +13,21 @@ # limitations under the License. steps: + +# Build the Python SDK +- name: 'debian' + entrypoint: '/bin/bash' + args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh /workspace/kfp.tar.gz'] + id: 'preparePythonSDK' +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] + id: 'copyPythonSDK' + waitFor: ['preparePythonSDK'] +- name: 'gcr.io/cloud-builders/gsutil' + args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz'] + id: 'copyPythonSDKToLatest' + waitFor: ['preparePythonSDK'] + # Build the pipeline system images and bootstrapper - name: 'debian' entrypoint: '/bin/bash' @@ -29,6 +44,7 @@ steps: '--build-arg', 'COMMIT_SHA=$COMMIT_SHA', '-f', '/workspace/backend/Dockerfile', '/workspace'] id: 'buildApiServer' + waitFor: ['copyPythonSDK'] - name: 'gcr.io/cloud-builders/docker' args: ['build', '-t', 'gcr.io/$PROJECT_ID/scheduledworkflow:$COMMIT_SHA', '-f', '/workspace/backend/Dockerfile.scheduledworkflow', '/workspace'] @@ -58,20 +74,6 @@ steps: id: 'copyBootstrapperYAMLToLatest' waitFor: ['prepareBootstrapperYAML'] -# Build the Python SDK -- name: 'debian' - entrypoint: '/bin/bash' - args: ['-c', 'apt-get update -y; apt-get install --no-install-recommends -y -q default-jdk wget python python-setuptools; wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O /tmp/swagger-codegen-cli.jar;cd /workspace/sdk/python;./build.sh /workspace/kfp.tar.gz'] - id: 'preparePythonSDK' -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/$COMMIT_SHA/kfp.tar.gz'] - id: 'copyPythonSDK' - waitFor: ['preparePythonSDK'] -- name: 'gcr.io/cloud-builders/gsutil' - args: ['cp', '/workspace/kfp.tar.gz', 'gs://$PROJECT_ID/builds/latest/kfp.tar.gz'] - id: 'copyPythonSDKToLatest' - waitFor: ['preparePythonSDK'] - # Copy the uninstaller config - name: 'debian' entrypoint: '/bin/bash'