Components - Google Cloud Storage

components/google-cloud/storage/download/component.yaml

@@ -0,0 +1,27 @@
+name: Download from GCS
+inputs:
+- {name: GCS path, type: URI}
+outputs:
+- {name: Data}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - bash # Pattern comparison only works in Bash
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+
+            stat_result=$(gsutil ls -d "$0")
+            if [[ "$stat_result" == */ ]]; then
+                mkdir -p "$1" # When source path is a directory, gsutil requires the destination to also be a directory
+            else
+                mkdir -p "$(dirname "$1")"
+            fi
+
+            gsutil -m rsync -r "$0" "$1" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem.
+        - inputValue: GCS path
+        - outputPath: Data

components/google-cloud/storage/download_blob/component.yaml

@@ -0,0 +1,20 @@
+name: Download from GCS
+inputs:
+- {name: GCS path, type: URI}
+outputs:
+- {name: Data}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - sh
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+            mkdir -p "$(dirname "$1")"
+            gsutil -m cp -r "$0" "$1"
+        - inputValue: GCS path
+        - outputPath: Data

components/google-cloud/storage/download_dir/component.yaml

@@ -0,0 +1,20 @@
+name: Download from GCS
+inputs:
+- {name: GCS path, type: URI}
+outputs:
+- {name: Data}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - sh
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+            mkdir -p "$1"
+            gsutil -m cp -r "$0" "$1"
+        - inputValue: GCS path
+        - outputPath: Data

components/google-cloud/storage/list/component.yaml

@@ -0,0 +1,20 @@
+name: List blobs
+inputs:
+- {name: GCS path, type: URI, description: 'GCS path for listing. For recursive listing use the "gs://bucket/path/**" syntax".'}
+outputs:
+- {name: Paths}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - sh
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+            mkdir -p "$(dirname "$1")"
+            gsutil ls "$0" > "$1"
+        - inputValue: GCS path
+        - outputPath: Paths

components/google-cloud/storage/upload_to_explicit_uri/component.yaml

@@ -0,0 +1,23 @@
+name: Upload to GCS
+inputs:
+- {name: Data}
+- {name: GCS path, type: URI}
+outputs:
+- {name: GCS path, type: URI}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - sh
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+            gsutil cp -r "$0" "$1"
+            mkdir -p "$(dirname "$2")"
+            echo "$1" > "$2"
+        - inputPath: Data
+        - inputValue: GCS path
+        - outputPath: GCS path

components/google-cloud/storage/upload_to_unique_uri/component.yaml

@@ -0,0 +1,24 @@
+name: Upload to GCS
+description: Upload to GCS with unique URI suffix
+inputs:
+- {name: Data}
+- {name: GCS path prefix, type: URI}
+outputs:
+- {name: GCS path, type: URI}
+implementation:
+    container:
+        image: google/cloud-sdk
+        command:
+        - sh
+        - -ex
+        - -c
+        - |
+            if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then
+                gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+            fi
+            gsutil cp -r "$0" "$1"
+            mkdir -p "$(dirname "$2")"
+            echo "$1" > "$2"
+        - inputPath: Data
+        - concat: [{inputValue: GCS path prefix}, '{{workflow.uid}}_{{pod.name}}']
+        - outputPath: GCS path

release/RELEASE.md

@@ -1,6 +1,9 @@
 # Current version under development
 * Added a release note.
 ## Major features and improvements
+### Components:
+
+* Added [Google Cloud Storage components](https://github.com/kubeflow/pipelines/tree/290fa55fb9c908be38cbe6bc4c3f477da6b0e97f/components/google-cloud/storage): [Download](https://github.com/kubeflow/pipelines/tree/290fa55fb9c908be38cbe6bc4c3f477da6b0e97f/components/google-cloud/storage/download), [Upload to explicit URI](https://github.com/kubeflow/pipelines/tree/290fa55fb9c908be38cbe6bc4c3f477da6b0e97f/components/google-cloud/storage/upload_to_explicit_uri), [Upload to unique URI](https://github.com/kubeflow/pipelines/tree/290fa55fb9c908be38cbe6bc4c3f477da6b0e97f/components/google-cloud/storage/upload_to_unique_uri) and [List](https://github.com/kubeflow/pipelines/tree/290fa55fb9c908be38cbe6bc4c3f477da6b0e97f/components/google-cloud/storage/list).
 
 ## Bug fixes and other changes