diff --git a/components/google-cloud/storage/download/component.yaml b/components/google-cloud/storage/download/component.yaml new file mode 100644 index 000000000000..2ae6f5988ace --- /dev/null +++ b/components/google-cloud/storage/download/component.yaml @@ -0,0 +1,27 @@ +name: Download from GCS +inputs: +- {name: GCS path, type: URI} +outputs: +- {name: Data} +implementation: + container: + image: google/cloud-sdk + command: + - bash # Pattern comparison only works in Bash + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + + stat_result=$(gsutil ls -d "$0") + if [[ "$stat_result" == */ ]]; then + mkdir -p "$1" # When source path is a directory, gsutil requires the destination to also be a directory + else + mkdir -p "$(dirname "$1")" + fi + + gsutil -m rsync -r "$0" "$1" # gsutil cp has different path handling than Linux cp. It always puts the source directory (name) inside the destination directory. gsutil rsync does not have that problem. + - inputValue: GCS path + - outputPath: Data diff --git a/components/google-cloud/storage/download_blob/component.yaml b/components/google-cloud/storage/download_blob/component.yaml new file mode 100644 index 000000000000..24b8b4ff143a --- /dev/null +++ b/components/google-cloud/storage/download_blob/component.yaml @@ -0,0 +1,20 @@ +name: Download from GCS +inputs: +- {name: GCS path, type: URI} +outputs: +- {name: Data} +implementation: + container: + image: google/cloud-sdk + command: + - sh + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + mkdir -p "$(dirname "$1")" + gsutil -m cp -r "$0" "$1" + - inputValue: GCS path + - outputPath: Data diff --git a/components/google-cloud/storage/download_dir/component.yaml b/components/google-cloud/storage/download_dir/component.yaml new file mode 100644 index 000000000000..fb1da2153fab --- /dev/null +++ b/components/google-cloud/storage/download_dir/component.yaml @@ -0,0 +1,20 @@ +name: Download from GCS +inputs: +- {name: GCS path, type: URI} +outputs: +- {name: Data} +implementation: + container: + image: google/cloud-sdk + command: + - sh + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + mkdir -p "$1" + gsutil -m cp -r "$0" "$1" + - inputValue: GCS path + - outputPath: Data diff --git a/components/google-cloud/storage/list/component.yaml b/components/google-cloud/storage/list/component.yaml new file mode 100644 index 000000000000..1b7892fa9958 --- /dev/null +++ b/components/google-cloud/storage/list/component.yaml @@ -0,0 +1,20 @@ +name: List blobs +inputs: +- {name: GCS path, type: URI, description: 'GCS path for listing. For recursive listing use the "gs://bucket/path/**" syntax".'} +outputs: +- {name: Paths} +implementation: + container: + image: google/cloud-sdk + command: + - sh + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + mkdir -p "$(dirname "$1")" + gsutil ls "$0" > "$1" + - inputValue: GCS path + - outputPath: Paths diff --git a/components/google-cloud/storage/upload_to_explicit_uri/component.yaml b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml new file mode 100644 index 000000000000..d75aaaed57c1 --- /dev/null +++ b/components/google-cloud/storage/upload_to_explicit_uri/component.yaml @@ -0,0 +1,23 @@ +name: Upload to GCS +inputs: +- {name: Data} +- {name: GCS path, type: URI} +outputs: +- {name: GCS path, type: URI} +implementation: + container: + image: google/cloud-sdk + command: + - sh + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + gsutil cp -r "$0" "$1" + mkdir -p "$(dirname "$2")" + echo "$1" > "$2" + - inputPath: Data + - inputValue: GCS path + - outputPath: GCS path diff --git a/components/google-cloud/storage/upload_to_unique_uri/component.yaml b/components/google-cloud/storage/upload_to_unique_uri/component.yaml new file mode 100644 index 000000000000..bcbe0c29f597 --- /dev/null +++ b/components/google-cloud/storage/upload_to_unique_uri/component.yaml @@ -0,0 +1,24 @@ +name: Upload to GCS +description: Upload to GCS with unique URI suffix +inputs: +- {name: Data} +- {name: GCS path prefix, type: URI} +outputs: +- {name: GCS path, type: URI} +implementation: + container: + image: google/cloud-sdk + command: + - sh + - -ex + - -c + - | + if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + fi + gsutil cp -r "$0" "$1" + mkdir -p "$(dirname "$2")" + echo "$1" > "$2" + - inputPath: Data + - concat: [{inputValue: GCS path prefix}, '{{workflow.uid}}_{{pod.name}}'] + - outputPath: GCS path