From a9355ddf41e8131090313a021287894705f7dd31 Mon Sep 17 00:00:00 2001 From: Christoph Deppisch Date: Thu, 9 Nov 2023 09:42:54 +0100 Subject: [PATCH] chore: Add forcePathStyle option to aws-s3 Kamelets - Since Apache Camel 4 the aws-s3 component supports to force the path style when accessing buckets on AWS S3 - Path style access is essential when testing with local AWS S3 instances such as LocalStack --- kamelets/aws-s3-sink.kamelet.yaml | 10 +++++++++- kamelets/aws-s3-source.kamelet.yaml | 8 ++++++++ kamelets/aws-s3-streaming-upload-sink.kamelet.yaml | 10 +++++++++- .../main/resources/kamelets/aws-s3-sink.kamelet.yaml | 10 +++++++++- .../main/resources/kamelets/aws-s3-source.kamelet.yaml | 8 ++++++++ .../kamelets/aws-s3-streaming-upload-sink.kamelet.yaml | 10 +++++++++- test/aws-s3/amazonS3Client.groovy | 3 ++- test/aws-s3/aws-s3-to-http.yaml | 1 + test/aws-s3/aws-s3-to-knative-broker.yaml | 1 + test/aws-s3/aws-s3-to-knative-channel.yaml | 1 + test/aws-s3/aws-s3-uri-pipe.yaml | 1 + 11 files changed, 58 insertions(+), 5 deletions(-) diff --git a/kamelets/aws-s3-sink.kamelet.yaml b/kamelets/aws-s3-sink.kamelet.yaml index 3e07c1567..9141f5a7c 100644 --- a/kamelets/aws-s3-sink.kamelet.yaml +++ b/kamelets/aws-s3-sink.kamelet.yaml @@ -96,6 +96,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false keyName: title: Key Name description: The key name for saving an element in the bucket. @@ -113,7 +120,7 @@ spec: when: - simple: '${propertiesExist:!keyName}' steps: - - choice: + - choice: when: - simple: "${header[file]}" steps: @@ -140,4 +147,5 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" keyName: "{{?keyName}}" diff --git a/kamelets/aws-s3-source.kamelet.yaml b/kamelets/aws-s3-source.kamelet.yaml index 6c09eb8ac..d3cceebac 100644 --- a/kamelets/aws-s3-source.kamelet.yaml +++ b/kamelets/aws-s3-source.kamelet.yaml @@ -96,6 +96,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false delay: title: Delay description: The number of milliseconds before the next poll of the selected bucket. @@ -188,6 +195,7 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" delay: "{{delay}}" maxMessagesPerPoll: "{{maxMessagesPerPoll}}" steps: diff --git a/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml b/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml index fcaad8dde..3570272d4 100644 --- a/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml +++ b/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml @@ -99,7 +99,7 @@ spec: description: The naming strategy to use in streaming upload mode. There are 2 enums and the value can be one of progressive, random type: string default: "progressive" - keyName: + keyName: title: Key Name description: Setting the key name for an element in the bucket through endpoint parameter. In Streaming Upload, with the default configuration, this will be the base for the progressive creation of files. type: string @@ -121,6 +121,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false dependencies: - "camel:aws2-s3" - "camel:kamelet" @@ -145,3 +152,4 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" diff --git a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-sink.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-sink.kamelet.yaml index 3e07c1567..9141f5a7c 100644 --- a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-sink.kamelet.yaml +++ b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-sink.kamelet.yaml @@ -96,6 +96,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false keyName: title: Key Name description: The key name for saving an element in the bucket. @@ -113,7 +120,7 @@ spec: when: - simple: '${propertiesExist:!keyName}' steps: - - choice: + - choice: when: - simple: "${header[file]}" steps: @@ -140,4 +147,5 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" keyName: "{{?keyName}}" diff --git a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-source.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-source.kamelet.yaml index 6c09eb8ac..d3cceebac 100644 --- a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-source.kamelet.yaml +++ b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-source.kamelet.yaml @@ -96,6 +96,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false delay: title: Delay description: The number of milliseconds before the next poll of the selected bucket. @@ -188,6 +195,7 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" delay: "{{delay}}" maxMessagesPerPoll: "{{maxMessagesPerPoll}}" steps: diff --git a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml index fcaad8dde..3570272d4 100644 --- a/library/camel-kamelets/src/main/resources/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml +++ b/library/camel-kamelets/src/main/resources/kamelets/aws-s3-streaming-upload-sink.kamelet.yaml @@ -99,7 +99,7 @@ spec: description: The naming strategy to use in streaming upload mode. There are 2 enums and the value can be one of progressive, random type: string default: "progressive" - keyName: + keyName: title: Key Name description: Setting the key name for an element in the bucket through endpoint parameter. In Streaming Upload, with the default configuration, this will be the base for the progressive creation of files. type: string @@ -121,6 +121,13 @@ spec: x-descriptors: - 'urn:alm:descriptor:com.tectonic.ui:checkbox' default: false + forcePathStyle: + title: Force Path Style + description: Forces path style when accessing AWS S3 buckets. + type: boolean + x-descriptors: + - 'urn:alm:descriptor:com.tectonic.ui:checkbox' + default: false dependencies: - "camel:aws2-s3" - "camel:kamelet" @@ -145,3 +152,4 @@ spec: useDefaultCredentialsProvider: "{{useDefaultCredentialsProvider}}" uriEndpointOverride: "{{?uriEndpointOverride}}" overrideEndpoint: "{{overrideEndpoint}}" + forcePathStyle: "{{forcePathStyle}}" diff --git a/test/aws-s3/amazonS3Client.groovy b/test/aws-s3/amazonS3Client.groovy index 7e31c675c..5a8c5f0d8 100644 --- a/test/aws-s3/amazonS3Client.groovy +++ b/test/aws-s3/amazonS3Client.groovy @@ -22,12 +22,13 @@ import software.amazon.awssdk.services.s3.S3Client S3Client s3 = S3Client .builder() - .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_LOCAL_URL}".replace('localhost', '127.0.0.1'))) + .endpointOverride(URI.create("${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL}")) .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create( "${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY}", "${YAKS_TESTCONTAINERS_LOCALSTACK_SECRET_KEY}") )) + .forcePathStyle(true) .region(Region.of("${YAKS_TESTCONTAINERS_LOCALSTACK_REGION}")) .build() diff --git a/test/aws-s3/aws-s3-to-http.yaml b/test/aws-s3/aws-s3-to-http.yaml index eea7cf7b4..702f87181 100644 --- a/test/aws-s3/aws-s3-to-http.yaml +++ b/test/aws-s3/aws-s3-to-http.yaml @@ -31,6 +31,7 @@ spec: properties: bucketNameOrArn: ${aws.s3.bucketNameOrArn} overrideEndpoint: true + forcePathStyle: true uriEndpointOverride: ${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL} accessKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY} secretKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_SECRET_KEY} diff --git a/test/aws-s3/aws-s3-to-knative-broker.yaml b/test/aws-s3/aws-s3-to-knative-broker.yaml index 4ee1d92dc..7ecacdfb1 100644 --- a/test/aws-s3/aws-s3-to-knative-broker.yaml +++ b/test/aws-s3/aws-s3-to-knative-broker.yaml @@ -28,6 +28,7 @@ spec: properties: bucketNameOrArn: ${aws.s3.bucketNameOrArn} overrideEndpoint: true + forcePathStyle: true uriEndpointOverride: ${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL} accessKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY} secretKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_SECRET_KEY} diff --git a/test/aws-s3/aws-s3-to-knative-channel.yaml b/test/aws-s3/aws-s3-to-knative-channel.yaml index 4d0f246c9..6314a5968 100644 --- a/test/aws-s3/aws-s3-to-knative-channel.yaml +++ b/test/aws-s3/aws-s3-to-knative-channel.yaml @@ -28,6 +28,7 @@ spec: properties: bucketNameOrArn: ${aws.s3.bucketNameOrArn} overrideEndpoint: true + forcePathStyle: true uriEndpointOverride: ${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL} accessKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY} secretKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_SECRET_KEY} diff --git a/test/aws-s3/aws-s3-uri-pipe.yaml b/test/aws-s3/aws-s3-uri-pipe.yaml index 1827a1258..4638a908c 100644 --- a/test/aws-s3/aws-s3-uri-pipe.yaml +++ b/test/aws-s3/aws-s3-uri-pipe.yaml @@ -28,6 +28,7 @@ spec: properties: bucketNameOrArn: ${aws.s3.bucketNameOrArn} overrideEndpoint: true + forcePathStyle: true uriEndpointOverride: ${YAKS_TESTCONTAINERS_LOCALSTACK_S3_URL} accessKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_ACCESS_KEY} secretKey: ${YAKS_TESTCONTAINERS_LOCALSTACK_SECRET_KEY}