From 7abfc8fefba4a366580eb329c32a30e28e8cc678 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Sat, 14 Aug 2021 10:29:27 -0400 Subject: [PATCH 1/9] Adding in include and exclude options for aws-s3-deployment --- packages/@aws-cdk/aws-s3-deployment/README.md | 25 ++++++ .../lib/bucket-deployment.ts | 18 ++++ .../aws-s3-deployment/lib/lambda/index.py | 12 ++- .../test/bucket-deployment.test.ts | 56 ++++++++++++ .../aws-s3-deployment/test/lambda/aws | 9 +- .../aws-s3-deployment/test/lambda/test.py | 86 +++++++++++++++++++ 6 files changed, 203 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index a896d55cfc575..f0e689031baa3 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -138,6 +138,31 @@ new BucketDeployment(this, 'HTMLBucketDeployment', { }); ``` +## Exclude and Include Filters + +When deploying your files to the destination bucket, you can specify exclude and include filters. These files will not be deployed to the destination bucket. In addition, if the file already exists in the destination bucket, it will not be deleted if you are using the `prune` option: + +```ts +new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket, + exclude: '*.txt' +}); +``` + +This can also be leveraged to exclude entire directories. In some cases, you may want to exclude all files from a directory while adding back in a smaller number of files from that directory. In those cases, you can use the `include` option alongside the `exclude` option. + +```ts +new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFilesWithInclusionFilter', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket, + exclude: 'build/*', + include: 'build/config.json' +}); +``` + +These filters follow the same format that is used for the AWS CLI. See the CLI documentation for information on [Using Include and Exclude Filters](https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters). + ## Objects metadata You can specify metadata to be set on all the objects in your deployment. diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 990a1ec359d0b..6ade94923bdad 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -35,6 +35,22 @@ export interface BucketDeploymentProps { */ readonly destinationKeyPrefix?: string; + /** + * If this is set, matching files or objects will be excluded from the command + * + * @default - No files are excluded + * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters + */ + readonly exclude?: string + + /** + * If this is set, matching files or objects will be included with the command + * + * @default - No files are excluded + * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters + */ + readonly include?: string + /** * If this is set to false, files in the destination bucket that * do not exist in the asset, will NOT be deleted during deployment (create/update). @@ -245,6 +261,8 @@ export class BucketDeployment extends CoreConstruct { DestinationBucketKeyPrefix: props.destinationKeyPrefix, RetainOnDelete: props.retainOnDelete, Prune: props.prune ?? true, + Exclude: props.exclude, + Include: props.include, UserMetadata: props.metadata ? mapUserMetadata(props.metadata) : undefined, SystemMetadata: mapSystemMetadata(props), DistributionId: props.distribution?.distributionId, diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 3935e3122529d..6d1cc4deffaa7 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -49,6 +49,8 @@ def cfn_error(message=None): user_metadata = props.get('UserMetadata', {}) system_metadata = props.get('SystemMetadata', {}) prune = props.get('Prune', 'true').lower() == 'true' + exclude = props.get('Exclude', '') + include = props.get('Include', '') default_distribution_path = dest_bucket_prefix if not default_distribution_path.endswith("/"): @@ -100,7 +102,7 @@ def cfn_error(message=None): aws_command("s3", "rm", old_s3_dest, "--recursive") if request_type == "Update" or request_type == "Create": - s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune) + s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include) if distribution_id: cloudfront_invalidate(distribution_id, distribution_paths) @@ -114,7 +116,7 @@ def cfn_error(message=None): #--------------------------------------------------------------------------------------------------- # populate all files from s3_source_zips to a destination bucket -def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune): +def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include): # create a temporary working directory workdir=tempfile.mkdtemp() logger.info("| workdir: %s" % workdir) @@ -139,6 +141,12 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune): if prune: s3_command.append("--delete") + if exclude: + s3_command.extend(["--exclude", exclude]) + + if include: + s3_command.extend(["--include", include]) + s3_command.extend([contents_dir, s3_dest]) s3_command.extend(create_metadata_args(user_metadata, system_metadata)) aws_command(*s3_command) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index 893c5ae03b35f..fc0bc1894954d 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -689,6 +689,62 @@ test('deploy without deleting missing files from destination', () => { }); }); +test('deploy with excluded files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: 'sample.js', + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: 'sample.js', + }); +}); + +test('deploy with included files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + include: 'sample.js', + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Include: 'sample.js', + }); +}); + +test('deploy with excluded and included files from destination', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: 'sample/*', + include: 'sample/include.json', + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: 'sample/*', + Include: 'sample/include.json', + }); +}); + test('deployment allows vpc to be implicitly supplied to lambda', () => { // GIVEN diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws index 969bb982cd08c..eb45da79af91d 100755 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws @@ -21,7 +21,14 @@ if sys.argv[2] == "cp" and not sys.argv[4].startswith("s3://"): sys.argv[4] = "archive.zip" if sys.argv[2] == "sync": - sys.argv[4 if '--delete' in sys.argv else 3] = "contents.zip" + contentsIdx = 3 + if '--include' in sys.argv: + contentsIdx += 2 + if '--exclude' in sys.argv: + contentsIdx += 2 + if '--delete' in sys.argv: + contentsIdx += 1 + sys.argv[contentsIdx] = "contents.zip" with open("./aws.out", "a") as myfile: myfile.write(json.dumps(sys.argv[1:]) + "\n") diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index 7813a13db3859..cc2c23df3cffe 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -68,6 +68,92 @@ def test_update_no_delete(self): ["s3", "sync", "contents.zip", "s3:///"] ) + def test_create_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": "sample.json" + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "sample.json", "contents.zip", "s3:///"] + ) + + def test_update_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": "sample.json" + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "sample.json", "contents.zip", "s3:///"] + ) + + def test_create_include(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Include": "/sample/*.json" + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_update_include(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Include": "/sample/*.json" + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_create_include_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": "/sample/*", + "Include": "/sample/*.json" + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + + def test_update_include_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": "/sample/*", + "Include": "/sample/*.json" + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--include", "/sample/*.json", "contents.zip", "s3:///"] + ) + def test_create_update_multiple_sources(self): invoke_handler("Create", { "SourceBucketNames": ["", ""], From 9898937a9d4b0e8784e0de8c07567c45b1fa05c0 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Sat, 14 Aug 2021 11:05:04 -0400 Subject: [PATCH 2/9] Adding in integration test updated JSON --- ...bucket-deployment-cloudfront.expected.json | 20 +++++++++---------- .../integ.bucket-deployment.expected.json | 20 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 76f906941f2bd..96ab4b64b9da5 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -295,7 +295,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" }, "S3Key": { "Fn::Join": [ @@ -308,7 +308,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -321,7 +321,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -365,17 +365,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", @@ -390,4 +390,4 @@ "Description": "Artifact hash for asset \"fc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222e\"" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 877298773816b..68c8b6e811e03 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -304,7 +304,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" }, "S3Key": { "Fn::Join": [ @@ -317,7 +317,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -330,7 +330,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -700,17 +700,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", @@ -725,4 +725,4 @@ "Description": "Artifact hash for asset \"fc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222e\"" } } -} +} \ No newline at end of file From fcaac07f9e72322b374c32db66bb119afc940f84 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Sat, 14 Aug 2021 18:14:11 -0400 Subject: [PATCH 3/9] Updating integration tests for aws-ecs --- .../ec2/integ.environment-file.expected.json | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json index c309c8a70f249..845c4465ec46a 100644 --- a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json +++ b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json @@ -108,15 +108,15 @@ "VpcPublicSubnet1NATGateway4D7517AA": { "Type": "AWS::EC2::NatGateway", "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet1Subnet5C2D37C4" + }, "AllocationId": { "Fn::GetAtt": [ "VpcPublicSubnet1EIPD7E02669", "AllocationId" ] }, - "SubnetId": { - "Ref": "VpcPublicSubnet1Subnet5C2D37C4" - }, "Tags": [ { "Key": "Name", @@ -205,15 +205,15 @@ "VpcPublicSubnet2NATGateway9182C01D": { "Type": "AWS::EC2::NatGateway", "Properties": { + "SubnetId": { + "Ref": "VpcPublicSubnet2Subnet691E08A3" + }, "AllocationId": { "Fn::GetAtt": [ "VpcPublicSubnet2EIP3C605A87", "AllocationId" ] }, - "SubnetId": { - "Ref": "VpcPublicSubnet2Subnet691E08A3" - }, "Tags": [ { "Key": "Name", @@ -1219,7 +1219,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" }, "S3Key": { "Fn::Join": [ @@ -1232,7 +1232,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -1245,7 +1245,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70" + "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" } ] } @@ -1348,17 +1348,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3Bucket55EFA30C": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { "Type": "String", - "Description": "S3 bucket for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfS3VersionKey60329B70": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { "Type": "String", - "Description": "S3 key for asset version \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, - "AssetParametersc24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cfArtifactHash85F58E48": { + "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { "Type": "String", - "Description": "Artifact hash for asset \"c24b999656e4fe6c609c31bae56a1cf4717a405619c3aa6ba1bc686b8c2c86cf\"" + "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" }, "AssetParameters972240f9dd6e036a93d5f081af9a24315b2053828ac049b3b19b2fa12d7ae64aS3Bucket1F1A8472": { "Type": "String", @@ -1385,4 +1385,4 @@ "Description": "Artifact hash for asset \"872561bf078edd1685d50c9ff821cdd60d2b2ddfb0013c4087e79bf2bb50724d\"" } } -} +} \ No newline at end of file From 04e09aaf9d4257e31d36ff75ef81624d151782e1 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Tue, 17 Aug 2021 05:32:35 -0400 Subject: [PATCH 4/9] Including support for multiple include and exclude filters --- .../lib/bucket-deployment.ts | 14 +++--- .../aws-s3-deployment/lib/lambda/index.py | 10 ++-- .../test/bucket-deployment.test.ts | 36 +++++++++++---- .../aws-s3-deployment/test/lambda/aws | 4 +- .../aws-s3-deployment/test/lambda/test.py | 46 +++++++++++++++---- 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 6ade94923bdad..5dc9bb9918c05 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -36,20 +36,22 @@ export interface BucketDeploymentProps { readonly destinationKeyPrefix?: string; /** - * If this is set, matching files or objects will be excluded from the command + * If this is set, matching files or objects will be excluded from the deployment. If + * no include or exclude filters are used, all files will be included with the deployment. * - * @default - No files are excluded + * @default - No exclude filters are included * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters */ - readonly exclude?: string + readonly exclude?: string[] /** - * If this is set, matching files or objects will be included with the command + * If this is set, matching files or objects will be included with the deployment. If + * no include or exclude filters are used, all files will be included with the deployment. * - * @default - No files are excluded + * @default - No include filters are included * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters */ - readonly include?: string + readonly include?: string[] /** * If this is set to false, files in the destination bucket that diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py index 6d1cc4deffaa7..ed7e1d9c5d7e9 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py +++ b/packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py @@ -49,8 +49,8 @@ def cfn_error(message=None): user_metadata = props.get('UserMetadata', {}) system_metadata = props.get('SystemMetadata', {}) prune = props.get('Prune', 'true').lower() == 'true' - exclude = props.get('Exclude', '') - include = props.get('Include', '') + exclude = props.get('Exclude', []) + include = props.get('Include', []) default_distribution_path = dest_bucket_prefix if not default_distribution_path.endswith("/"): @@ -142,10 +142,12 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex s3_command.append("--delete") if exclude: - s3_command.extend(["--exclude", exclude]) + for filter in exclude: + s3_command.extend(["--exclude", filter]) if include: - s3_command.extend(["--include", include]) + for filter in include: + s3_command.extend(["--include", filter]) s3_command.extend([contents_dir, s3_dest]) s3_command.extend(create_metadata_args(user_metadata, system_metadata)) diff --git a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts index fc0bc1894954d..704f6378b8313 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.test.ts @@ -699,11 +699,11 @@ test('deploy with excluded files from destination', () => { new s3deploy.BucketDeployment(stack, 'Deploy', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket: bucket, - exclude: 'sample.js', + exclude: ['sample.js'], }); expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { - Exclude: 'sample.js', + Exclude: ['sample.js'], }); }); @@ -717,11 +717,11 @@ test('deploy with included files from destination', () => { new s3deploy.BucketDeployment(stack, 'Deploy', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket: bucket, - include: 'sample.js', + include: ['sample.js'], }); expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { - Include: 'sample.js', + Include: ['sample.js'], }); }); @@ -735,13 +735,33 @@ test('deploy with excluded and included files from destination', () => { new s3deploy.BucketDeployment(stack, 'Deploy', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket: bucket, - exclude: 'sample/*', - include: 'sample/include.json', + exclude: ['sample/*'], + include: ['sample/include.json'], }); expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { - Exclude: 'sample/*', - Include: 'sample/include.json', + Exclude: ['sample/*'], + Include: ['sample/include.json'], + }); +}); + +test('deploy with multiple exclude and include filters', () => { + + // GIVEN + const stack = new cdk.Stack(); + const bucket = new s3.Bucket(stack, 'Dest'); + + // WHEN + new s3deploy.BucketDeployment(stack, 'Deploy', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket: bucket, + exclude: ['sample/*', 'another/*'], + include: ['sample/include.json', 'another/include.json'], + }); + + expect(stack).toHaveResourceLike('Custom::CDKBucketDeployment', { + Exclude: ['sample/*', 'another/*'], + Include: ['sample/include.json', 'another/include.json'], }); }); diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws index eb45da79af91d..5cfd9275eeb71 100755 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/aws @@ -23,9 +23,9 @@ if sys.argv[2] == "cp" and not sys.argv[4].startswith("s3://"): if sys.argv[2] == "sync": contentsIdx = 3 if '--include' in sys.argv: - contentsIdx += 2 + contentsIdx += sys.argv.count("--include") * 2 if '--exclude' in sys.argv: - contentsIdx += 2 + contentsIdx += sys.argv.count("--exclude") * 2 if '--delete' in sys.argv: contentsIdx += 1 sys.argv[contentsIdx] = "contents.zip" diff --git a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py index cc2c23df3cffe..c7a371fafa01e 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py +++ b/packages/@aws-cdk/aws-s3-deployment/test/lambda/test.py @@ -73,7 +73,7 @@ def test_create_exclude(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Exclude": "sample.json" + "Exclude": ["sample.json"] }) self.assertAwsCommands( @@ -86,7 +86,7 @@ def test_update_exclude(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Exclude": "sample.json" + "Exclude": ["sample.json"] }, old_resource_props={ "DestinationBucketName": "", }, physical_id="") @@ -101,7 +101,7 @@ def test_create_include(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Include": "/sample/*.json" + "Include": ["/sample/*.json"] }) self.assertAwsCommands( @@ -114,7 +114,7 @@ def test_update_include(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Include": "/sample/*.json" + "Include": ["/sample/*.json"] }, old_resource_props={ "DestinationBucketName": "", }, physical_id="") @@ -129,8 +129,8 @@ def test_create_include_exclude(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Exclude": "/sample/*", - "Include": "/sample/*.json" + "Exclude": ["/sample/*"], + "Include": ["/sample/*.json"] }) self.assertAwsCommands( @@ -143,8 +143,8 @@ def test_update_include_exclude(self): "SourceBucketNames": [""], "SourceObjectKeys": [""], "DestinationBucketName": "", - "Exclude": "/sample/*", - "Include": "/sample/*.json" + "Exclude": ["/sample/*"], + "Include": ["/sample/*.json"] }, old_resource_props={ "DestinationBucketName": "", }, physical_id="") @@ -154,6 +154,36 @@ def test_update_include_exclude(self): ["s3", "sync", "--delete", "--exclude", "/sample/*", "--include", "/sample/*.json", "contents.zip", "s3:///"] ) + def test_create_multiple_include_exclude(self): + invoke_handler("Create", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*", "/another/*"], + "Include": ["/sample/*.json", "/another/*.json"] + }) + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--exclude", "/another/*", "--include", "/sample/*.json", "--include", "/another/*.json", "contents.zip", "s3:///"] + ) + + def test_update_multiple_include_exclude(self): + invoke_handler("Update", { + "SourceBucketNames": [""], + "SourceObjectKeys": [""], + "DestinationBucketName": "", + "Exclude": ["/sample/*", "/another/*"], + "Include": ["/sample/*.json", "/another/*.json"] + }, old_resource_props={ + "DestinationBucketName": "", + }, physical_id="") + + self.assertAwsCommands( + ["s3", "cp", "s3:///", "archive.zip"], + ["s3", "sync", "--delete", "--exclude", "/sample/*", "--exclude", "/another/*", "--include", "/sample/*.json", "--include", "/another/*.json", "contents.zip", "s3:///"] + ) + def test_create_update_multiple_sources(self): invoke_handler("Create", { "SourceBucketNames": ["", ""], From 90acb7e2b9a80119e49c3ad942d89758d7d18900 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Tue, 17 Aug 2021 06:38:43 -0400 Subject: [PATCH 5/9] Updating integration tests and expected JSON --- .../ec2/integ.environment-file.expected.json | 18 +-- ...bucket-deployment-cloudfront.expected.json | 18 +-- .../integ.bucket-deployment.expected.json | 123 ++++++++++++++++-- .../test/integ.bucket-deployment.ts | 7 + 4 files changed, 139 insertions(+), 27 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json index 845c4465ec46a..dc3aad690d805 100644 --- a/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json +++ b/packages/@aws-cdk/aws-ecs/test/ec2/integ.environment-file.expected.json @@ -1219,7 +1219,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -1232,7 +1232,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -1245,7 +1245,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -1348,17 +1348,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParameters972240f9dd6e036a93d5f081af9a24315b2053828ac049b3b19b2fa12d7ae64aS3Bucket1F1A8472": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json index 96ab4b64b9da5..a0d375ee6e3a1 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment-cloudfront.expected.json @@ -295,7 +295,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -308,7 +308,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -321,7 +321,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -365,17 +365,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json index 68c8b6e811e03..320bf196acb7b 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.expected.json @@ -304,7 +304,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A" }, "S3Key": { "Fn::Join": [ @@ -317,7 +317,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -330,7 +330,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477" + "Ref": "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3" } ] } @@ -685,6 +685,111 @@ }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" + }, + "DeployMeWithExcludedFilesOnDestinationAwsCliLayer68F5E11D": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "S3Bucket": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + } + ] + ] + } + }, + "Description": "/opt/awscli/aws" + } + }, + "DeployMeWithExcludedFilesOnDestinationCustomResource48D69581": { + "Type": "Custom::CDKBucketDeployment", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomCDKBucketDeployment8693BB64968944B69AAFB0CC9EB8756C81C01536", + "Arn" + ] + }, + "SourceBucketNames": [ + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A" + } + ], + "SourceObjectKeys": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3VersionKeyA58D380C" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3VersionKeyA58D380C" + } + ] + } + ] + } + ] + ] + } + ], + "DestinationBucketName": { + "Ref": "Destination920A3C57" + }, + "RetainOnDelete": false, + "Prune": true, + "Exclude": [ + "*.gif" + ] + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" } }, "Parameters": { @@ -700,17 +805,17 @@ "Type": "String", "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3BucketED84C949": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3BucketD8D20B9A": { "Type": "String", - "Description": "S3 bucket for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 bucket for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48S3VersionKeyDE5C6477": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaS3VersionKeyCD2774D3": { "Type": "String", - "Description": "S3 key for asset version \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "S3 key for asset version \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, - "AssetParameters2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48ArtifactHash59C0A5BF": { + "AssetParameters1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24eaArtifactHash3943F7F3": { "Type": "String", - "Description": "Artifact hash for asset \"2eac0752d10844050dc9bb7542ad2dcb11bbf8a9b8a589814284462cd52d8f48\"" + "Description": "Artifact hash for asset \"1f6de40da10b415b255c07df709f791e772ffb9f7bdd14ad81fb75643aad24ea\"" }, "AssetParametersfc4481abf279255619ff7418faa5d24456fef3432ea0da59c95542578ff0222eS3Bucket9CD8B20A": { "Type": "String", diff --git a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts index b2b1754083bd7..b6f20f62e6024 100644 --- a/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/test/integ.bucket-deployment.ts @@ -46,6 +46,13 @@ class TestBucketDeployment extends cdk.Stack { retainOnDelete: false, }); + new s3deploy.BucketDeployment(this, 'DeployMeWithExcludedFilesOnDestination', { + sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], + destinationBucket, + exclude: ['*.gif'], + retainOnDelete: false, + }); + } } From 05c4482244c0f3d76937e32ab42bce5d891b01e9 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 17 Aug 2021 14:16:28 +0300 Subject: [PATCH 6/9] Update packages/@aws-cdk/aws-s3-deployment/README.md --- packages/@aws-cdk/aws-s3-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index f0e689031baa3..90b96f5569a9d 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -146,7 +146,7 @@ When deploying your files to the destination bucket, you can specify exclude and new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket, - exclude: '*.txt' + exclude: ['*.txt'] }); ``` From dbd3e0ecb00354f8ef33c4f73d3d19dcad358c13 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Tue, 17 Aug 2021 14:16:36 +0300 Subject: [PATCH 7/9] Update packages/@aws-cdk/aws-s3-deployment/README.md --- packages/@aws-cdk/aws-s3-deployment/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 90b96f5569a9d..8c584fa3d3a8d 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -156,8 +156,8 @@ This can also be leveraged to exclude entire directories. In some cases, you ma new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFilesWithInclusionFilter', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket, - exclude: 'build/*', - include: 'build/config.json' + exclude: ['build/*'], + include: ['build/config.json'] }); ``` From 7feac33bf58d6c128cd7cb3434412a8652585e80 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Tue, 17 Aug 2021 07:17:44 -0400 Subject: [PATCH 8/9] Updating README --- packages/@aws-cdk/aws-s3-deployment/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index f0e689031baa3..5285fde5750eb 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -146,7 +146,7 @@ When deploying your files to the destination bucket, you can specify exclude and new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket, - exclude: '*.txt' + exclude: ['*.txt'] }); ``` @@ -156,8 +156,8 @@ This can also be leveraged to exclude entire directories. In some cases, you ma new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFilesWithInclusionFilter', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket, - exclude: 'build/*', - include: 'build/config.json' + exclude: ['build/*', 'test/*'], + include: ['build/config.json'] }); ``` From f558345c404d7857fa7bfce8cb3a5ee082277603 Mon Sep 17 00:00:00 2001 From: David Tucker Date: Thu, 19 Aug 2021 09:45:02 -0400 Subject: [PATCH 9/9] Updating documentation for include and exclude filters --- packages/@aws-cdk/aws-s3-deployment/README.md | 16 +++++++--------- .../aws-s3-deployment/lib/bucket-deployment.ts | 17 +++++++++++------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/aws-s3-deployment/README.md b/packages/@aws-cdk/aws-s3-deployment/README.md index 8c584fa3d3a8d..74b31d98c41fd 100644 --- a/packages/@aws-cdk/aws-s3-deployment/README.md +++ b/packages/@aws-cdk/aws-s3-deployment/README.md @@ -140,24 +140,22 @@ new BucketDeployment(this, 'HTMLBucketDeployment', { ## Exclude and Include Filters -When deploying your files to the destination bucket, you can specify exclude and include filters. These files will not be deployed to the destination bucket. In addition, if the file already exists in the destination bucket, it will not be deleted if you are using the `prune` option: +There are two points at which filters are evaluated in a deployment: asset bundling and the actual deployment. If you simply want to exclude files in the asset bundling process, you should leverage the `exclude` property of `AssetOptions` when defining your source: ```ts -new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { - sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], - destinationBucket, - exclude: ['*.txt'] +new BucketDeployment(this, 'HTMLBucketDeployment', { + sources: [Source.asset('./website', { exclude: ['*', '!index.html'] })], + destinationBucket: bucket, }); ``` -This can also be leveraged to exclude entire directories. In some cases, you may want to exclude all files from a directory while adding back in a smaller number of files from that directory. In those cases, you can use the `include` option alongside the `exclude` option. +If you want to specify filters to be used in the deployment process, you can use the `exclude` and `include` filters on `BucketDeployment`. If excluded, these files will not be deployed to the destination bucket. In addition, if the file already exists in the destination bucket, it will not be deleted if you are using the `prune` option: ```ts -new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFilesWithInclusionFilter', { +new s3deploy.BucketDeployment(this, 'DeployButExcludeSpecificFiles', { sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))], destinationBucket, - exclude: ['build/*'], - include: ['build/config.json'] + exclude: ['*.txt'] }); ``` diff --git a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts index 5dc9bb9918c05..4281a82b74140 100644 --- a/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts +++ b/packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts @@ -36,19 +36,24 @@ export interface BucketDeploymentProps { readonly destinationKeyPrefix?: string; /** - * If this is set, matching files or objects will be excluded from the deployment. If - * no include or exclude filters are used, all files will be included with the deployment. + * If this is set, matching files or objects will be excluded from the deployment's sync + * command. This can be used to exclude a file from being pruned in the destination bucket. * - * @default - No exclude filters are included + * If you want to just exclude files from the deployment package (which excludes these files + * evaluated when invalidating the asset), you should leverage the `exclude` property of + * `AssetOptions` when defining your source. + * + * @default - No exclude filters are used * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters */ readonly exclude?: string[] /** - * If this is set, matching files or objects will be included with the deployment. If - * no include or exclude filters are used, all files will be included with the deployment. + * If this is set, matching files or objects will be included with the deployment's sync + * command. Since all files from the deployment package are included by default, this property + * is usually leveraged alongside an `exclude` filter. * - * @default - No include filters are included + * @default - No include filters are used and all files are included with the sync command * @see https://docs.aws.amazon.com/cli/latest/reference/s3/index.html#use-of-exclude-and-include-filters */ readonly include?: string[]