feat(s3): add option to auto delete objects upon bucket removal

packages/@aws-cdk/aws-s3/README.md

@@ -341,3 +341,20 @@ bucket.urlForObject('objectname'); // Path-Style URL
 bucket.virtualHostedUrlForObject('objectname'); // Virtual Hosted-Style URL
 bucket.virtualHostedUrlForObject('objectname', { regional: false }); // Virtual Hosted-Style URL but non-regional
 ```
+
+### Bucket deletion
+
+When a bucket is removed from a stack (or the stack is deleted), the S3
+bucket will be removed according to its removal policy (which by default will
+simply orphan the bucket and leave it in your AWS account). If the removal
+policy is set to `RemovalPolicy.DESTROY`, the bucket will be deleted as long
+as it does not contain any objects.
+
+To override this and force all objects to get deleted during bucket deletion,
+enable the`autoDeleteObjects` option.
+
+```ts
+const bucket = new Bucket(this, 'MyTempFileBucket', {
+  removalPolicy: RemovalPolicy.DESTROY,
+  autoDeleteObjects: true,
+});

packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts

@@ -0,0 +1,43 @@
+export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent) {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports, import/no-extraneous-dependencies
+  const s3 = new (require('aws-sdk').S3)();
+  if (event.RequestType === 'Create') { return onCreate(s3, event); }
+  if (event.RequestType === 'Update') { return onUpdate(s3, event); }
+  if (event.RequestType === 'Delete') { return onDelete(s3, event); }
+  throw new Error('Invalid request type.');
+}
+
+/**
+ * Recursively delete all items in the bucket
+ *
+ * @param {AWS.S3} s3 the S3 client
+ * @param {*} bucketName the bucket name
+ */
+async function emptyBucket(s3: any, bucketName: string) {
+  const listedObjects = await s3.listObjectVersions({ Bucket: bucketName }).promise();
+  const contents = (listedObjects.Versions || []).concat(listedObjects.DeleteMarkers || []);
+  if (contents.length === 0) {
+    return;
+  };
+
+  let records = contents.map((record: any) => ({ Key: record.Key, VersionId: record.VersionId }));
+  await s3.deleteObjects({ Bucket: bucketName, Delete: { Objects: records } }).promise();
+
+  if (listedObjects?.IsTruncated === 'true' ) await emptyBucket(s3, bucketName);
+}
+
+async function onCreate(_s3: any, _event: AWSLambda.CloudFormationCustomResourceCreateEvent) {
+  return;
+}
+
+async function onUpdate(_s3: any, _event: AWSLambda.CloudFormationCustomResourceUpdateEvent) {
+  return;
+}
+
+async function onDelete(s3: any, deleteEvent: AWSLambda.CloudFormationCustomResourceDeleteEvent) {
+  const bucketName = deleteEvent.ResourceProperties?.BucketName;
+  if (!bucketName) {
+    throw new Error('No BucketName was provided.');
+  }
+  await emptyBucket(s3, bucketName);
+}

packages/@aws-cdk/aws-s3/lib/bucket.ts

@@ -1,8 +1,9 @@
 import { EOL } from 'os';
+import * as path from 'path';
 import * as events from '@aws-cdk/aws-events';
 import * as iam from '@aws-cdk/aws-iam';
 import * as kms from '@aws-cdk/aws-kms';
-import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core';
+import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token, CustomResource, CustomResourceProvider, CustomResourceProviderRuntime } from '@aws-cdk/core';
 import { Construct } from 'constructs';
 import { BucketPolicy } from './bucket-policy';
 import { IBucketNotificationDestination } from './destination';
@@ -12,6 +13,8 @@ import { LifecycleRule } from './rule';
 import { CfnBucket } from './s3.generated';
 import { parseBucketArn, parseBucketName } from './util';
 
+const AUTO_DELETE_OBJECTS_RESOURCE_TYPE = 'Custom::AutoDeleteObjects';
+
 export interface IBucket extends IResource {
   /**
    * The ARN of the bucket.
@@ -1026,6 +1029,16 @@ export interface BucketProps {
    */
   readonly removalPolicy?: RemovalPolicy;
 
+  /**
+   * Whether all objects should be automatically deleted when the bucket is
+   * removed from the stack.
+   *
+   * Requires the removal policy to be set to destroy.
+   *
+   * @default false
+   */
+  readonly autoDeleteObjects?: boolean;
+
   /**
    * Whether this bucket should have versioning turned on or not.
    *
@@ -1301,6 +1314,20 @@ export class Bucket extends BucketBase {
     if (props.publicReadAccess) {
       this.grantPublicAccess();
     }
+
+    if (props.autoDeleteObjects) {
+      if (props.removalPolicy !== RemovalPolicy.DESTROY) {
+        throw new Error("Cannot use 'autoDeleteObjects' property on a bucket without setting removal policy to 'destroy'.");
+      }
+
+      new CustomResource(this, 'AutoDeleteObjectsResource', {
+        resourceType: AUTO_DELETE_OBJECTS_RESOURCE_TYPE,
+        serviceToken: this.getOrCreateAutoDeleteObjectsResource(),
+        properties: {
+          BucketName: this.bucketName,
+        },
+      });
+    }
   }
 
   /**
@@ -1692,6 +1719,23 @@ export class Bucket extends BucketBase {
       };
     });
   }
+
+  private getOrCreateAutoDeleteObjectsResource() {
+    return CustomResourceProvider.getOrCreate(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, {
+      codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'),
+      runtime: CustomResourceProviderRuntime.NODEJS_12,
+      policyStatements: [
+        {
+          Effect: 'Allow',
+          Resource: '*',
+          Action: [
+            ...perms.BUCKET_READ_ACTIONS,
+            ...perms.BUCKET_DELETE_ACTIONS,
+          ],
+        },
+      ],
+    });
+  }
 }
 
 /**

packages/@aws-cdk/aws-s3/test/auto-delete-objects.test.ts

@@ -0,0 +1,250 @@
+import '@aws-cdk/assert/jest';
+import * as cdk from '@aws-cdk/core';
+import * as s3 from '../lib';
+import { handler } from '../lib/auto-delete-objects-handler/index';
+
+test('when autoDeleteObjects is enabled, a custom resource is provisioned + a lambda handler for it', () => {
+  const stack = new cdk.Stack();
+
+  new s3.Bucket(stack, 'MyBucket', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: true,
+  });
+
+  expect(stack).toHaveResource('AWS::S3::Bucket');
+  expect(stack).toHaveResource('AWS::Lambda::Function');
+  expect(stack).toHaveResource('Custom::AutoDeleteObjects',
+    {
+      BucketName: { Ref: 'MyBucketF68F3FF0' },
+      ServiceToken: {
+        'Fn::GetAtt': [
+          'CustomAutoDeleteObjectsCustomResourceProviderHandlerE060A45A',
+          'Arn',
+        ],
+      },
+    });
+});
+
+test('two buckets with autoDeleteObjects enabled share the same cr provider', () => {
+  const app = new cdk.App();
+  const stack = new cdk.Stack(app);
+
+  new s3.Bucket(stack, 'MyBucketOne', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: true,
+  });
+  new s3.Bucket(stack, 'MyBucketTwo', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: true,
+  });
+
+  const template = app.synth().getStackArtifact(stack.artifactId).template;
+  const resourceTypes = Object.values(template.Resources).map((r: any) => r.Type).sort();
+
+  expect(resourceTypes).toStrictEqual([
+    // custom resource provider resources (shared)
+    'AWS::IAM::Role',
+    'AWS::Lambda::Function',
+
+    // buckets
+    'AWS::S3::Bucket',
+    'AWS::S3::Bucket',
+
+    // auto delete object resources
+    'Custom::AutoDeleteObjects',
+    'Custom::AutoDeleteObjects',
+  ]);
+});
+
+test('when only one bucket has autoDeleteObjects enabled, only that bucket gets assigned a custom resource', () => {
+  const app = new cdk.App();
+  const stack = new cdk.Stack(app);
+
+  new s3.Bucket(stack, 'MyBucketOne', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: true,
+  });
+  new s3.Bucket(stack, 'MyBucketTwo', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: false,
+  });
+
+  const template = app.synth().getStackArtifact(stack.artifactId).template;
+  const resourceTypes = Object.values(template.Resources).map((r: any) => r.Type).sort();
+
+  expect(resourceTypes).toStrictEqual([
+    // custom resource provider resources
+    'AWS::IAM::Role',
+    'AWS::Lambda::Function',
+
+    // buckets
+    'AWS::S3::Bucket',
+    'AWS::S3::Bucket',
+
+    // auto delete object resource
+    'Custom::AutoDeleteObjects',
+  ]);
+
+  // custom resource for MyBucket1 is present
+  expect(stack).toHaveResource('Custom::AutoDeleteObjects',
+    { BucketName: { Ref: 'MyBucketOneA6BE54C9' } });
+
+  // custom resource for MyBucket2 is not present
+  expect(stack).not.toHaveResource('Custom::AutoDeleteObjects',
+    { BucketName: { Ref: 'MyBucketTwoC7437026' } });
+});
+
+test('iam policy is created', () => {
+  const stack = new cdk.Stack();
+
+  new s3.Bucket(stack, 'MyBucket', {
+    removalPolicy: cdk.RemovalPolicy.DESTROY,
+    autoDeleteObjects: true,
+  });
+
+  expect(stack).toHaveResource('AWS::IAM::Role', {
+    Policies: [
+      {
+        PolicyName: 'Inline',
+        PolicyDocument: {
+          Version: '2012-10-17',
+          Statement: [
+            {
+              Effect: 'Allow',
+              Resource: '*',
+              Action: [
+                's3:GetObject*',
+                's3:GetBucket*',
+                's3:List*',
+                's3:DeleteObject*',
+              ],
+            },
+          ],
+        },
+      },
+    ],
+  });
+});
+
+test('throws if autoDeleteObjects is enabled but if removalPolicy is not specified', () => {
+  const stack = new cdk.Stack();
+
+  expect(() => new s3.Bucket(stack, 'MyBucket', { autoDeleteObjects: true })).toThrowError(/removal policy/);
+});
+
+describe('custom resource handler', () => {
+
+  const mockS3Client = {
+    listObjectVersions: jest.fn().mockReturnThis(),
+    deleteObjects: jest.fn().mockReturnThis(),
+    promise: jest.fn(),
+  };
+
+  jest.mock('aws-sdk', () => {
+    return { S3: jest.fn(() => mockS3Client) };
+  });
+
+  beforeEach(() => {
+    mockS3Client.deleteObjects.mockClear();
+    mockS3Client.listObjectVersions.mockClear();
+    mockS3Client.promise.mockClear();
+  });
+
+  test('does nothing on create event', async () => {
+    const event: Partial<AWSLambda.CloudFormationCustomResourceCreateEvent> = {
+      RequestType: 'Create',
+      ResourceProperties: {
+        ServiceToken: 'Foo',
+        BucketName: 'MyBucket',
+      },
+    };
+    await invokeHandler(event);
+
+    expect(mockS3Client.listObjectVersions).not.toHaveBeenCalled();
+    expect(mockS3Client.deleteObjects).not.toHaveBeenCalled();
+  });
+
+  test('does nothing on update event', async () => {
+    const event: Partial<AWSLambda.CloudFormationCustomResourceUpdateEvent> = {
+      RequestType: 'Update',
+      ResourceProperties: {
+        ServiceToken: 'Foo',
+        BucketName: 'MyBucket',
+      },
+    };
+    await invokeHandler(event);
+
+    expect(mockS3Client.listObjectVersions).not.toHaveBeenCalled();
+    expect(mockS3Client.deleteObjects).not.toHaveBeenCalled();
+  });
+
+  test('deletes no objects on delete event when bucket has no objects', async () => {
+    mockS3Client.listObjectVersions().promise.mockResolvedValue({ Versions: [] }); // this counts as one call
+
+    const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+      RequestType: 'Delete',
+      ResourceProperties: {
+        ServiceToken: 'Foo',
+        BucketName: 'MyBucket',
+      },
+    };
+    await invokeHandler(event);
+
+    expect(mockS3Client.listObjectVersions.mock.calls.length).toEqual(2);
+    expect(mockS3Client.deleteObjects).not.toHaveBeenCalled();
+  });
+
+  test('deletes all objects on delete event', async () => {
+    mockS3Client.listObjectVersions().promise.mockResolvedValue({ // this counts as one call
+      Versions: [
+        { Key: 'Key1', VersionId: 'VersionId1' },
+        { Key: 'Key2', VersionId: 'VersionId2' },
+      ],
+    });
+
+    const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+      RequestType: 'Delete',
+      ResourceProperties: {
+        ServiceToken: 'Foo',
+        BucketName: 'MyBucket',
+      },
+    };
+    await invokeHandler(event);
+
+    expect(mockS3Client.listObjectVersions.mock.calls.length).toEqual(2);
+    expect(mockS3Client.deleteObjects.mock.calls.length).toEqual(1);
+  });
+
+  test('delete event where bucket has many objects does recurse appropriately', async () => {
+    mockS3Client.listObjectVersions().promise.mockResolvedValueOnce({ // this counts as one call
+      Versions: [
+        { Key: 'Key1', VersionId: 'VersionId1' },
+        { Key: 'Key2', VersionId: 'VersionId2' },
+      ],
+      IsTruncated: 'true',
+    }).mockResolvedValueOnce({
+      Versions: [
+        { Key: 'Key3', VersionId: 'VersionId3' },
+        { Key: 'Key4', VersionId: 'VersionId4' },
+      ],
+    });
+
+    const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+      RequestType: 'Delete',
+      ResourceProperties: {
+        ServiceToken: 'Foo',
+        BucketName: 'MyBucket',
+      },
+    };
+    await invokeHandler(event);
+
+    expect(mockS3Client.listObjectVersions.mock.calls.length).toEqual(3);
+    expect(mockS3Client.deleteObjects.mock.calls.length).toEqual(2);
+  });
+});
+
+// helper function to get around TypeScript expecting a complete event object,
+// even though our tests require some of the fields
+async function invokeHandler(event: Partial<AWSLambda.CloudFormationCustomResourceEvent>) {
+  return handler(event as AWSLambda.CloudFormationCustomResourceEvent);
+}