diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts index 2299b74929d4f..33fea7ed959cd 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts @@ -331,6 +331,30 @@ export interface CdkModernBootstrapCommandOptions extends CommonCdkBootstrapComm readonly usePreviousParameters?: boolean; } +export interface CdkGarbageCollectionCommandOptions { + /** + * The amount of days an asset should stay isolated before deletion, to + * guard against some pipeline rollback scenarios + * + * @default 0 + */ + readonly rollbackBufferDays?: number; + + /** + * The type of asset that is getting garbage collected. + * + * @default 'all' + */ + readonly type?: 'ecr' | 's3' | 'all'; + + /** + * The name of the bootstrap stack + * + * @default 'CdkToolkit' + */ + readonly bootstrapStackName?: string; +} + export class TestFixture extends ShellHelper { public readonly qualifier = this.randomString.slice(0, 10); private readonly bucketsToDelete = new Array(); @@ -464,6 +488,26 @@ export class TestFixture extends ShellHelper { }); } + public async cdkGarbageCollect(options: CdkGarbageCollectionCommandOptions): Promise { + const args = [ + 'gc', + '--unstable=gc', // TODO: remove when stabilizing + '--confirm=false', + '--created-buffer-days=0', // Otherwise all assets created during integ tests are too young + ]; + if (options.rollbackBufferDays) { + args.push('--rollback-buffer-days', String(options.rollbackBufferDays)); + } + if (options.type) { + args.push('--type', options.type); + } + if (options.bootstrapStackName) { + args.push('--bootstrapStackName', options.bootstrapStackName); + } + + return this.cdk(args); + } + public async cdkMigrate(language: string, stackName: string, inputPath?: string, options?: CdkCliOptions) { return this.cdk([ 'migrate', @@ -634,6 +678,7 @@ async function ensureBootstrapped(fixture: TestFixture) { CDK_NEW_BOOTSTRAP: '1', }, }); + ALREADY_BOOTSTRAPPED_IN_THIS_RUN.add(envSpecifier); } diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts new file mode 100644 index 0000000000000..1ded54aa7f05f --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/garbage-collection.integtest.ts @@ -0,0 +1,202 @@ +import { GetObjectTaggingCommand, ListObjectsV2Command, PutObjectTaggingCommand } from '@aws-sdk/client-s3'; +import { integTest, randomString, withoutBootstrap } from '../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +integTest( + 'Garbage Collection deletes unused assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + + await fixture.cdkGarbageCollect({ + rollbackBufferDays: 0, + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket is empty + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then((result) => { + expect(result.Contents).toBeUndefined(); + }); + }), +); + +integTest( + 'Garbage Collection keeps in use assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + await fixture.cdkGarbageCollect({ + rollbackBufferDays: 0, + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket has the object + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then((result) => { + expect(result.Contents).toHaveLength(1); + }); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Teardown complete!'); + }), +); + +integTest( + 'Garbage Collection tags unused assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + await fixture.cdkDestroy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + + await fixture.cdkGarbageCollect({ + rollbackBufferDays: 100, // this will ensure that we do not delete assets immediately (and just tag them) + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the bootstrap bucket has the object and is tagged + await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })) + .then(async (result) => { + expect(result.Contents).toHaveLength(2); // also the CFN template + const key = result.Contents![0].Key; + const tags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); + expect(tags.TagSet).toHaveLength(1); + }); + }), +); + +integTest( + 'Garbage Collection untags in-use assets', + withoutBootstrap(async (fixture) => { + const toolkitStackName = fixture.bootstrapStackName; + const bootstrapBucketName = `aws-cdk-garbage-collect-integ-test-bckt-${randomString()}`; + fixture.rememberToDeleteBucket(bootstrapBucketName); // just in case + + await fixture.cdkBootstrapModern({ + toolkitStackName, + bootstrapBucketName, + }); + + await fixture.cdkDeploy('lambda', { + options: [ + '--context', `bootstrapBucket=${bootstrapBucketName}`, + '--context', `@aws-cdk/core:bootstrapQualifier=${fixture.qualifier}`, + '--toolkit-stack-name', toolkitStackName, + '--force', + ], + }); + fixture.log('Setup complete!'); + + // Artificially add tagging to the asset in the bootstrap bucket + const result = await fixture.aws.s3.send(new ListObjectsV2Command({ Bucket: bootstrapBucketName })); + const key = result.Contents!.filter((c) => c.Key?.split('.')[1] == 'zip')[0].Key; // fancy footwork to make sure we have the asset key + await fixture.aws.s3.send(new PutObjectTaggingCommand({ + Bucket: bootstrapBucketName, + Key: key, + Tagging: { + TagSet: [{ + Key: 'aws-cdk:isolated', + Value: '12345', + }, { + Key: 'bogus', + Value: 'val', + }], + }, + })); + + await fixture.cdkGarbageCollect({ + rollbackBufferDays: 100, // this will ensure that we do not delete assets immediately (and just tag them) + type: 's3', + bootstrapStackName: toolkitStackName, + }); + fixture.log('Garbage collection complete!'); + + // assert that the isolated object tag is removed while the other tag remains + const newTags = await fixture.aws.s3.send(new GetObjectTaggingCommand({ Bucket: bootstrapBucketName, Key: key })); + + expect(newTags.TagSet).toEqual([{ + Key: 'bogus', + Value: 'val', + }]); + }), +); diff --git a/packages/aws-cdk/README.md b/packages/aws-cdk/README.md index 643880ccbc4ab..deec121451f77 100644 --- a/packages/aws-cdk/README.md +++ b/packages/aws-cdk/README.md @@ -25,6 +25,7 @@ The AWS CDK Toolkit provides the `cdk` command-line interface that can be used t | [`cdk watch`](#cdk-watch) | Watches a CDK app for deployable and hotswappable changes | | [`cdk destroy`](#cdk-destroy) | Deletes a stack from an AWS account | | [`cdk bootstrap`](#cdk-bootstrap) | Deploy a toolkit stack to support deploying large stacks & artifacts | +| [`cdk gc`](#cdk-gc) | Garbage collect assets associated with the bootstrapped stack | | [`cdk doctor`](#cdk-doctor) | Inspect the environment and produce information useful for troubleshooting | | [`cdk acknowledge`](#cdk-acknowledge) | Acknowledge (and hide) a notice by issue number | | [`cdk notices`](#cdk-notices) | List all relevant notices for the application | @@ -876,6 +877,78 @@ In order to remove that permissions boundary you have to specify the cdk bootstrap --no-previous-parameters ``` +### `cdk gc` + +CDK Garbage Collection. + +> [!CAUTION] +> CDK Garbage Collection is under development and therefore must be opted in via the `--unstable` flag: `cdk gc --unstable=gc`. +> +> [!WARNING] +> `cdk gc` currently only supports garbage collecting S3 Assets. You must specify `cdk gc --unstable=gc --type=s3` as ECR asset garbage collection has not yet been implemented. + +`cdk gc` garbage collects unused S3 assets from your bootstrap bucket via the following mechanism: + +- for each object in the bootstrap S3 Bucket, check to see if it is referenced in any existing CloudFormation templates +- if not, it is treated as unused and gc will either tag it or delete it, depending on your configuration. + +The most basic usage looks like this: + +```console +cdk gc --unstable=gc --type=s3 +``` + +This will garbage collect S3 assets from the current bootstrapped environment(s) and immediately delete them. Note that, since the default bootstrap S3 Bucket is versioned, object deletion will be handled by the lifecycle +policy on the bucket. + +Before we begin to delete your assets, you will be prompted: + +```console +cdk gc --unstable=gc --type=s3 + +Found X objects to delete based off of the following criteria: +- objects have been isolated for > 0 days +- objects were created > 1 days ago + +Delete this batch (yes/no/delete-all)? +``` + +Since it's quite possible that the bootstrap bucket has many objects, we work in batches of 1000 objects. To skip the +prompt either reply with `delete-all`, or use the `--confirm=false` option. + +```console +cdk gc --unstable=gc --type=s3 --confirm=false +``` + +If you are concerned about deleting assets too aggressively, there are multiple levers you can configure: + +- rollback-buffer-days: this is the amount of days an asset has to be marked as isolated before it is elligible for deletion. +- created-buffer-days: this is the amount of days an asset must live before it is elligible for deletion. + +When using `rollback-buffer-days`, instead of deleting unused objects, `cdk gc` will tag them with +today's date instead. It will also check if any objects have been tagged by previous runs of `cdk gc` +and delete them if they have been tagged for longer than the buffer days. + +When using `created-buffer-days`, we simply filter out any assets that have not persisted that number +of days. + +```console +cdk gc --unstable=gc --type=s3 --rollback-buffer-days=30 --created-buffer-days=1 +``` + +You can also configure the scope that `cdk gc` performs via the `--action` option. By default, all actions +are performed, but you can specify `print`, `tag`, or `delete-tagged`. + +- `print` performs no changes to your AWS account, but finds and prints the number of unused assets. +- `tag` tags any newly unused assets, but does not delete any unused assets. +- `delete-tagged` deletes assets that have been tagged for longer than the buffer days, but does not tag newly unused assets. + +```console +cdk gc --unstable=gc --type=s3 --action=delete-tagged --rollback-buffer-days=30 +``` + +This will delete assets that have been unused for >30 days, but will not tag additional assets. + ### `cdk doctor` Inspect the current command-line environment and configurations, and collect information that can be useful for diff --git a/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts new file mode 100644 index 0000000000000..7fe512805e0fc --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collection/garbage-collector.ts @@ -0,0 +1,455 @@ +import * as cxapi from '@aws-cdk/cx-api'; +import { S3 } from 'aws-sdk'; +import * as chalk from 'chalk'; +import * as promptly from 'promptly'; +import { debug, print } from '../../logging'; +import { ISDK, Mode, SdkProvider } from '../aws-auth'; +import { DEFAULT_TOOLKIT_STACK_NAME, ToolkitInfo } from '../toolkit-info'; +import { ProgressPrinter } from './progress-printer'; +import { ActiveAssetCache, BackgroundStackRefresh, refreshStacks } from './stack-refresh'; + +// Must use a require() otherwise esbuild complains +// eslint-disable-next-line @typescript-eslint/no-require-imports +const pLimit: typeof import('p-limit') = require('p-limit'); + +const ISOLATED_TAG = 'aws-cdk:isolated'; +const P_LIMIT = 50; +const DAY = 24 * 60 * 60 * 1000; // Number of milliseconds in a day + +export class S3Asset { + private cached_tags: S3.TagSet | undefined = undefined; + + public constructor(private readonly bucket: string, public readonly key: string, public readonly size: number) {} + + public fileName(): string { + return this.key.split('.')[0]; + } + + public async allTags(s3: S3) { + if (this.cached_tags) { + return this.cached_tags; + } + + const response = await s3.getObjectTagging({ Bucket: this.bucket, Key: this.key }).promise(); + this.cached_tags = response.TagSet; + return this.cached_tags; + } + + private getTag(tag: string) { + if (!this.cached_tags) { + throw new Error('Cannot call getTag before allTags'); + } + return this.cached_tags.find(t => t.Key === tag)?.Value; + } + + private hasTag(tag: string) { + if (!this.cached_tags) { + throw new Error('Cannot call hasTag before allTags'); + } + return this.cached_tags.some(t => t.Key === tag); + } + + public hasIsolatedTag() { + return this.hasTag(ISOLATED_TAG); + } + + public isolatedTagBefore(date: Date) { + const tagValue = this.getTag(ISOLATED_TAG); + if (!tagValue || tagValue == '') { + return false; + } + return new Date(tagValue) < date; + } +} + +/** + * Props for the Garbage Collector + */ +interface GarbageCollectorProps { + /** + * The action to perform. Specify this if you want to perform a truncated set + * of actions available. + */ + readonly action: 'print' | 'tag' | 'delete-tagged' | 'full'; + + /** + * The type of asset to garbage collect. + */ + readonly type: 's3' | 'ecr' | 'all'; + + /** + * The days an asset must be in isolation before being actually deleted. + */ + readonly rollbackBufferDays: number; + + /** + * Refuse deletion of any assets younger than this number of days. + */ + readonly createdBufferDays: number; + + /** + * The environment to deploy this stack in + * + * The environment on the stack artifact may be unresolved, this one + * must be resolved. + */ + readonly resolvedEnvironment: cxapi.Environment; + + /** + * SDK provider (seeded with default credentials) + * + * Will be used to make SDK calls to CloudFormation, S3, and ECR. + */ + readonly sdkProvider: SdkProvider; + + /** + * The name of the bootstrap stack to look for. + * + * @default DEFAULT_TOOLKIT_STACK_NAME + */ + readonly bootstrapStackName?: string; + + /** + * Max wait time for retries in milliseconds (for testing purposes). + * + * @default 60000 + */ + readonly maxWaitTime?: number; + + /** + * Confirm with the user before actual deletion happens + * + * @default true + */ + readonly confirm?: boolean; +} + +/** + * A class to facilitate Garbage Collection of S3 and ECR assets + */ +export class GarbageCollector { + private garbageCollectS3Assets: boolean; + private garbageCollectEcrAssets: boolean; + private permissionToDelete: boolean; + private permissionToTag: boolean; + private bootstrapStackName: string; + private maxWaitTime: number; + private confirm: boolean; + + public constructor(readonly props: GarbageCollectorProps) { + this.garbageCollectS3Assets = ['s3', 'all'].includes(props.type); + this.garbageCollectEcrAssets = ['ecr', 'all'].includes(props.type); + + debug(`${this.garbageCollectS3Assets} ${this.garbageCollectEcrAssets}`); + + this.permissionToDelete = ['delete-tagged', 'full'].includes(props.action); + this.permissionToTag = ['tag', 'full'].includes(props.action); + this.maxWaitTime = props.maxWaitTime ?? 60000; + this.confirm = props.confirm ?? true; + + this.bootstrapStackName = props.bootstrapStackName ?? DEFAULT_TOOLKIT_STACK_NAME; + + // TODO: ECR garbage collection + if (this.garbageCollectEcrAssets) { + throw new Error('ECR garbage collection is not yet supported'); + } + } + + /** + * Perform garbage collection on the resolved environment. + */ + public async garbageCollect() { + // SDKs + const sdk = (await this.props.sdkProvider.forEnvironment(this.props.resolvedEnvironment, Mode.ForWriting)).sdk; + const cfn = sdk.cloudFormation(); + const s3 = sdk.s3(); + + const qualifier = await this.bootstrapQualifier(sdk, this.bootstrapStackName); + const activeAssets = new ActiveAssetCache(); + + // Grab stack templates first + await refreshStacks(cfn, activeAssets, this.maxWaitTime, qualifier); + // Start the background refresh + const backgroundStackRefresh = new BackgroundStackRefresh({ + cfn, + activeAssets, + qualifier, + maxWaitTime: this.maxWaitTime, + }); + backgroundStackRefresh.start(); + + const bucket = await this.bootstrapBucketName(sdk, this.bootstrapStackName); + const numObjects = await this.numObjectsInBucket(s3, bucket); + const printer = new ProgressPrinter(numObjects, 1000); + + debug(`Found bootstrap bucket ${bucket}`); + + try { + const batches = 1; + const batchSize = 1000; + const currentTime = Date.now(); + const graceDays = this.props.rollbackBufferDays; + + debug(`Parsing through ${numObjects} objects in batches`); + + // Process objects in batches of 1000 + // This is the batch limit of s3.DeleteObject and we intend to optimize for the "worst case" scenario + // where gc is run for the first time on a long-standing bucket where ~100% of objects are isolated. + for await (const batch of this.readBucketInBatches(s3, bucket, batchSize, currentTime)) { + await backgroundStackRefresh.noOlderThan(600_000); // 10 mins + print(chalk.green(`Processing batch ${batches} of ${Math.floor(numObjects / batchSize) + 1}`)); + printer.start(); + + const { included: isolated, excluded: notIsolated } = partition(batch, asset => !activeAssets.contains(asset.fileName())); + + debug(`${isolated.length} isolated assets`); + debug(`${notIsolated.length} not isolated assets`); + debug(`${batch.length} objects total`); + + let deletables: S3Asset[] = isolated; + let taggables: S3Asset[] = []; + let untaggables: S3Asset[] = []; + + if (graceDays > 0) { + debug('Filtering out assets that are not old enough to delete'); + await this.parallelReadAllTags(s3, batch); + + // We delete objects that are not referenced in ActiveAssets and have the Isolated Tag with a date + // earlier than the current time - grace period. + deletables = isolated.filter(obj => obj.isolatedTagBefore(new Date(currentTime - (graceDays * DAY)))); + + // We tag objects that are not referenced in ActiveAssets and do not have the Isolated Tag. + taggables = isolated.filter(obj => !obj.hasIsolatedTag()); + + // We untag objects that are referenced in ActiveAssets and currently have the Isolated Tag. + untaggables = notIsolated.filter(obj => obj.hasIsolatedTag()); + } + + debug(`${deletables.length} deletable assets`); + debug(`${taggables.length} taggable assets`); + debug(`${untaggables.length} assets to untag`); + + if (this.permissionToDelete && deletables.length > 0) { + if (this.confirm) { + const message = [ + `Found ${deletables.length} objects to delete based off of the following criteria:`, + `- objects have been isolated for > ${this.props.rollbackBufferDays} days`, + `- objects were created > ${this.props.createdBufferDays} days ago`, + '', + 'Delete this batch (yes/no/delete-all)?', + ].join('\n'); + printer.pause(); + const response = await promptly.prompt(message, + { trim: true }, + ); + + // Anything other than yes/y/delete-all is treated as no + if (!response || !['yes', 'y', 'delete-all'].includes(response.toLowerCase())) { + throw new Error('Deletion aborted by user'); + } else if (response.toLowerCase() == 'delete-all') { + this.confirm = false; + } + } + printer.resume(); + await this.parallelDelete(s3, bucket, deletables, printer); + } + + if (this.permissionToTag && taggables.length > 0) { + await this.parallelTag(s3, bucket, taggables, currentTime, printer); + } + + if (this.permissionToTag && untaggables.length > 0) { + await this.parallelUntag(s3, bucket, untaggables); + } + + printer.reportScannedObjects(batch.length); + } + } catch (err: any) { + throw new Error(err); + } finally { + backgroundStackRefresh.stop(); + printer.stop(); + } + } + + private async parallelReadAllTags(s3: S3, objects: S3Asset[]) { + const limit = pLimit(P_LIMIT); + + for (const obj of objects) { + await limit(() => obj.allTags(s3)); + } + } + + /** + * Untag assets that were previously tagged, but now currently referenced. + * Since this is treated as an implementation detail, we do not print the results in the printer. + */ + private async parallelUntag(s3: S3, bucket: string, untaggables: S3Asset[]) { + const limit = pLimit(P_LIMIT); + + for (const obj of untaggables) { + const tags = await obj.allTags(s3); + const updatedTags = tags.filter(tag => tag.Key !== ISOLATED_TAG); + await limit(() => + s3.deleteObjectTagging({ + Bucket: bucket, + Key: obj.key, + + }).promise(), + ); + await limit(() => + s3.putObjectTagging({ + Bucket: bucket, + Key: obj.key, + Tagging: { + TagSet: updatedTags, + }, + }).promise(), + ); + } + + debug(`Untagged ${untaggables.length} assets`); + } + + /** + * Tag objects in parallel using p-limit. The putObjectTagging API does not + * support batch tagging so we must handle the parallelism client-side. + */ + private async parallelTag(s3: S3, bucket: string, taggables: S3Asset[], date: number, printer: ProgressPrinter) { + const limit = pLimit(P_LIMIT); + + for (const obj of taggables) { + await limit(() => + s3.putObjectTagging({ + Bucket: bucket, + Key: obj.key, + Tagging: { + TagSet: [ + { + Key: ISOLATED_TAG, + Value: String(date), + }, + ], + }, + }).promise(), + ); + } + + printer.reportTaggedObjects(taggables); + debug(`Tagged ${taggables.length} assets`); + } + + /** + * Delete objects in parallel. The deleteObjects API supports batches of 1000. + */ + private async parallelDelete(s3: S3, bucket: string, deletables: S3Asset[], printer: ProgressPrinter) { + const batchSize = 1000; + const objectsToDelete: S3.ObjectIdentifierList = deletables.map(asset => ({ + Key: asset.key, + })); + + try { + const batches = []; + for (let i = 0; i < objectsToDelete.length; i += batchSize) { + batches.push(objectsToDelete.slice(i, i + batchSize)); + } + // Delete objects in batches + for (const batch of batches) { + await s3.deleteObjects({ + Bucket: bucket, + Delete: { + Objects: batch, + Quiet: true, + }, + }).promise(); + + const deletedCount = batch.length; + debug(`Deleted ${deletedCount} assets`); + printer.reportDeletedObjects(deletables.slice(0, deletedCount)); + } + } catch (err) { + print(chalk.red(`Error deleting objects: ${err}`)); + } + } + + private async bootstrapBucketName(sdk: ISDK, bootstrapStackName: string): Promise { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); + return info.bucketName; + } + + private async bootstrapQualifier(sdk: ISDK, bootstrapStackName: string): Promise { + const info = await ToolkitInfo.lookup(this.props.resolvedEnvironment, sdk, bootstrapStackName); + return info.bootstrapStack.parameters.Qualifier; + } + + private async numObjectsInBucket(s3: S3, bucket: string): Promise { + let totalCount = 0; + let continuationToken: string | undefined; + + do { + const response = await s3.listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken, + }).promise(); + + totalCount += response.KeyCount ?? 0; + continuationToken = response.NextContinuationToken; + } while (continuationToken); + + return totalCount; + } + + /** + * Generator function that reads objects from the S3 Bucket in batches. + */ + private async *readBucketInBatches(s3: S3, bucket: string, batchSize: number = 1000, currentTime: number): AsyncGenerator { + let continuationToken: string | undefined; + + do { + const batch: S3Asset[] = []; + + while (batch.length < batchSize) { + const response = await s3.listObjectsV2({ + Bucket: bucket, + ContinuationToken: continuationToken, + }).promise(); + + response.Contents?.forEach((obj) => { + const key = obj.Key ?? ''; + const size = obj.Size ?? 0; + const lastModified = obj.LastModified ?? new Date(currentTime); + // Store the object if it has a Key and + // if it has not been modified since today - createdBufferDays + if (key && lastModified < new Date(currentTime - (this.props.createdBufferDays * DAY))) { + batch.push(new S3Asset(bucket, key, size)); + } + }); + + continuationToken = response.NextContinuationToken; + + if (!continuationToken) break; // No more objects to fetch + } + + if (batch.length > 0) { + yield batch; + } + } while (continuationToken); + } +} + +function partition(xs: Iterable, pred: (x: A) => boolean): { included: A[]; excluded: A[] } { + const result = { + included: [] as A[], + excluded: [] as A[], + }; + + for (const x of xs) { + if (pred(x)) { + result.included.push(x); + } else { + result.excluded.push(x); + } + } + + return result; +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts new file mode 100644 index 0000000000000..441bb5fb3977c --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collection/progress-printer.ts @@ -0,0 +1,76 @@ +import * as chalk from 'chalk'; +import { S3Asset } from './garbage-collector'; +import { print } from '../../logging'; + +export class ProgressPrinter { + private totalObjects: number; + private objectsScanned: number; + private taggedObjects: number; + private taggedObjectsSizeMb: number; + private deletedObjects: number; + private deletedObjectsSizeMb: number; + private interval: number; + private setInterval?: NodeJS.Timer; + private isPaused: boolean; + + constructor(totalObjects: number, interval?: number) { + this.totalObjects = totalObjects; + this.objectsScanned = 0; + this.taggedObjects = 0; + this.taggedObjectsSizeMb = 0; + this.deletedObjects = 0; + this.deletedObjectsSizeMb = 0; + this.interval = interval ?? 10_000; + this.isPaused = false; + } + + public reportScannedObjects(amt: number) { + this.objectsScanned += amt; + } + + public reportTaggedObjects(objects: S3Asset[]) { + this.taggedObjects += objects.length; + const sizeInBytes = objects.reduce((total, asset) => total + asset.size, 0); + this.taggedObjectsSizeMb += sizeInBytes / 1_048_576; + } + + public reportDeletedObjects(objects: S3Asset[]) { + this.deletedObjects += objects.length; + const sizeInBytes = objects.reduce((total, asset) => total + asset.size, 0); + this.deletedObjectsSizeMb += sizeInBytes / 1_048_576; + } + + public start() { + this.setInterval = setInterval(() => { + if (!this.isPaused) { + this.print(); + } + }, this.interval); + } + + public pause() { + this.isPaused = true; + } + + public resume() { + this.isPaused = false; + } + + public stop() { + clearInterval(this.setInterval); + // print one last time if not paused + if (!this.isPaused) { + this.print(); + } + } + + private print() { + const percentage = ((this.objectsScanned / this.totalObjects) * 100).toFixed(2); + // print in MiB until we hit at least 1 GiB of data tagged/deleted + if (Math.max(this.taggedObjectsSizeMb, this.deletedObjectsSizeMb) >= 1000) { + print(chalk.green(`[${percentage}%] ${this.objectsScanned} files scanned: ${this.taggedObjects} objects (${(this.taggedObjectsSizeMb / 1000).toFixed(2)} GiB) tagged, ${this.deletedObjects} objects (${(this.deletedObjectsSizeMb / 1000).toFixed(2)} GiB) deleted.`)); + } else { + print(chalk.green(`[${percentage}%] ${this.objectsScanned} files scanned: ${this.taggedObjects} objects (${this.taggedObjectsSizeMb.toFixed(2)} MiB) tagged, ${this.deletedObjects} objects (${this.deletedObjectsSizeMb.toFixed(2)} MiB) deleted.`)); + } + } +} \ No newline at end of file diff --git a/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts new file mode 100644 index 0000000000000..14ba12069a130 --- /dev/null +++ b/packages/aws-cdk/lib/api/garbage-collection/stack-refresh.ts @@ -0,0 +1,220 @@ +import { CloudFormation } from 'aws-sdk'; +import { sleep } from '../../../test/util'; +import { debug } from '../../logging'; + +export class ActiveAssetCache { + private readonly stacks: Set = new Set(); + + public rememberStack(stackTemplate: string) { + this.stacks.add(stackTemplate); + } + + public contains(asset: string): boolean { + for (const stack of this.stacks) { + if (stack.includes(asset)) { + return true; + } + } + return false; + } +} + +async function paginateSdkCall(cb: (nextToken?: string) => Promise) { + let finished = false; + let nextToken: string | undefined; + while (!finished) { + nextToken = await cb(nextToken); + if (nextToken === undefined) { + finished = true; + } + } +} + +/** We cannot operate on REVIEW_IN_PROGRESS stacks because we do not know what the template looks like in this case + * If we encounter this status, we will wait up to the maxWaitTime before erroring out + */ +async function listStacksNotBeingReviewed(cfn: CloudFormation, maxWaitTime: number, nextToken: string | undefined) { + let sleepMs = 500; + const deadline = Date.now() + maxWaitTime; + + while (Date.now() <= deadline) { + let stacks = await cfn.listStacks({ NextToken: nextToken }).promise(); + if (!stacks.StackSummaries?.some(s => s.StackStatus == 'REVIEW_IN_PROGRESS')) { + return stacks; + } + await sleep(Math.floor(Math.random() * sleepMs)); + sleepMs = sleepMs * 2; + } + + throw new Error(`Stacks still in REVIEW_IN_PROGRESS state after waiting for ${maxWaitTime} ms.`); +} + +/** + * Fetches all relevant stack templates from CloudFormation. It ignores the following stacks: + * - stacks in DELETE_COMPLETE or DELETE_IN_PROGRESS stage + * - stacks that are using a different bootstrap qualifier + * + * It fails on the following stacks because we cannot get the template and therefore have an imcomplete + * understanding of what assets are being used. + * - stacks in REVIEW_IN_PROGRESS stage + */ +async function fetchAllStackTemplates(cfn: CloudFormation, maxWaitTime: number, qualifier?: string) { + const stackNames: string[] = []; + await paginateSdkCall(async (nextToken) => { + const stacks = await listStacksNotBeingReviewed(cfn, maxWaitTime, nextToken); + + // We ignore stacks with these statuses because their assets are no longer live + const ignoredStatues = ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_IN_PROGRESS', 'DELETE_FAILED']; + stackNames.push( + ...(stacks.StackSummaries ?? []) + .filter(s => !ignoredStatues.includes(s.StackStatus)) + .map(s => s.StackId ?? s.StackName), + ); + + return stacks.NextToken; + }); + + debug(`Parsing through ${stackNames.length} stacks`); + + const templates: string[] = []; + for (const stack of stackNames) { + let summary; + summary = await cfn.getTemplateSummary({ + StackName: stack, + }).promise(); + + if (bootstrapFilter(summary.Parameters, qualifier)) { + // This stack is definitely bootstrapped to a different qualifier so we can safely ignore it + continue; + } else { + const template = await cfn.getTemplate({ + StackName: stack, + }).promise(); + + templates.push((template.TemplateBody ?? '') + JSON.stringify(summary?.Parameters)); + } + } + + debug('Done parsing through stacks'); + + return templates; +} + +/** + * Filter out stacks that we KNOW are using a different bootstrap qualifier + * This is mostly necessary for the integration tests that can run the same app (with the same assets) + * under different qualifiers. + * This is necessary because a stack under a different bootstrap could coincidentally reference the same hash + * and cause a false negative (cause an asset to be preserved when its isolated) + * This is intentionally done in a way where we ONLY filter out stacks that are meant for a different qualifier + * because we are okay with false positives. + */ +function bootstrapFilter(parameters?: CloudFormation.ParameterDeclarations, qualifier?: string) { + const bootstrapVersion = parameters?.find((p) => p.ParameterKey === 'BootstrapVersion'); + const splitBootstrapVersion = bootstrapVersion?.DefaultValue?.split('/'); + // We find the qualifier in a specific part of the bootstrap version parameter + return (qualifier && + splitBootstrapVersion && + splitBootstrapVersion.length == 4 && + splitBootstrapVersion[2] != qualifier); +} + +export async function refreshStacks(cfn: CloudFormation, activeAssets: ActiveAssetCache, maxWaitTime: number, qualifier?: string) { + try { + const stacks = await fetchAllStackTemplates(cfn, maxWaitTime, qualifier); + for (const stack of stacks) { + activeAssets.rememberStack(stack); + } + } catch (err) { + throw new Error(`Error refreshing stacks: ${err}`); + } +} + +/** + * Background Stack Refresh properties + */ +export interface BackgroundStackRefreshProps { + /** + * The CFN SDK handler + */ + readonly cfn: CloudFormation; + + /** + * Active Asset storage + */ + readonly activeAssets: ActiveAssetCache; + + /** + * Stack bootstrap qualifier + */ + readonly qualifier?: string; + + /** + * Maximum wait time when waiting for stacks to leave REVIEW_IN_PROGRESS stage. + * + * @default 60000 + */ + readonly maxWaitTime?: number; +} + +/** + * Class that controls scheduling of the background stack refresh + */ +export class BackgroundStackRefresh { + private timeout?: NodeJS.Timeout; + private lastRefreshTime: number; + private queuedPromises: Array<(value: unknown) => void> = []; + + constructor(private readonly props: BackgroundStackRefreshProps) { + this.lastRefreshTime = Date.now(); + } + + public start() { + // Since start is going to be called right after the first invocation of refreshStacks, + // lets wait some time before beginning the background refresh. + this.timeout = setTimeout(() => this.refresh(), 300_000); // 5 minutes + } + + private async refresh() { + const startTime = Date.now(); + + await refreshStacks(this.props.cfn, this.props.activeAssets, this.props.maxWaitTime ?? 60000, this.props.qualifier); + this.justRefreshedStacks(); + + // If the last invocation of refreshStacks takes <5 minutes, the next invocation starts 5 minutes after the last one started. + // If the last invocation of refreshStacks takes >5 minutes, the next invocation starts immediately. + this.timeout = setTimeout(() => this.refresh(), Math.max(startTime + 300_000 - Date.now(), 0)); + } + + private justRefreshedStacks() { + this.lastRefreshTime = Date.now(); + for (const p of this.queuedPromises.splice(0, this.queuedPromises.length)) { + p(undefined); + } + } + + /** + * Checks if the last successful background refresh happened within the specified time frame. + * If the last refresh is older than the specified time frame, it returns a Promise that resolves + * when the next background refresh completes or rejects if the refresh takes too long. + */ + public noOlderThan(ms: number) { + const horizon = Date.now() - ms; + + // The last refresh happened within the time frame + if (this.lastRefreshTime >= horizon) { + return Promise.resolve(); + } + + // The last refresh happened earlier than the time frame + // We will wait for the latest refresh to land or reject if it takes too long + return Promise.race([ + new Promise(resolve => this.queuedPromises.push(resolve)), + new Promise((_, reject) => setTimeout(() => reject(new Error('refreshStacks took too long; the background thread likely threw an error')), ms)), + ]); + } + + public stop() { + clearTimeout(this.timeout); + } +} diff --git a/packages/aws-cdk/lib/api/index.ts b/packages/aws-cdk/lib/api/index.ts index 5671f05837205..9f3ba4e355a7c 100644 --- a/packages/aws-cdk/lib/api/index.ts +++ b/packages/aws-cdk/lib/api/index.ts @@ -1,5 +1,6 @@ export * from './aws-auth/credentials'; export * from './bootstrap'; +export * from './garbage-collection/garbage-collector'; export * from './deploy-stack'; export * from './toolkit-info'; export * from './aws-auth'; diff --git a/packages/aws-cdk/lib/cdk-toolkit.ts b/packages/aws-cdk/lib/cdk-toolkit.ts index 5b823548413b1..64a9a0b4dd20c 100644 --- a/packages/aws-cdk/lib/cdk-toolkit.ts +++ b/packages/aws-cdk/lib/cdk-toolkit.ts @@ -12,6 +12,7 @@ import { Bootstrapper, BootstrapEnvironmentOptions } from './api/bootstrap'; import { CloudAssembly, DefaultSelection, ExtendedStackSelection, StackCollection, StackSelector } from './api/cxapp/cloud-assembly'; import { CloudExecutable } from './api/cxapp/cloud-executable'; import { Deployments } from './api/deployments'; +import { GarbageCollector } from './api/garbage-collection/garbage-collector'; import { HotswapMode } from './api/hotswap/common'; import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs'; import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor'; @@ -766,6 +767,50 @@ export class CdkToolkit { // If there is an '--app' argument and an environment looks like a glob, we // select the environments from the app. Otherwise, use what the user said. + const environments = await this.defineEnvironments(userEnvironmentSpecs); + + const limit = pLimit(20); + + // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism + await Promise.all(environments.map((environment) => limit(async () => { + success(' ⏳ Bootstrapping environment %s...', chalk.blue(environment.name)); + try { + const result = await bootstrapper.bootstrapEnvironment(environment, this.props.sdkProvider, options); + const message = result.noOp + ? ' ✅ Environment %s bootstrapped (no changes).' + : ' ✅ Environment %s bootstrapped.'; + success(message, chalk.blue(environment.name)); + } catch (e) { + error(' ❌ Environment %s failed bootstrapping: %s', chalk.blue(environment.name), e); + throw e; + } + }))); + } + + /** + * Garbage collects assets from a CDK app's environment + * @param options Options for Garbage Collection + */ + public async garbageCollect(userEnvironmentSpecs: string[], options: GarbageCollectionOptions) { + const environments = await this.defineEnvironments(userEnvironmentSpecs); + + for (const environment of environments) { + success(' ⏳ Garbage Collecting environment %s...', chalk.blue(environment.name)); + const gc = new GarbageCollector({ + sdkProvider: this.props.sdkProvider, + resolvedEnvironment: environment, + bootstrapStackName: options.bootstrapStackName, + rollbackBufferDays: options.rollbackBufferDays, + createdBufferDays: options.createdBufferDays, + action: options.action ?? 'full', + type: options.type ?? 'all', + confirm: options.confirm ?? true, + }); + await gc.garbageCollect(); + }; + } + + private async defineEnvironments(userEnvironmentSpecs: string[]): Promise { // By default, glob for everything const environmentSpecs = userEnvironmentSpecs.length > 0 ? [...userEnvironmentSpecs] : ['**']; @@ -790,22 +835,7 @@ export class CdkToolkit { environments.push(...await globEnvironmentsFromStacks(await this.selectStacksForList([]), globSpecs, this.props.sdkProvider)); } - const limit = pLimit(20); - - // eslint-disable-next-line @cdklabs/promiseall-no-unbounded-parallelism - await Promise.all(environments.map((environment) => limit(async () => { - success(' ⏳ Bootstrapping environment %s...', chalk.blue(environment.name)); - try { - const result = await bootstrapper.bootstrapEnvironment(environment, this.props.sdkProvider, options); - const message = result.noOp - ? ' ✅ Environment %s bootstrapped (no changes).' - : ' ✅ Environment %s bootstrapped.'; - success(message, chalk.blue(environment.name)); - } catch (e) { - error(' ❌ Environment %s failed bootstrapping: %s', chalk.blue(environment.name), e); - throw e; - } - }))); + return environments; } /** @@ -1500,6 +1530,51 @@ export interface DestroyOptions { readonly ci?: boolean; } +/** + * Options for the garbage collection + */ +export interface GarbageCollectionOptions { + /** + * The action to perform. + * + * @default 'full' + */ + readonly action: 'print' | 'tag' | 'delete-tagged' | 'full'; + + /** + * The type of the assets to be garbage collected. + * + * @default 'all' + */ + readonly type: 's3' | 'ecr' | 'all'; + + /** + * Elapsed time between an asset being marked as isolated and actually deleted. + * + * @default 0 + */ + readonly rollbackBufferDays: number; + + /** + * Refuse deletion of any assets younger than this number of days. + */ + readonly createdBufferDays: number; + + /** + * The stack name of the bootstrap stack. + * + * @default DEFAULT_TOOLKIT_STACK_NAME + */ + readonly bootstrapStackName?: string; + + /** + * Skips the prompt before actual deletion begins + * + * @default false + */ + readonly confirm?: boolean; +} + export interface MigrateOptions { /** * The name assigned to the generated stack. This is also used to get diff --git a/packages/aws-cdk/lib/cli.ts b/packages/aws-cdk/lib/cli.ts index 0e28a41a4c106..1cc58f1312435 100644 --- a/packages/aws-cdk/lib/cli.ts +++ b/packages/aws-cdk/lib/cli.ts @@ -86,6 +86,7 @@ async function parseCommandLineArguments(args: string[]) { .option('notices', { type: 'boolean', desc: 'Show relevant notices' }) .option('no-color', { type: 'boolean', desc: 'Removes colors and other style from console output', default: false }) .option('ci', { type: 'boolean', desc: 'Force CI detection. If CI=true then logs will be sent to stdout instead of stderr', default: process.env.CI !== undefined }) + .option('unstable', { type: 'array', desc: 'Opt in to specific unstable features. Can be specified multiple times.', default: [] }) .command(['list [STACKS..]', 'ls [STACKS..]'], 'Lists all stacks in the app', (yargs: Argv) => yargs .option('long', { type: 'boolean', default: false, alias: 'l', desc: 'Display environment information for each stack' }) .option('show-dependencies', { type: 'boolean', default: false, alias: 'd', desc: 'Display stack dependency information for each stack' }), @@ -114,6 +115,14 @@ async function parseCommandLineArguments(args: string[]) { .option('template', { type: 'string', requiresArg: true, desc: 'Use the template from the given file instead of the built-in one (use --show-template to obtain an example)' }) .option('previous-parameters', { type: 'boolean', default: true, desc: 'Use previous values for existing parameters (you must specify all parameters on every deployment if this is disabled)' }), ) + .command('gc [ENVIRONMENTS..]', 'Garbage collect assets', (yargs: Argv) => yargs + .option('action', { type: 'string', desc: 'The action (or sub-action) you want to perform. Valid entires are "print", "tag", "delete-tagged", "full".', default: 'full' }) + .option('type', { type: 'string', desc: 'Specify either ecr, s3, or all', default: 'all' }) + .option('rollback-buffer-days', { type: 'number', desc: 'Delete assets that have been marked as isolated for this many days', default: 0 }) + .option('created-buffer-days', { type: 'number', desc: 'Never delete assets younger than this (in days)', default: 1 }) + .option('confirm', { type: 'boolean', desc: 'Confirm via manual prompt before deletion', default: true }) + .option('bootstrap-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack, if different from the default "CDKToolkit"', requiresArg: true }), + ) .command('deploy [STACKS..]', 'Deploys the stack(s) named STACKS into your AWS account', (yargs: Argv) => yargs .option('all', { type: 'boolean', default: false, desc: 'Deploy all available stacks' }) .option('build-exclude', { type: 'array', alias: 'E', nargs: 1, desc: 'Do not rebuild asset with the given ID. Can be specified multiple times', default: [] }) @@ -678,6 +687,19 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise AWS.CloudFormation.Types.ListStacksOutput; +let mockDescribeStacks: (params: AWS.CloudFormation.Types.DescribeStacksInput) => AWS.CloudFormation.Types.DescribeStacksOutput; +let mockGetTemplateSummary: (params: AWS.CloudFormation.Types.GetTemplateSummaryInput) => AWS.CloudFormation.Types.GetTemplateSummaryOutput; +let mockGetTemplate: (params: AWS.CloudFormation.Types.GetTemplateInput) => AWS.CloudFormation.Types.GetTemplateOutput; +let mockListObjectsV2: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; +let mockGetObjectTagging: (params: AWS.S3.Types.GetObjectTaggingRequest) => AWS.S3.Types.GetObjectTaggingOutput; +let mockDeleteObjects: (params: AWS.S3.Types.DeleteObjectsRequest) => AWS.S3.Types.DeleteObjectsOutput; +let mockDeleteObjectTagging: (params: AWS.S3.Types.DeleteObjectTaggingRequest) => AWS.S3.Types.DeleteObjectTaggingOutput; +let mockPutObjectTagging: (params: AWS.S3.Types.PutObjectTaggingRequest) => AWS.S3.Types.PutObjectTaggingOutput; + +let stderrMock: jest.SpyInstance; +let sdk: MockSdkProvider; + +const ISOLATED_TAG = 'aws-cdk:isolated'; +const DAY = 24 * 60 * 60 * 1000; // Number of milliseconds in a day + +function mockTheToolkitInfo(stackProps: Partial) { + const mockSdk = new MockSdk(); + (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.fromStack(mockBootstrapStack(mockSdk, stackProps))); +} + +function gc(props: { + type: 's3' | 'ecr' | 'all'; + rollbackBufferDays?: number; + createdAtBufferDays?: number; + action: 'full' | 'print' | 'tag' | 'delete-tagged'; + maxWaitTime?: number; +}): GarbageCollector { + return new GarbageCollector({ + sdkProvider: sdk, + action: props.action, + resolvedEnvironment: { + account: '123456789012', + region: 'us-east-1', + name: 'mock', + }, + bootstrapStackName: 'GarbageStack', + rollbackBufferDays: props.rollbackBufferDays ?? 0, + createdBufferDays: props.createdAtBufferDays ?? 0, + type: props.type, + maxWaitTime: props.maxWaitTime, + confirm: false, + }); +} + +beforeEach(() => { + sdk = new MockSdkProvider({ realSdk: false }); + // By default, we'll return a non-found toolkit info + (ToolkitInfo as any).lookup = jest.fn().mockResolvedValue(ToolkitInfo.bootstrapStackNotFoundInfo('GarbageStack')); + stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); +}); + +afterEach(() => { + stderrMock.mockRestore(); +}); + +describe('Garbage Collection', () => { + beforeEach(() => { + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', + }], + }); + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: 'abcde', + }); + mockListObjectsV2 = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: [ + { Key: 'asset1', LastModified: new Date(Date.now() - (2 * DAY)) }, + { Key: 'asset2', LastModified: new Date(Date.now() - (10 * DAY)) }, + { Key: 'asset3', LastModified: new Date(Date.now() - (100 * DAY)) }, + ], + KeyCount: 3, + }); + }); + mockGetObjectTagging = jest.fn().mockImplementation((params) => { + return Promise.resolve({ + TagSet: params.Key === 'asset2' ? [{ Key: ISOLATED_TAG, Value: new Date().toISOString() }] : [], + }); + }); + mockPutObjectTagging = jest.fn(); + mockDeleteObjects = jest.fn(); + mockDeleteObjectTagging = jest.fn(); + mockDescribeStacks = jest.fn(); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + describeStacks: mockDescribeStacks, + }); + sdk.stubS3({ + listObjectsV2: mockListObjectsV2, + getObjectTagging: mockGetObjectTagging, + deleteObjects: mockDeleteObjects, + deleteObjectTagging: mockDeleteObjectTagging, + putObjectTagging: mockPutObjectTagging, + }); + }); + + afterEach(() => { + mockGarbageCollect.mockClear(); + }); + + test('rollbackBufferDays = 0 -- assets to be deleted', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 0, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + // no tagging + expect(mockGetObjectTagging).toHaveBeenCalledTimes(0); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + { Key: 'asset1' }, + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + + test('rollbackBufferDays > 0 -- assets to be tagged', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // assets tagged + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one asset already has the tag + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('type = ecr -- throws error', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + expect(() => garbageCollector = gc({ + type: 'ecr', + rollbackBufferDays: 3, + action: 'full', + })).toThrow(/ECR garbage collection is not yet supported/); + }); + + test('createdAtBufferDays > 0 -- assets to be tagged', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 0, + createdAtBufferDays: 5, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + + test('action = print -- does not tag or delete', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'print', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // get tags, but dont put tags + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('action = tag -- does not delete', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'tag', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // tags objects + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one object already has the tag + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }); + + test('action = delete-tagged -- does not tag', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'delete-tagged', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + + // get tags, but dont put tags + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + }); + + test('ignore objects that are modified after gc start', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + const mockListObjectsV2Future = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: [ + { Key: 'asset1', LastModified: new Date(0) }, + { Key: 'asset2', LastModified: new Date(0) }, + { Key: 'asset3', LastModified: new Date(new Date().setFullYear(new Date().getFullYear() + 1)) }, // future date ignored everywhere + ], + KeyCount: 3, + }); + }); + + sdk.stubS3({ + listObjectsV2: mockListObjectsV2Future, + getObjectTagging: mockGetObjectTagging, + deleteObjects: mockDeleteObjects, + putObjectTagging: mockPutObjectTagging, + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 0, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + { Key: 'asset1' }, + { Key: 'asset2' }, + // no asset3 + ], + Quiet: true, + }, + }); + }); + + test('bootstrap filters out other bootstrap versions', async () => { + mockTheToolkitInfo({ + Parameters: [{ + ParameterKey: 'Qualifier', + ParameterValue: 'zzzzzz', + }], + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockGetTemplateSummary).toHaveBeenCalledTimes(2); + expect(mockGetTemplate).toHaveBeenCalledTimes(0); + }); + + test('parameter hashes are included', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + const mockGetTemplateSummaryAssets = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'AssetParametersasset1', + DefaultValue: 'asset1', + }], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummaryAssets, + getTemplate: mockGetTemplate, + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 0, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2).toHaveBeenCalledTimes(2); + // no tagging + expect(mockGetObjectTagging).toHaveBeenCalledTimes(0); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(0); + + // assets are to be deleted + expect(mockDeleteObjects).toHaveBeenCalledWith({ + Bucket: 'BUCKET_NAME', + Delete: { + Objects: [ + // no 'asset1' + { Key: 'asset2' }, + { Key: 'asset3' }, + ], + Quiet: true, + }, + }); + }); + + test('stackStatus in REVIEW_IN_PROGRESS means we wait until it changes', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + // Mock the listStacks call + const mockListStacksStatus = jest.fn() + .mockResolvedValueOnce({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }) + .mockResolvedValueOnce({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'UPDATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacksStatus, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'full', + }); + await garbageCollector.garbageCollect(); + + // list are called as expected + expect(mockListStacksStatus).toHaveBeenCalledTimes(2); + + // everything else runs as expected: + // assets tagged + expect(mockGetObjectTagging).toHaveBeenCalledTimes(3); + expect(mockPutObjectTagging).toHaveBeenCalledTimes(2); // one object already has the tag + + // no deleting + expect(mockDeleteObjects).toHaveBeenCalledTimes(0); + }, 60000); + + test('fails when stackStatus stuck in REVIEW_IN_PROGRESS', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + // Mock the listStacks call + const mockListStacksStatus = jest.fn() + .mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'REVIEW_IN_PROGRESS' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacksStatus, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 3, + action: 'full', + maxWaitTime: 600, // Wait only 600 ms in tests + }); + + await expect(garbageCollector.garbageCollect()).rejects.toThrow(/Stacks still in REVIEW_IN_PROGRESS state after waiting/); + }, 60000); +}); + +let mockListObjectsV2Large: (params: AWS.S3.Types.ListObjectsV2Request) => AWS.S3.Types.ListObjectsV2Output; +let mockGetObjectTaggingLarge: (params: AWS.S3.Types.GetObjectTaggingRequest) => AWS.S3.Types.GetObjectTaggingOutput; +describe('Garbage Collection with large # of objects', () => { + const keyCount = 10000; + + beforeEach(() => { + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + ], + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', + }], + }); + // add every 5th asset hash to the mock template body: 8000 assets are isolated + const mockTemplateBody = []; + for (let i = 0; i < keyCount; i+=5) { + mockTemplateBody.push(`asset${i}hash`); + } + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: mockTemplateBody.join('-'), + }); + + const contents: { Key: string; LastModified: Date }[] = []; + for (let i = 0; i < keyCount; i++) { + contents.push({ + Key: `asset${i}hash`, + LastModified: new Date(0), + }); + } + mockListObjectsV2Large = jest.fn().mockImplementation(() => { + return Promise.resolve({ + Contents: contents, + KeyCount: keyCount, + }); + }); + + // every other object has the isolated tag: of the 8000 isolated assets, 4000 already are tagged. + // of the 2000 in use assets, 1000 are tagged. + mockGetObjectTaggingLarge = jest.fn().mockImplementation((params) => { + return Promise.resolve({ + TagSet: Number(params.Key[params.Key.length - 5]) % 2 === 0 ? [{ Key: ISOLATED_TAG, Value: new Date(2000, 1, 1).toISOString() }] : [], + }); + }); + mockPutObjectTagging = jest.fn(); + mockDeleteObjects = jest.fn(); + mockDeleteObjectTagging = jest.fn(); + mockDescribeStacks = jest.fn(); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + describeStacks: mockDescribeStacks, + }); + sdk.stubS3({ + listObjectsV2: mockListObjectsV2Large, + getObjectTagging: mockGetObjectTaggingLarge, + deleteObjects: mockDeleteObjects, + deleteObjectTagging: mockDeleteObjectTagging, + putObjectTagging: mockPutObjectTagging, + }); + }); + + afterEach(() => { + mockGarbageCollect.mockClear(); + }); + + test('tag only', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 1, + action: 'tag', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2Large).toHaveBeenCalledTimes(2); + + // tagging is performed + expect(mockGetObjectTaggingLarge).toHaveBeenCalledTimes(keyCount); + expect(mockDeleteObjectTagging).toHaveBeenCalledTimes(1000); // 1000 in use assets are erroneously tagged + expect(mockPutObjectTagging).toHaveBeenCalledTimes(5000); // 8000-4000 assets need to be tagged, + 1000 (since untag also calls this) + }); + + test('delete-tagged only', async () => { + mockTheToolkitInfo({ + Outputs: [ + { + OutputKey: 'BootstrapVersion', + OutputValue: '999', + }, + ], + }); + + garbageCollector = garbageCollector = gc({ + type: 's3', + rollbackBufferDays: 1, + action: 'delete-tagged', + }); + await garbageCollector.garbageCollect(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(mockListObjectsV2Large).toHaveBeenCalledTimes(2); + + // delete previously tagged objects + expect(mockGetObjectTaggingLarge).toHaveBeenCalledTimes(keyCount); + expect(mockDeleteObjects).toHaveBeenCalledTimes(4); // 4000 isolated assets are already tagged, deleted in batches of 1000 + }); +}); + +describe('BackgroundStackRefresh', () => { + let backgroundRefresh: BackgroundStackRefresh; + let refreshProps: BackgroundStackRefreshProps; + let setTimeoutSpy: jest.SpyInstance; + + beforeEach(() => { + jest.useFakeTimers(); + setTimeoutSpy = jest.spyOn(global, 'setTimeout'); + + mockListStacks = jest.fn().mockResolvedValue({ + StackSummaries: [ + { StackName: 'Stack1', StackStatus: 'CREATE_COMPLETE' }, + { StackName: 'Stack2', StackStatus: 'UPDATE_COMPLETE' }, + ], + }); + mockGetTemplateSummary = jest.fn().mockReturnValue({ + Parameters: [{ + ParameterKey: 'BootstrapVersion', + DefaultValue: '/cdk-bootstrap/abcde/version', + }], + }); + mockGetTemplate = jest.fn().mockReturnValue({ + TemplateBody: 'abcde', + }); + + sdk.stubCloudFormation({ + listStacks: mockListStacks, + getTemplateSummary: mockGetTemplateSummary, + getTemplate: mockGetTemplate, + describeStacks: jest.fn(), + }); + + refreshProps = { + cfn: sdk.mockSdk.cloudFormation(), + activeAssets: new ActiveAssetCache(), + maxWaitTime: 60000, // 1 minute + }; + + backgroundRefresh = new BackgroundStackRefresh(refreshProps); + }); + + afterEach(() => { + jest.clearAllTimers(); + setTimeoutSpy.mockRestore(); + }); + + test('should start after a delay', () => { + void backgroundRefresh.start(); + expect(setTimeoutSpy).toHaveBeenCalledTimes(1); + expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 300000); + }); + + test('should refresh stacks and schedule next refresh', async () => { + void backgroundRefresh.start(); + + // Run the first timer (which should trigger the first refresh) + await jest.runOnlyPendingTimersAsync(); + + expect(mockListStacks).toHaveBeenCalledTimes(1); + expect(setTimeoutSpy).toHaveBeenCalledTimes(2); // Once for start, once for next refresh + expect(setTimeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 300000); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + + expect(mockListStacks).toHaveBeenCalledTimes(2); + expect(setTimeoutSpy).toHaveBeenCalledTimes(3); // Two refreshes plus one more scheduled + }); + + test('should wait for the next refresh if called within time frame', async () => { + void backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + + const waitPromise = backgroundRefresh.noOlderThan(180000); // 3 minutes + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + await expect(waitPromise).resolves.toBeUndefined(); + }); + + test('should wait for the next refresh if refresh lands before the timeout', async () => { + void backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + jest.advanceTimersByTime(24000); // Advance time by 4 minutes + + const waitPromise = backgroundRefresh.noOlderThan(300000); // 5 minutes + jest.advanceTimersByTime(120000); // Advance time by 2 minutes, refresh should fire + + await expect(waitPromise).resolves.toBeUndefined(); + }); + + test('should reject if the refresh takes too long', async () => { + void backgroundRefresh.start(); + + // Run the first timer (which triggers the first refresh) + await jest.runOnlyPendingTimersAsync(); + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + const waitPromise = backgroundRefresh.noOlderThan(0); // 0 seconds + jest.advanceTimersByTime(120000); // Advance time by 2 minutes + + await expect(waitPromise).rejects.toThrow('refreshStacks took too long; the background thread likely threw an error'); + }); +});