From c99da16aae68437d1546c8ad431d7050f954ffac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Wed, 29 Dec 2021 08:28:39 +0100 Subject: [PATCH 1/8] feat(codepipeline): variables for CodeStar Connections source Action (#18086) Adds `variables` method that allows to retrieve action's [variables](https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-variables.html) using type safe approach. With that we can rely on compiler to pick up incorrect access instead of hacking it with plain strings. fixes: #17807 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-codepipeline-actions/README.md | 29 ++++ .../lib/codestar-connections/source-action.ts | 30 ++++ ...codestar-connections-source-action.test.ts | 142 ++++++++++++++++-- 3 files changed, 188 insertions(+), 13 deletions(-) diff --git a/packages/@aws-cdk/aws-codepipeline-actions/README.md b/packages/@aws-cdk/aws-codepipeline-actions/README.md index 7625ed08bfb8e..d3569b4ea5872 100644 --- a/packages/@aws-cdk/aws-codepipeline-actions/README.md +++ b/packages/@aws-cdk/aws-codepipeline-actions/README.md @@ -198,6 +198,35 @@ const sourceAction = new codepipeline_actions.CodeStarConnectionsSourceAction({ You can also use the `CodeStarConnectionsSourceAction` to connect to GitHub, in the same way (you just have to select GitHub as the source when creating the connection in the console). +Similarly to `GitHubSourceAction`, `CodeStarConnectionsSourceAction` also emits the variables: + +```ts +declare const project: codebuild.Project; + +const sourceOutput = new codepipeline.Artifact(); +const sourceAction = new codepipeline_actions.CodeStarConnectionsSourceAction({ + actionName: 'BitBucket_Source', + owner: 'aws', + repo: 'aws-cdk', + output: sourceOutput, + connectionArn: 'arn:aws:codestar-connections:us-east-1:123456789012:connection/12345678-abcd-12ab-34cdef5678gh', + variablesNamespace: 'SomeSpace', // optional - by default, a name will be generated for you +}); + +// later: + +new codepipeline_actions.CodeBuildAction({ + actionName: 'CodeBuild', + project, + input: sourceOutput, + environmentVariables: { + COMMIT_ID: { + value: sourceAction.variables.commitId, + }, + }, +}); +``` + ### AWS S3 Source To use an S3 Bucket as a source in CodePipeline: diff --git a/packages/@aws-cdk/aws-codepipeline-actions/lib/codestar-connections/source-action.ts b/packages/@aws-cdk/aws-codepipeline-actions/lib/codestar-connections/source-action.ts index ca47af5c29c8e..71ff24eb35114 100644 --- a/packages/@aws-cdk/aws-codepipeline-actions/lib/codestar-connections/source-action.ts +++ b/packages/@aws-cdk/aws-codepipeline-actions/lib/codestar-connections/source-action.ts @@ -8,6 +8,24 @@ import { sourceArtifactBounds } from '../common'; // eslint-disable-next-line no-duplicate-imports, import/order import { Construct } from '@aws-cdk/core'; +/** + * The CodePipeline variables emitted by CodeStar source Action. + */ +export interface CodeStarSourceVariables { + /** The name of the repository this action points to. */ + readonly fullRepositoryName: string; + /** The name of the branch this action tracks. */ + readonly branchName: string; + /** The date the currently last commit on the tracked branch was authored, in ISO-8601 format. */ + readonly authorDate: string; + /** The SHA1 hash of the currently last commit on the tracked branch. */ + readonly commitId: string; + /** The message of the currently last commit on the tracked branch. */ + readonly commitMessage: string; + /** The connection ARN this source uses. */ + readonly connectionArn: string; +} + /** * Construction properties for {@link CodeStarConnectionsSourceAction}. */ @@ -101,6 +119,18 @@ export class CodeStarConnectionsSourceAction extends Action { this.props = props; } + /** The variables emitted by this action. */ + public get variables(): CodeStarSourceVariables { + return { + fullRepositoryName: this.variableExpression('FullRepositoryName'), + branchName: this.variableExpression('BranchName'), + authorDate: this.variableExpression('AuthorDate'), + commitId: this.variableExpression('CommitId'), + commitMessage: this.variableExpression('CommitMessage'), + connectionArn: this.variableExpression('ConnectionArn'), + }; + } + protected bound(_scope: Construct, _stage: codepipeline.IStage, options: codepipeline.ActionBindOptions): codepipeline.ActionConfig { // https://docs.aws.amazon.com/codepipeline/latest/userguide/security-iam.html#how-to-update-role-new-services options.role.addToPolicy(new iam.PolicyStatement({ diff --git a/packages/@aws-cdk/aws-codepipeline-actions/test/codestar-connections/codestar-connections-source-action.test.ts b/packages/@aws-cdk/aws-codepipeline-actions/test/codestar-connections/codestar-connections-source-action.test.ts index 75fe764109b25..312251bd8457a 100644 --- a/packages/@aws-cdk/aws-codepipeline-actions/test/codestar-connections/codestar-connections-source-action.test.ts +++ b/packages/@aws-cdk/aws-codepipeline-actions/test/codestar-connections/codestar-connections-source-action.test.ts @@ -1,5 +1,5 @@ import '@aws-cdk/assert-internal/jest'; -import { arrayWith, objectLike } from '@aws-cdk/assert-internal'; +import { arrayWith, objectLike, SynthUtils } from '@aws-cdk/assert-internal'; import * as codebuild from '@aws-cdk/aws-codebuild'; import * as codepipeline from '@aws-cdk/aws-codepipeline'; import { Stack } from '@aws-cdk/core'; @@ -148,26 +148,139 @@ describe('CodeStar Connections source Action', () => { ], }); + }); + + test('exposes variables', () => { + const stack = new Stack(); + createBitBucketAndCodeBuildPipeline(stack); + + expect(stack).toHaveResourceLike('AWS::CodePipeline::Pipeline', { + 'Stages': [ + { + 'Name': 'Source', + }, + { + 'Name': 'Build', + 'Actions': [ + { + 'Name': 'CodeBuild', + 'Configuration': { + 'EnvironmentVariables': '[{"name":"CommitId","type":"PLAINTEXT","value":"#{Source_BitBucket_NS.CommitId}"}]', + }, + }, + ], + }, + ], + }); + }); + + test('exposes variables with custom namespace', () => { + const stack = new Stack(); + createBitBucketAndCodeBuildPipeline(stack, { + variablesNamespace: 'kornicameister', + }); + + expect(stack).toHaveResourceLike('AWS::CodePipeline::Pipeline', { + 'Stages': [ + { + 'Name': 'Source', + 'Actions': [ + { + 'Name': 'BitBucket', + 'Namespace': 'kornicameister', + }, + ], + }, + { + 'Name': 'Build', + 'Actions': [ + { + 'Name': 'CodeBuild', + 'Configuration': { + 'EnvironmentVariables': '[{"name":"CommitId","type":"PLAINTEXT","value":"#{kornicameister.CommitId}"}]', + }, + }, + ], + }, + ], + }); + }); + + test('fail if variable from unused action is referenced', () => { + const stack = new Stack(); + const pipeline = createBitBucketAndCodeBuildPipeline(stack); + const unusedSourceOutput = new codepipeline.Artifact(); + const unusedSourceAction = new cpactions.CodeStarConnectionsSourceAction({ + actionName: 'UnusedBitBucket', + owner: 'aws', + repo: 'aws-cdk', + output: unusedSourceOutput, + connectionArn: 'arn:aws:codestar-connections:us-east-1:123456789012:connection/12345678-abcd-12ab-34cdef5678gh', + }); + const unusedBuildAction = new cpactions.CodeBuildAction({ + actionName: 'UnusedCodeBuild', + project: new codebuild.PipelineProject(stack, 'UnusedMyProject'), + input: unusedSourceOutput, + environmentVariables: { + CommitId: { value: unusedSourceAction.variables.commitId }, + }, + }); + pipeline.stage('Build').addAction(unusedBuildAction); + + expect(() => { + SynthUtils.synthesize(stack); + }).toThrow(/Cannot reference variables of action 'UnusedBitBucket', as that action was never added to a pipeline/); + }); + + test('fail if variable from unused action with custom namespace is referenced', () => { + const stack = new Stack(); + const pipeline = createBitBucketAndCodeBuildPipeline(stack, { + variablesNamespace: 'kornicameister', + }); + const unusedSourceOutput = new codepipeline.Artifact(); + const unusedSourceAction = new cpactions.CodeStarConnectionsSourceAction({ + actionName: 'UnusedBitBucket', + owner: 'aws', + repo: 'aws-cdk', + output: unusedSourceOutput, + connectionArn: 'arn:aws:codestar-connections:us-east-1:123456789012:connection/12345678-abcd-12ab-34cdef5678gh', + variablesNamespace: 'kornicameister', + }); + const unusedBuildAction = new cpactions.CodeBuildAction({ + actionName: 'UnusedCodeBuild', + project: new codebuild.PipelineProject(stack, 'UnusedProject'), + input: unusedSourceOutput, + environmentVariables: { + CommitId: { value: unusedSourceAction.variables.commitId }, + }, + }); + pipeline.stage('Build').addAction(unusedBuildAction); + + expect(() => { + SynthUtils.synthesize(stack); + }).toThrow(/Cannot reference variables of action 'UnusedBitBucket', as that action was never added to a pipeline/); }); }); -function createBitBucketAndCodeBuildPipeline(stack: Stack, props: Partial): void { +function createBitBucketAndCodeBuildPipeline( + stack: Stack, props: Partial = {}, +): codepipeline.Pipeline { const sourceOutput = new codepipeline.Artifact(); - new codepipeline.Pipeline(stack, 'Pipeline', { + const sourceAction = new cpactions.CodeStarConnectionsSourceAction({ + actionName: 'BitBucket', + owner: 'aws', + repo: 'aws-cdk', + output: sourceOutput, + connectionArn: 'arn:aws:codestar-connections:us-east-1:123456789012:connection/12345678-abcd-12ab-34cdef5678gh', + ...props, + }); + + return new codepipeline.Pipeline(stack, 'Pipeline', { stages: [ { stageName: 'Source', - actions: [ - new cpactions.CodeStarConnectionsSourceAction({ - actionName: 'BitBucket', - owner: 'aws', - repo: 'aws-cdk', - output: sourceOutput, - connectionArn: 'arn:aws:codestar-connections:us-east-1:123456789012:connection/12345678-abcd-12ab-34cdef5678gh', - ...props, - }), - ], + actions: [sourceAction], }, { stageName: 'Build', @@ -177,6 +290,9 @@ function createBitBucketAndCodeBuildPipeline(stack: Stack, props: Partial Date: Wed, 29 Dec 2021 07:30:24 -0600 Subject: [PATCH 2/8] fix(cloudfront-origins): policy not added for custom OAI (#18192) Closes [**#18185**](https://github.com/aws/aws-cdk/issues/18185): When creating an `S3Origin` without including an existing `Origin Access Identity`, an `OriginAccessIdentity` is created and added to the bucket's resource policy. However, since the adding to the resource policy is inside of the `if (!this.originAccessIdentity)`closure, custom OAI's are not added to the bucket policy by default. Since using `bucket.grantRead` creates an overly permissive policy (as noted in the source code comments), adding the OAI to the bucket policy by default for both cases would create a more consistent result. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-cloudfront-origins/lib/s3-origin.ts | 19 +++++---- .../test/integ.s3-origin-oai.expected.json | 39 +++++++++++++++++++ .../test/s3-origin.test.ts | 14 +++++++ 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-cloudfront-origins/lib/s3-origin.ts b/packages/@aws-cdk/aws-cloudfront-origins/lib/s3-origin.ts index 4bdd39a2e5888..4243c0dff1fb6 100644 --- a/packages/@aws-cdk/aws-cloudfront-origins/lib/s3-origin.ts +++ b/packages/@aws-cdk/aws-cloudfront-origins/lib/s3-origin.ts @@ -73,17 +73,16 @@ class S3BucketOrigin extends cloudfront.OriginBase { this.originAccessIdentity = new cloudfront.OriginAccessIdentity(oaiScope, oaiId, { comment: `Identity for ${options.originId}`, }); - - // Used rather than `grantRead` because `grantRead` will grant overly-permissive policies. - // Only GetObject is needed to retrieve objects for the distribution. - // This also excludes KMS permissions; currently, OAI only supports SSE-S3 for buckets. - // Source: https://aws.amazon.com/blogs/networking-and-content-delivery/serving-sse-kms-encrypted-content-from-s3-using-cloudfront/ - this.bucket.addToResourcePolicy(new iam.PolicyStatement({ - resources: [this.bucket.arnForObjects('*')], - actions: ['s3:GetObject'], - principals: [this.originAccessIdentity.grantPrincipal], - })); } + // Used rather than `grantRead` because `grantRead` will grant overly-permissive policies. + // Only GetObject is needed to retrieve objects for the distribution. + // This also excludes KMS permissions; currently, OAI only supports SSE-S3 for buckets. + // Source: https://aws.amazon.com/blogs/networking-and-content-delivery/serving-sse-kms-encrypted-content-from-s3-using-cloudfront/ + this.bucket.addToResourcePolicy(new iam.PolicyStatement({ + resources: [this.bucket.arnForObjects('*')], + actions: ['s3:GetObject'], + principals: [this.originAccessIdentity.grantPrincipal], + })); return super.bind(scope, options); } diff --git a/packages/@aws-cdk/aws-cloudfront-origins/test/integ.s3-origin-oai.expected.json b/packages/@aws-cdk/aws-cloudfront-origins/test/integ.s3-origin-oai.expected.json index 3d17b0944873e..0f0d9e291dc77 100644 --- a/packages/@aws-cdk/aws-cloudfront-origins/test/integ.s3-origin-oai.expected.json +++ b/packages/@aws-cdk/aws-cloudfront-origins/test/integ.s3-origin-oai.expected.json @@ -5,6 +5,45 @@ "UpdateReplacePolicy": "Retain", "DeletionPolicy": "Retain" }, + "BucketPolicyE9A3008A": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "Bucket83908E77" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Principal": { + "CanonicalUser": { + "Fn::GetAtt": [ + "OriginAccessIdentityDF1E3CAC", + "S3CanonicalUserId" + ] + } + }, + "Resource": { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + }, + "/*" + ] + ] + } + } + ], + "Version": "2012-10-17" + } + } + }, "OriginAccessIdentityDF1E3CAC": { "Type": "AWS::CloudFront::CloudFrontOriginAccessIdentity", "Properties": { diff --git a/packages/@aws-cdk/aws-cloudfront-origins/test/s3-origin.test.ts b/packages/@aws-cdk/aws-cloudfront-origins/test/s3-origin.test.ts index e8e2c4b2c41b9..9f817d1d7e986 100644 --- a/packages/@aws-cdk/aws-cloudfront-origins/test/s3-origin.test.ts +++ b/packages/@aws-cdk/aws-cloudfront-origins/test/s3-origin.test.ts @@ -82,6 +82,20 @@ describe('With bucket', () => { Comment: 'Identity for bucket provided by test', }, }); + + expect(stack).toHaveResourceLike('AWS::S3::BucketPolicy', { + PolicyDocument: { + Statement: [{ + Action: 's3:GetObject', + Principal: { + CanonicalUser: { 'Fn::GetAtt': ['OriginAccessIdentityDF1E3CAC', 'S3CanonicalUserId'] }, + }, + Resource: { + 'Fn::Join': ['', [{ 'Fn::GetAtt': ['Bucket83908E77', 'Arn'] }, '/*']], + }, + }], + }, + }); }); test('creates an OriginAccessIdentity and grants read permissions on the bucket', () => { From aa51b6cfbe4ca3c917067ab9421cdc663e413eec Mon Sep 17 00:00:00 2001 From: AWS CDK Automation <43080478+aws-cdk-automation@users.noreply.github.com> Date: Wed, 29 Dec 2021 15:34:52 +0100 Subject: [PATCH 3/8] chore: npm-check-updates && yarn upgrade (#18209) Ran npm-check-updates and yarn upgrade to keep the `yarn.lock` file up-to-date. --- package.json | 4 +- .../aws-apigatewayv2-authorizers/package.json | 2 +- .../package.json | 2 +- .../@aws-cdk/aws-cloudformation/package.json | 2 +- packages/@aws-cdk/aws-dynamodb/package.json | 2 +- packages/@aws-cdk/aws-ec2/package.json | 2 +- packages/@aws-cdk/aws-eks/package.json | 2 +- packages/@aws-cdk/aws-iam/package.json | 2 +- .../@aws-cdk/aws-lambda-nodejs/package.json | 2 +- packages/@aws-cdk/aws-lambda/package.json | 2 +- packages/@aws-cdk/aws-logs/package.json | 2 +- packages/@aws-cdk/aws-route53/package.json | 2 +- packages/@aws-cdk/aws-s3/package.json | 2 +- packages/@aws-cdk/aws-ses/package.json | 2 +- packages/@aws-cdk/core/package.json | 2 +- .../@aws-cdk/custom-resources/package.json | 2 +- packages/aws-cdk-lib/package.json | 2 +- yarn.lock | 256 +++++++++--------- 18 files changed, 146 insertions(+), 146 deletions(-) diff --git a/package.json b/package.json index 7fdd1c6da6154..462d2c605eabd 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "devDependencies": { "@yarnpkg/lockfile": "^1.1.0", + "cdk-generate-synthetic-examples": "^0.1.1", "conventional-changelog-cli": "^2.2.2", "fs-extra": "^9.1.0", "graceful-fs": "^4.2.8", @@ -27,8 +28,7 @@ "lerna": "^4.0.0", "patch-package": "^6.4.7", "standard-version": "^9.3.2", - "typescript": "~3.9.10", - "cdk-generate-synthetic-examples": "^0.1.1" + "typescript": "~3.9.10" }, "resolutions": { "string-width": "^4.2.3" diff --git a/packages/@aws-cdk/aws-apigatewayv2-authorizers/package.json b/packages/@aws-cdk/aws-apigatewayv2-authorizers/package.json index b79e93746ceb7..6eb99a2d3202a 100644 --- a/packages/@aws-cdk/aws-apigatewayv2-authorizers/package.json +++ b/packages/@aws-cdk/aws-apigatewayv2-authorizers/package.json @@ -85,7 +85,7 @@ "@aws-cdk/cdk-build-tools": "0.0.0", "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3" }, "dependencies": { diff --git a/packages/@aws-cdk/aws-certificatemanager/lambda-packages/dns_validated_certificate_handler/package.json b/packages/@aws-cdk/aws-certificatemanager/lambda-packages/dns_validated_certificate_handler/package.json index 6f88650c21512..9ac256541f4e3 100644 --- a/packages/@aws-cdk/aws-certificatemanager/lambda-packages/dns_validated_certificate_handler/package.json +++ b/packages/@aws-cdk/aws-certificatemanager/lambda-packages/dns_validated_certificate_handler/package.json @@ -29,7 +29,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/sinon": "^9.0.11", "@aws-cdk/cdk-build-tools": "0.0.0", "aws-sdk": "^2.596.0", diff --git a/packages/@aws-cdk/aws-cloudformation/package.json b/packages/@aws-cdk/aws-cloudformation/package.json index e3f5390886e91..18f689c9fa90f 100644 --- a/packages/@aws-cdk/aws-cloudformation/package.json +++ b/packages/@aws-cdk/aws-cloudformation/package.json @@ -79,7 +79,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "jest": "^27.4.5" }, diff --git a/packages/@aws-cdk/aws-dynamodb/package.json b/packages/@aws-cdk/aws-dynamodb/package.json index 9ae375727b136..da8456e834c22 100644 --- a/packages/@aws-cdk/aws-dynamodb/package.json +++ b/packages/@aws-cdk/aws-dynamodb/package.json @@ -84,7 +84,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "@types/sinon": "^9.0.11", "aws-sdk": "^2.848.0", diff --git a/packages/@aws-cdk/aws-ec2/package.json b/packages/@aws-cdk/aws-ec2/package.json index 5557ceecb1dca..8ef7a50a8b3c3 100644 --- a/packages/@aws-cdk/aws-ec2/package.json +++ b/packages/@aws-cdk/aws-ec2/package.json @@ -86,7 +86,7 @@ "@aws-cdk/cloud-assembly-schema": "0.0.0", "@aws-cdk/cx-api": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "jest": "^27.4.5" }, diff --git a/packages/@aws-cdk/aws-eks/package.json b/packages/@aws-cdk/aws-eks/package.json index 414dac1ffbd48..c28d366a31447 100644 --- a/packages/@aws-cdk/aws-eks/package.json +++ b/packages/@aws-cdk/aws-eks/package.json @@ -84,7 +84,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "@types/sinon": "^9.0.11", "@types/yaml": "1.9.6", diff --git a/packages/@aws-cdk/aws-iam/package.json b/packages/@aws-cdk/aws-iam/package.json index c725e84b14edb..bb376f95e3bab 100644 --- a/packages/@aws-cdk/aws-iam/package.json +++ b/packages/@aws-cdk/aws-iam/package.json @@ -85,7 +85,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "@types/sinon": "^9.0.11", "jest": "^27.4.5", diff --git a/packages/@aws-cdk/aws-lambda-nodejs/package.json b/packages/@aws-cdk/aws-lambda-nodejs/package.json index 1c00cd14bf6f3..bbb6b09e99693 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/package.json +++ b/packages/@aws-cdk/aws-lambda-nodejs/package.json @@ -78,7 +78,7 @@ "@aws-cdk/pkglint": "0.0.0", "@types/jest": "^27.0.3", "delay": "5.0.0", - "esbuild": "^0.14.8" + "esbuild": "^0.14.9" }, "dependencies": { "@aws-cdk/aws-lambda": "0.0.0", diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 90f171113516b..b5c73d139aa42 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -89,7 +89,7 @@ "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/cfnspec": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "@types/lodash": "^4.14.178", "jest": "^27.4.5", diff --git a/packages/@aws-cdk/aws-logs/package.json b/packages/@aws-cdk/aws-logs/package.json index f0c08d52caadd..a30de23a679a7 100644 --- a/packages/@aws-cdk/aws-logs/package.json +++ b/packages/@aws-cdk/aws-logs/package.json @@ -84,7 +84,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "@types/sinon": "^9.0.11", "aws-sdk": "^2.848.0", diff --git a/packages/@aws-cdk/aws-route53/package.json b/packages/@aws-cdk/aws-route53/package.json index e56a3464f8559..b3e12466c510b 100644 --- a/packages/@aws-cdk/aws-route53/package.json +++ b/packages/@aws-cdk/aws-route53/package.json @@ -84,7 +84,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "aws-sdk": "^2.848.0", "jest": "^27.4.5" diff --git a/packages/@aws-cdk/aws-s3/package.json b/packages/@aws-cdk/aws-s3/package.json index 585e80b894276..86bbb57e431e8 100644 --- a/packages/@aws-cdk/aws-s3/package.json +++ b/packages/@aws-cdk/aws-s3/package.json @@ -84,7 +84,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "jest": "^27.4.5" }, diff --git a/packages/@aws-cdk/aws-ses/package.json b/packages/@aws-cdk/aws-ses/package.json index 7dab505b39da2..611d336d3dbfa 100644 --- a/packages/@aws-cdk/aws-ses/package.json +++ b/packages/@aws-cdk/aws-ses/package.json @@ -77,7 +77,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/jest": "^27.0.3", "jest": "^27.4.5" }, diff --git a/packages/@aws-cdk/core/package.json b/packages/@aws-cdk/core/package.json index 6328493355c2d..ffcd1f5c9e3fe 100644 --- a/packages/@aws-cdk/core/package.json +++ b/packages/@aws-cdk/core/package.json @@ -178,7 +178,7 @@ "@aws-cdk/cdk-build-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/fs-extra": "^8.1.2", "@types/jest": "^27.0.3", "@types/lodash": "^4.14.178", diff --git a/packages/@aws-cdk/custom-resources/package.json b/packages/@aws-cdk/custom-resources/package.json index a6e4f9aba808f..e14d766ee5367 100644 --- a/packages/@aws-cdk/custom-resources/package.json +++ b/packages/@aws-cdk/custom-resources/package.json @@ -80,7 +80,7 @@ "@aws-cdk/cdk-integ-tools": "0.0.0", "@aws-cdk/cfn2ts": "0.0.0", "@aws-cdk/pkglint": "0.0.0", - "@types/aws-lambda": "^8.10.88", + "@types/aws-lambda": "^8.10.89", "@types/fs-extra": "^8.1.2", "@types/jest": "^27.0.3", "@types/sinon": "^9.0.11", diff --git a/packages/aws-cdk-lib/package.json b/packages/aws-cdk-lib/package.json index 600eff0bd95ae..f6fd2eca54f1e 100644 --- a/packages/aws-cdk-lib/package.json +++ b/packages/aws-cdk-lib/package.json @@ -346,7 +346,7 @@ "@types/fs-extra": "^8.1.2", "@types/node": "^10.17.60", "constructs": "^3.3.69", - "esbuild": "^0.14.8", + "esbuild": "^0.14.9", "fs-extra": "^9.1.0", "ts-node": "^9.1.1", "typescript": "~3.8.3" diff --git a/yarn.lock b/yarn.lock index ffa1846adde18..e478f529f0427 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1582,10 +1582,10 @@ dependencies: "@types/glob" "*" -"@types/aws-lambda@^8.10.88": - version "8.10.88" - resolved "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.88.tgz#1f18ac2e15be30376e86a688a943390e7d6683e5" - integrity sha512-Gbdr5tmGMGV1bgWDEfgNnfqtS9YVKDCkyAgYPmYIeEQFTSjU+VzVoE0Gc1MyrzREdk3Iu5daUCRU9eQL5s+iYQ== +"@types/aws-lambda@^8.10.89": + version "8.10.89" + resolved "https://registry.npmjs.org/@types/aws-lambda/-/aws-lambda-8.10.89.tgz#22617ecc1eef9571abebb50c947553362da6051b" + integrity sha512-jwtSuEZj4rY4R2pAEOXi+RutS8RWbwMzoGlRVukdyOpnfqA/XPkAf8QoGWmg4o/UaNpQ8Mj0Xhkp5SZ1t/Zq4Q== "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": version "7.1.17" @@ -2009,9 +2009,9 @@ acorn@^7.1.1, acorn@^7.4.0: integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== acorn@^8.2.4, acorn@^8.4.1: - version "8.6.0" - resolved "https://registry.npmjs.org/acorn/-/acorn-8.6.0.tgz#e3692ba0eb1a0c83eaa4f37f5fa7368dd7142895" - integrity sha512-U1riIR+lBSNi3IbxtaHOIKdH8sLFv3NYfNv8sg7ZsNhcfl4HF2++BfqqrNAxoCLQW1iiylOj76ecnaUxz+z9yw== + version "8.7.0" + resolved "https://registry.npmjs.org/acorn/-/acorn-8.7.0.tgz#90951fde0f8f09df93549481e5fc141445b791cf" + integrity sha512-V/LGr1APy+PXIwKebEWrkZPwoeoF+w1jiOBUmuxuiUIaOHtob8Qc9BTrYo7VuI5fR8tqsy+buA2WFooR5olqvQ== add-stream@^1.0.0: version "1.0.0" @@ -2598,9 +2598,9 @@ camelcase@^6.2.0, camelcase@^6.2.1: integrity sha512-tVI4q5jjFV5CavAU8DXfza/TJcZutVKo/5Foskmsqcm0MsL91moHvwiGNnqaa2o6PF/7yT5ikDRcVcl8Rj6LCA== caniuse-lite@^1.0.30001286: - version "1.0.30001292" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001292.tgz#4a55f61c06abc9595965cfd77897dc7bc1cdc456" - integrity sha512-jnT4Tq0Q4ma+6nncYQVe7d73kmDmE9C3OGTx3MvW7lBM/eY1S1DZTMBON7dqV481RhNiS5OxD7k9JQvmDOTirw== + version "1.0.30001294" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001294.tgz#4849f27b101fd59ddee3751598c663801032533d" + integrity sha512-LiMlrs1nSKZ8qkNhpUf5KD0Al1KCBE3zaT7OLOwEkagXMEDij98SiOovn9wxVGQpklk9vVC/pUSqgYmkmKOS8g== case@1.6.3, case@^1.6.3: version "1.6.3" @@ -2922,9 +2922,9 @@ console-control-strings@^1.0.0, console-control-strings@~1.1.0: integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= constructs@^3.3.69: - version "3.3.169" - resolved "https://registry.npmjs.org/constructs/-/constructs-3.3.169.tgz#bbedb666a191486306a81bd5887c60e687b550c3" - integrity sha512-DMgpUU7+91LeN+7/gZoo1zaEuAFttZoP9me80zFMRyPYlDcISq+PfGqCKhAqP2um2fQs8OIhNOZ7bGb8PEOoFg== + version "3.3.172" + resolved "https://registry.npmjs.org/constructs/-/constructs-3.3.172.tgz#04396a7b6a3aa544e05a0a401938ea72c0aab2d2" + integrity sha512-VspQBS+pHn/UkcR36RHSJR8kSUquPS2vN6zuEfx5/9ADETfrY9X8/8ecLALfehGzcvuxllU7s7dPQ+W4KDYd1w== conventional-changelog-angular@^5.0.12: version "5.0.13" @@ -3569,9 +3569,9 @@ ecc-jsbn@~0.1.1: safer-buffer "^2.1.0" electron-to-chromium@^1.4.17: - version "1.4.28" - resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.28.tgz#fef0e92e281df6d568f482d8d53c34ca5374de48" - integrity sha512-Gzbf0wUtKfyPaqf0Plz+Ctinf9eQIzxEqBHwSvbGfeOm9GMNdLxyu1dNiCUfM+x6r4BE0xUJNh3Nmg9gfAtTmg== + version "1.4.29" + resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.29.tgz#a9b85ab888d0122124c9647c04d8dd246fae94b6" + integrity sha512-N2Jbwxo5Rum8G2YXeUxycs1sv4Qme/ry71HG73bv8BvZl+I/4JtRgK/En+ST/Wh/yF1fqvVCY4jZBgMxnhjtBA== emittery@^0.8.1: version "0.8.1" @@ -3726,119 +3726,119 @@ es6-weak-map@^2.0.3: es6-iterator "^2.0.3" es6-symbol "^3.1.1" -esbuild-android-arm64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.8.tgz#69324e08ba68c7d9a541e7b825d7235b83e17bd6" - integrity sha512-tAEoSHnPBSH0cCAFa/aYs3LPsoTY4SwsP6wDKi4PaelbQYNJjqNpAeweyJ8l98g1D6ZkLyqsHbkYj+209sezkA== - -esbuild-darwin-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.8.tgz#7176b692b9de746ba2f9dd4dd81dc4f1b7670786" - integrity sha512-t7p7WzTb+ybiD/irkMt5j/NzB+jY+8yPTsrXk5zCOH1O7DdthRnAUJ7pJPwImdL7jAGRbLtYRxUPgCHs/0qUPw== - -esbuild-darwin-arm64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.8.tgz#59167584e58428877e48e05c4cca58755f843327" - integrity sha512-5FeaT2zMUajKnBwUMSsjZev5iA38YHrDmXhkOCwZQIFUvhqojinqCrvv/X7dyxb1987bcY9KGwJ+EwDwd922HQ== - -esbuild-freebsd-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.8.tgz#00b7d6e00abba9c2eccc9acd576c796333671e9c" - integrity sha512-pGHBLSf7ynfyDZXUtbq/GsA2VIwQlWXrUj1AMcE0id47mRdEUM8/1ZuqMGZx63hRnNgtK9zNJ8OIu2c7qq76Qw== - -esbuild-freebsd-arm64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.8.tgz#57f0cd5a1cb37fa2c0e84e780677fe62f1e8c894" - integrity sha512-g4GgAnrx6Gh1BjKJjJWgPnOR4tW2FcAx9wFvyUjRsIjB35gT+aAFR+P/zStu5OG9LnbS8Pvjd4wS68QIXk+2dA== - -esbuild-linux-32@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.8.tgz#bbf3e5d3fb30f949030d0c2241ac93a172917d56" - integrity sha512-wPfQJadF5vTzriw/B8Ide74PeAJlZW7czNx3NIUHkHlXb+En1SeIqNzl6jG9DuJUl57xD9Ucl9YJFEkFeX8eLg== - -esbuild-linux-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.8.tgz#08631e9e0da613603bcec782f29fecbc6f4596de" - integrity sha512-+RNuLk9RhRDL2kG+KTEYl5cIgF6AGLkRnKKWEu9DpCZaickONEqrKyQSVn410Hj105DLdW6qvIXQQHPycJhExg== - -esbuild-linux-arm64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.8.tgz#206d39c8dfbb7c72aa2f5fc52f7402b5b8a77366" - integrity sha512-BtWoKNYul9UoxUvQUSdSrvSmJyFL1sGnNPTSqWCg1wMe4kmc8UY2yVsXSSkKO8N2jtHxlgFyz/XhvNBzEwGVcw== - -esbuild-linux-arm@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.8.tgz#e28e70420d187f5e403bfa4a72df676d53d707fd" - integrity sha512-HIct38SvUAIJbiTwV/PVQroimQo96TGtzRDAEZxTorB4vsAj1r8bd0keXExPU4RH7G0zIqC4loQQpWYL+nH4Vg== - -esbuild-linux-mips64le@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.8.tgz#04997ac1a0df794a4d5e04d78015863d48490590" - integrity sha512-0DxnCl9XTvaQtsX6Qa+Phr5i9b04INwwSv2RbQ2UWRLoQ/037iaFzbmuhgrcmaGOcRwPkCa+4Qo5EgI01MUgsQ== - -esbuild-linux-ppc64le@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.8.tgz#1827378feff9702c156047ba118c1f3bd74da67e" - integrity sha512-Uzr/OMj97Q0qoWLXCvXCKUY/z1SNI4iSZEuYylM5Nd71HGStL32XWq/MReJ0PYMvUMKKJicKSKw2jWM1uBQ84Q== - -esbuild-linux-s390x@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.8.tgz#200ac44cda59b81135b325c3a29d016969650876" - integrity sha512-vURka7aCA5DrRoOqOn6pXYwFlDSoQ4qnqam8AC0Ikn6tibutuhgar6M3Ek2DCuz9yqd396mngdYr5A8x2TPkww== - -esbuild-netbsd-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.8.tgz#8159d8eae111f80ea6e4cbfa5d4cf658388a72d4" - integrity sha512-tjyDak2/pp0VUAhBW6/ueuReMd5qLHNlisXl5pq0Xn0z+kH9urA/t1igm0JassWbdMz123td5ZEQWoD9KbtOAw== - -esbuild-openbsd-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.8.tgz#2a9498d881a3ab94927c724f34dd1160eef1f3b8" - integrity sha512-zAKKV15fIyAuDDga5rQv0lW2ufBWj/OCjqjDBb3dJf5SfoAi/DMIHuzmkKQeDQ+oxt9Rp1D7ZOlOBVflutFTqQ== - -esbuild-sunos-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.8.tgz#2447de7d79848ad528c7d44caab4938eb8f5a0cc" - integrity sha512-xV41Wa8imziM/2dbWZjLKQbIETRgo5dE0oc/uPsgaecJhsrdA0VkGa/V432LJSUYv967xHDQdoRRl5tr80+NnQ== - -esbuild-windows-32@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.8.tgz#3287281552d7e4c851b3106940ff5826f518043e" - integrity sha512-AxpdeLKQSyCZo7MzdOyV4OgEbEJcjnrS/2niAjbHESbjuS5P1DN/5vZoJ/JSWDVa/40OkBuHBhAXMx1HK3UDsg== - -esbuild-windows-64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.8.tgz#b4052868438b4f17b5c2a908cf344ed2bd267c38" - integrity sha512-/3pllNoy8mrz/E1rYalwiwwhzJBrYQhEapwAteHZbFVhGzYuB8F80e8x5eA8dhFHxDiZh1VzK+hREwwSt8UTQA== - -esbuild-windows-arm64@0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.8.tgz#512d06097cb4b848526a37c48a47223f1c6cc667" - integrity sha512-lTm5naoNgaUvzIiax3XYIEebqwr3bIIEEtqUhzQ2UQ+JMBmvhr02w3sJIJqF3axTX6TgWrC1OtM7DYNvFG+aXA== - -esbuild@^0.14.8: - version "0.14.8" - resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.14.8.tgz#f60a07ca9400d61d09a98f96d666c50613972550" - integrity sha512-stMsCBmxwaMpeK8GC/49L/cRGIwsHwoEN7Twk5zDTHlm/63c0KXFKzDC8iM2Mi3fyCKwS002TAH6IlAvqR6t3g== +esbuild-android-arm64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.9.tgz#81491ba904eb0dd636e372d3d95fa8424a0d9e95" + integrity sha512-VpSCuUR07G4Re/5QzqtdxS5ZgxkCRyzu4Kf5SH1/EkXzRGeoWQt8xirkOMK58pfmg/FlS/fQNgwl3Txej4LoVg== + +esbuild-darwin-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.14.9.tgz#085f6ba06c52c747696903de3755f50ffa50c89d" + integrity sha512-F/RcRHMG5ccAL8n9VIy8ZC4D0IHZrN/1IhHQbY4qPXrMlh42FucR0TW4lr3vdHF3caaId1jdDSQQJ7jXR+ZC5Q== + +esbuild-darwin-arm64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.9.tgz#6b61afa38640beaa12a1da679640a921a3ebf741" + integrity sha512-3ue+1T4FR5TaAu4/V1eFMG8Uwn0pgAwQZb/WwL1X78d5Cy8wOVQ67KNH1lsjU+y/9AcwMKZ9x0GGNxBB4a1Rbw== + +esbuild-freebsd-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.9.tgz#6fae644d8aa52e809d6238cb364ae1dd31f05b87" + integrity sha512-0YEjWt6ijaf5Y3Q50YS1lZxuWZWMV/T7atQEuQnF8ioq5jamrVr8j1TZ9+rxcLgH1lBMsXj8IwW+6BleXredEg== + +esbuild-freebsd-arm64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.9.tgz#03d4dff441c043f94bdce93b1645b4b927099fb4" + integrity sha512-82w5qMgEeYvf8+vX/2KE5TOZf8rv8VK4TFiK6lDzdgdwwmBU5C8kdT3rO5Llan2K2LKndrou1eyi/fHwFcwPJQ== + +esbuild-linux-32@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.9.tgz#4301aa9824a51b198bb299c226a046f42e1b42e1" + integrity sha512-eu8J8HNpco7Mkd7T7djQRzGBeuve41kbXRxFHOwwbZXMNQojXjBqLuradi5i/Vsw+CA4G/yVpmJI2S75Cit2mQ== + +esbuild-linux-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.9.tgz#b7b8aaa9e7d395d3ee9af6badf88033d7da4e924" + integrity sha512-WoEI+R6/PLZAxS7XagfQMFgRtLUi5cjqqU9VCfo3tnWmAXh/wt8QtUfCVVCcXVwZLS/RNvI19CtfjlrJU61nOg== + +esbuild-linux-arm64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.9.tgz#535e61282156db2c10443529cf33498d710b2ec2" + integrity sha512-joUE0yQgWMDkQqBx3+6SdNCVZ10F1O4+WM94moghvhdTzkYpECIc/WvfqMF/w0V8Hecw3QJ7vugO7jsFlXXd4Q== + +esbuild-linux-arm@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.9.tgz#abc456952f89deb5ec5563335793781ea5ce1c7e" + integrity sha512-d3k1ZPREjaKYyhsS8x3jvc4ekjIZ8SmuihP60mrN1f6p5y07NKWw9i0OWD1p6hy+7g6cjMWq00tstMIikGB9Yg== + +esbuild-linux-mips64le@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.9.tgz#372c317a80df4e5e1b227bacb57a497897e31af5" + integrity sha512-ZAuheiDRo2c4rxx8GUTEwPvos0zUwCYjP9K2WfCSmDL6m3RpaObCQhZghrDuoIUwvc/D6SWuABsKE9VzogsltQ== + +esbuild-linux-ppc64le@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.9.tgz#f08a9aa4318ae5ec9ef7ee8c9016dd5027392a81" + integrity sha512-Pm8FeG5l314k3a2mbu3SAc5E2eLFuGUsGiSlw8V6xtA4whxJ7rit7951w9jBhz+1Vqqtqprg2IYTng3j2CGhVw== + +esbuild-linux-s390x@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.9.tgz#85a921fb714f5aa7cdddb192f211ea20e46b2fd0" + integrity sha512-G8FNZygV82N1/LOfPD8ZX7Mn1dPpKKPrZc93ebSJ8/VgNIafOAhV5vaeK1lhcx6ZSu+jJU/UyQQMG1CIvHRIaw== + +esbuild-netbsd-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.9.tgz#2b4d4267a8bdf7e340655e12a1307cf89de275c0" + integrity sha512-b7vPrn5XN0GRtNAQ3w+gq8AwUfWSRBkcPAdA5UUT5rkrw7wKFyMqi2/zREBc/Knu5YOsLmZPQSoM8QL6qy79cg== + +esbuild-openbsd-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.9.tgz#1aa3b39b717040d506cdafcdd417aadf6530cf59" + integrity sha512-w95Rt/vmVhZWfzZmeoMIHxbFiOFDmxC7GEdnCbDTXX2vlwKu+CIDIKOgWW+R1T2JqTNo5tu9dRkngKZMfbUo/A== + +esbuild-sunos-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.9.tgz#d2c8e091a13dbff8117403c76b637a7647402e8c" + integrity sha512-mzgmQZAVGo+uLkQXTY0viqVSEQKesmR5OEMMq1jM/2jucbZUcyaq8dVKRIWJJEzwNgZ6MpeOpshUtOzGxxy8ag== + +esbuild-windows-32@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.9.tgz#7f778a74de7849c14e9bbb4e749974d508c9be58" + integrity sha512-sYHEJLwdDJpjjSUyIGqPC1GRXl0Z/YT1K85Tcrv4iqZEXFR0rT7sTV+E0XC911FbTJHfuAdUJixkwAQeLMdrUg== + +esbuild-windows-64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.9.tgz#20d158b316488dd073b0b02cd32130a1040820ee" + integrity sha512-xJTpyFzpH51LGlVR2C3P+Gpnjujsx5kEtJj5V/x8TyD94VW+EpszyND/pay15CIF64pWywyQt2jmGUDl6kzkEw== + +esbuild-windows-arm64@0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.9.tgz#3ab19cac75965f509b20ad31b06122c58318ac37" + integrity sha512-NKPPsYVlHqdF0yMuMJrjuAzqS/BHrMXZ8TN1Du+Pgi8KkmxzNXRPDHQV0NPPJ+Z7Lp09joEHSz1zrvQRs1j6jw== + +esbuild@^0.14.9: + version "0.14.9" + resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.14.9.tgz#f48bedfbdb21051b0d47e299bc2eb3110a2301d6" + integrity sha512-uuT3kFsfUvzNW6I2RKKIHuCvutY/U9KFcAP6emUm98WvBhyhEr5vGkZLeN3r3vXfoykl+7xekAH8Ky09LXBd0Q== optionalDependencies: - esbuild-android-arm64 "0.14.8" - esbuild-darwin-64 "0.14.8" - esbuild-darwin-arm64 "0.14.8" - esbuild-freebsd-64 "0.14.8" - esbuild-freebsd-arm64 "0.14.8" - esbuild-linux-32 "0.14.8" - esbuild-linux-64 "0.14.8" - esbuild-linux-arm "0.14.8" - esbuild-linux-arm64 "0.14.8" - esbuild-linux-mips64le "0.14.8" - esbuild-linux-ppc64le "0.14.8" - esbuild-linux-s390x "0.14.8" - esbuild-netbsd-64 "0.14.8" - esbuild-openbsd-64 "0.14.8" - esbuild-sunos-64 "0.14.8" - esbuild-windows-32 "0.14.8" - esbuild-windows-64 "0.14.8" - esbuild-windows-arm64 "0.14.8" + esbuild-android-arm64 "0.14.9" + esbuild-darwin-64 "0.14.9" + esbuild-darwin-arm64 "0.14.9" + esbuild-freebsd-64 "0.14.9" + esbuild-freebsd-arm64 "0.14.9" + esbuild-linux-32 "0.14.9" + esbuild-linux-64 "0.14.9" + esbuild-linux-arm "0.14.9" + esbuild-linux-arm64 "0.14.9" + esbuild-linux-mips64le "0.14.9" + esbuild-linux-ppc64le "0.14.9" + esbuild-linux-s390x "0.14.9" + esbuild-netbsd-64 "0.14.9" + esbuild-openbsd-64 "0.14.9" + esbuild-sunos-64 "0.14.9" + esbuild-windows-32 "0.14.9" + esbuild-windows-64 "0.14.9" + esbuild-windows-arm64 "0.14.9" escalade@^3.1.1: version "3.1.1" From c071367def4382c630057546c74fa56f00d9294c Mon Sep 17 00:00:00 2001 From: Kaizen Conroy <36202692+kaizen3031593@users.noreply.github.com> Date: Wed, 29 Dec 2021 12:22:13 -0500 Subject: [PATCH 4/8] feat(glue): support partition index on tables (#17998) This PR adds support for creating partition indexes on tables via custom resources. It offers two different ways to create indexes: ```ts // via table definition const table = new glue.Table(this, 'Table', { database, bucket, tableName: 'table', columns, partitionKeys, partitionIndexes: [{ indexName: 'my-index', keyNames: ['month'], }], dataFormat: glue.DataFormat.CSV, }); ``` ```ts // or as a function table.AddPartitionIndex([{ indexName: 'my-other-index', keyNames: ['month', 'year'], }); ``` I also refactored the format of some tests, which is what accounts for the large diff in `test.table.ts`. Motivation: Creating partition indexes on a table is something you can do via the console, but is not an exposed property in cloudformation. In this case, I think it makes sense to support this feature via custom resources as it will significantly reduce the customer pain of either provisioning a custom resource with correct permissions or manually going into the console after resource creation. Supporting this feature allows for synth-time checks and dependency chaining for multiple indexes (reason detailed in the FAQ) which removes a rather sharp edge for users provisioning custom resource indexes themselves. FAQ: Why do we need to chain dependencies between different Partition Index Custom Resources? - Because Glue only allows 1 index to be created or deleted simultaneously per table. Without dependencies the resources will try to create partition indexes simultaneously and the second sdk call with be dropped. Why is it called `partitionIndexes`? Is that really how you pluralize index? - [Yesish](https://www.nasdaq.com/articles/indexes-or-indices-whats-the-deal-2016-05-12). If you hate it it can be `partitionIndices`. Why is `keyNames` of type `string[]` and not `Column[]`? `PartitionKey` is of type `Column[]` and partition indexes must be a subset of partition keys... - This could be a debate. But my argument is that the pattern I see for defining a Table is to define partition keys inline and not declare them each as variables. It would be pretty clunky from a UX perspective: ```ts const key1 = { name: 'mykey', type: glue.Schema.STRING }; const key2 = { name: 'mykey2', type: glue.Schema.STRING }; const key3 = { name: 'mykey3', type: glue.Schema.STRING }; new glue.Table(this, 'table', { database, bucket, tableName: 'table', columns, partitionKeys: [key1, key2, key3], partitionIndexes: [key1, key2], dataFormat: glue.DataFormat.CSV, }); ``` Why are there 2 different checks for having > 3 partition indexes? - It's possible someone decides to define 3 indexes in the definition and then try to add another with `table.addPartitionIndex()`. This would be a nasty deploy time error, its better if it is synth time. It's also possible someone decides to define 4 indexes in the definition. It's better to fast-fail here before we create 3 custom resources. What if I deploy a table, manually add 3 partition indexes, and then try to call `table.addPartitionIndex()` and update the stack? Will that still be a synth time failure? - Sorry, no. Why do we need to generate names? - We don't. I just thought it would be helpful. Why is `grantToUnderlyingResources` public? - I thought it would be helpful. Some permissions need to be added to the table, the database, and the catalog. Closes #17589. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-glue/README.md | 48 +- packages/@aws-cdk/aws-glue/lib/table.ts | 132 ++- packages/@aws-cdk/aws-glue/package.json | 2 + .../test/integ.partition-index.expected.json | 594 +++++++++++ .../aws-glue/test/integ.partition-index.ts | 75 ++ .../@aws-cdk/aws-glue/test/integ.table.ts | 5 +- packages/@aws-cdk/aws-glue/test/table.test.ts | 943 ++++++++++-------- 7 files changed, 1379 insertions(+), 420 deletions(-) create mode 100644 packages/@aws-cdk/aws-glue/test/integ.partition-index.expected.json create mode 100644 packages/@aws-cdk/aws-glue/test/integ.partition-index.ts diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md index 069b572b6cd71..639198c67ed9a 100644 --- a/packages/@aws-cdk/aws-glue/README.md +++ b/packages/@aws-cdk/aws-glue/README.md @@ -194,7 +194,7 @@ new glue.Table(this, 'MyTable', { By default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`: -### Partitions +### Partition Keys To improve query performance, a table can specify `partitionKeys` on which data is stored and queried separately. For example, you might partition a table by `year` and `month` to optimize queries based on a time window: @@ -218,6 +218,52 @@ new glue.Table(this, 'MyTable', { }); ``` +### Partition Indexes + +Another way to improve query performance is to specify partition indexes. If no partition indexes are +present on the table, AWS Glue loads all partitions of the table and filters the loaded partitions using +the query expression. The query takes more time to run as the number of partitions increase. With an +index, the query will try to fetch a subset of the partitions instead of loading all partitions of the +table. + +The keys of a partition index must be a subset of the partition keys of the table. You can have a +maximum of 3 partition indexes per table. To specify a partition index, you can use the `partitionIndexes` +property: + +```ts +declare const myDatabase: glue.Database; +new glue.Table(this, 'MyTable', { + database: myDatabase, + tableName: 'my_table', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'year', + type: glue.Schema.SMALL_INT, + }, { + name: 'month', + type: glue.Schema.SMALL_INT, + }], + partitionIndexes: [{ + indexName: 'my-index', // optional + keyNames: ['year'], + }], // supply up to 3 indexes + dataFormat: glue.DataFormat.JSON, +}); +``` + +Alternatively, you can call the `addPartitionIndex()` function on a table: + +```ts +declare const myTable: glue.Table; +myTable.addPartitionIndex({ + indexName: 'my-index', + keyNames: ['year'], +}); +``` + ## [Encryption](https://docs.aws.amazon.com/athena/latest/ug/encryption.html) You can enable encryption on a Table's data: diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts index 14ff98dd3b3c4..1b17da32e5454 100644 --- a/packages/@aws-cdk/aws-glue/lib/table.ts +++ b/packages/@aws-cdk/aws-glue/lib/table.ts @@ -1,13 +1,33 @@ import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; import * as s3 from '@aws-cdk/aws-s3'; -import { ArnFormat, Fn, IResource, Resource, Stack } from '@aws-cdk/core'; +import { ArnFormat, Fn, IResource, Names, Resource, Stack } from '@aws-cdk/core'; +import * as cr from '@aws-cdk/custom-resources'; +import { AwsCustomResource } from '@aws-cdk/custom-resources'; import { Construct } from 'constructs'; import { DataFormat } from './data-format'; import { IDatabase } from './database'; import { CfnTable } from './glue.generated'; import { Column } from './schema'; +/** + * Properties of a Partition Index. + */ +export interface PartitionIndex { + /** + * The name of the partition index. + * + * @default - a name will be generated for you. + */ + readonly indexName?: string; + + /** + * The partition key names that comprise the partition + * index. The names must correspond to a name in the + * table's partition keys. + */ + readonly keyNames: string[]; +} export interface ITable extends IResource { /** * @attribute @@ -102,7 +122,16 @@ export interface TableProps { * * @default table is not partitioned */ - readonly partitionKeys?: Column[] + readonly partitionKeys?: Column[]; + + /** + * Partition indexes on the table. A maximum of 3 indexes + * are allowed on a table. Keys in the index must be part + * of the table's partition keys. + * + * @default table has no partition indexes + */ + readonly partitionIndexes?: PartitionIndex[]; /** * Storage type of the table's data. @@ -230,6 +259,18 @@ export class Table extends Resource implements ITable { */ public readonly partitionKeys?: Column[]; + /** + * This table's partition indexes. + */ + public readonly partitionIndexes?: PartitionIndex[]; + + /** + * Partition indexes must be created one at a time. To avoid + * race conditions, we store the resource and add dependencies + * each time a new partition index is created. + */ + private partitionIndexCustomResources: AwsCustomResource[] = []; + constructor(scope: Construct, id: string, props: TableProps) { super(scope, id, { physicalName: props.tableName, @@ -287,6 +328,77 @@ export class Table extends Resource implements ITable { resourceName: `${this.database.databaseName}/${this.tableName}`, }); this.node.defaultChild = tableResource; + + // Partition index creation relies on created table. + if (props.partitionIndexes) { + this.partitionIndexes = props.partitionIndexes; + this.partitionIndexes.forEach((index) => this.addPartitionIndex(index)); + } + } + + /** + * Add a partition index to the table. You can have a maximum of 3 partition + * indexes to a table. Partition index keys must be a subset of the table's + * partition keys. + * + * @see https://docs.aws.amazon.com/glue/latest/dg/partition-indexes.html + */ + public addPartitionIndex(index: PartitionIndex) { + const numPartitions = this.partitionIndexCustomResources.length; + if (numPartitions >= 3) { + throw new Error('Maximum number of partition indexes allowed is 3'); + } + this.validatePartitionIndex(index); + + const indexName = index.indexName ?? this.generateIndexName(index.keyNames); + const partitionIndexCustomResource = new cr.AwsCustomResource(this, `partition-index-${indexName}`, { + onCreate: { + service: 'Glue', + action: 'createPartitionIndex', + parameters: { + DatabaseName: this.database.databaseName, + TableName: this.tableName, + PartitionIndex: { + IndexName: indexName, + Keys: index.keyNames, + }, + }, + physicalResourceId: cr.PhysicalResourceId.of( + indexName, + ), + }, + policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ + resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE, + }), + }); + this.grantToUnderlyingResources(partitionIndexCustomResource, ['glue:UpdateTable']); + + // Depend on previous partition index if possible, to avoid race condition + if (numPartitions > 0) { + this.partitionIndexCustomResources[numPartitions-1].node.addDependency(partitionIndexCustomResource); + } + this.partitionIndexCustomResources.push(partitionIndexCustomResource); + } + + private generateIndexName(keys: string[]): string { + const prefix = keys.join('-') + '-'; + const uniqueId = Names.uniqueId(this); + const maxIndexLength = 80; // arbitrarily specified + const startIndex = Math.max(0, uniqueId.length - (maxIndexLength - prefix.length)); + return prefix + uniqueId.substring(startIndex); + } + + private validatePartitionIndex(index: PartitionIndex) { + if (index.indexName !== undefined && (index.indexName.length < 1 || index.indexName.length > 255)) { + throw new Error(`Index name must be between 1 and 255 characters, but got ${index.indexName.length}`); + } + if (!this.partitionKeys || this.partitionKeys.length === 0) { + throw new Error('The table must have partition keys to create a partition index'); + } + const keyNames = this.partitionKeys.map(pk => pk.name); + if (!index.keyNames.every(k => keyNames.includes(k))) { + throw new Error(`All index keys must also be partition keys. Got ${index.keyNames} but partition key names are ${keyNames}`); + } } /** @@ -336,6 +448,22 @@ export class Table extends Resource implements ITable { }); } + /** + * Grant the given identity custom permissions to ALL underlying resources of the table. + * Permissions will be granted to the catalog, the database, and the table. + */ + public grantToUnderlyingResources(grantee: iam.IGrantable, actions: string[]) { + return iam.Grant.addToPrincipal({ + grantee, + resourceArns: [ + this.tableArn, + this.database.catalogArn, + this.database.databaseArn, + ], + actions, + }); + } + private getS3PrefixForGrant() { return this.s3Prefix + '*'; } diff --git a/packages/@aws-cdk/aws-glue/package.json b/packages/@aws-cdk/aws-glue/package.json index 509b196a45dea..99d837bdb1fb9 100644 --- a/packages/@aws-cdk/aws-glue/package.json +++ b/packages/@aws-cdk/aws-glue/package.json @@ -99,6 +99,7 @@ "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", "constructs": "^3.3.69" }, "homepage": "https://github.com/aws/aws-cdk", @@ -113,6 +114,7 @@ "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", "constructs": "^3.3.69" }, "engines": { diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.expected.json b/packages/@aws-cdk/aws-glue/test/integ.partition-index.expected.json new file mode 100644 index 0000000000000..a4b3cad50cea3 --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.expected.json @@ -0,0 +1,594 @@ +{ + "Resources": { + "DataBucketE3889A50": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "MyDatabase1E2517DB": { + "Type": "AWS::Glue::Database", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseInput": { + "Name": "database" + } + } + }, + "CSVTableE499CABA": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "csv_table generated by CDK", + "Name": "csv_table", + "Parameters": { + "classification": "csv", + "has_encrypted_data": false + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + }, + { + "Name": "month", + "Type": "bigint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "string" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.apache.hadoop.hive.serde2.OpenCSVSerde" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "CSVTablepartitionindexindex1CustomResourcePolicy4983F2A9": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "glue:CreatePartitionIndex", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "CSVTablepartitionindexindex1CustomResourcePolicy4983F2A9", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "DependsOn": [ + "CSVTablepartitionindexindex2CustomResourcePolicy4FF1AF9F", + "CSVTablepartitionindexindex29D554319" + ] + }, + "CSVTablepartitionindexindex16247ABF6": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"service\":\"Glue\",\"action\":\"createPartitionIndex\",\"parameters\":{\"DatabaseName\":\"", + { + "Ref": "MyDatabase1E2517DB" + }, + "\",\"TableName\":\"", + { + "Ref": "CSVTableE499CABA" + }, + "\",\"PartitionIndex\":{\"IndexName\":\"index1\",\"Keys\":[\"month\"]}},\"physicalResourceId\":{\"id\":\"index1\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "CSVTablepartitionindexindex1CustomResourcePolicy4983F2A9", + "CSVTablepartitionindexindex2CustomResourcePolicy4FF1AF9F", + "CSVTablepartitionindexindex29D554319" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "CSVTablepartitionindexindex2CustomResourcePolicy4FF1AF9F": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "glue:CreatePartitionIndex", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "CSVTablepartitionindexindex2CustomResourcePolicy4FF1AF9F", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + } + }, + "CSVTablepartitionindexindex29D554319": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"service\":\"Glue\",\"action\":\"createPartitionIndex\",\"parameters\":{\"DatabaseName\":\"", + { + "Ref": "MyDatabase1E2517DB" + }, + "\",\"TableName\":\"", + { + "Ref": "CSVTableE499CABA" + }, + "\",\"PartitionIndex\":{\"IndexName\":\"index2\",\"Keys\":[\"month\",\"year\"]}},\"physicalResourceId\":{\"id\":\"index2\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "CSVTablepartitionindexindex2CustomResourcePolicy4FF1AF9F" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleDefaultPolicyD28E1A5E": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "glue:UpdateTable", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "CSVTableE499CABA" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":catalog" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":database/", + { + "Ref": "MyDatabase1E2517DB" + } + ] + ] + } + ] + }, + { + "Action": "glue:UpdateTable", + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":table/", + { + "Ref": "MyDatabase1E2517DB" + }, + "/", + { + "Ref": "JSONTable00348F1D" + } + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":catalog" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":glue:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":database/", + { + "Ref": "MyDatabase1E2517DB" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleDefaultPolicyD28E1A5E", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + } + }, + "AWS679f53fac002430cb0da5b7982bd22872D164C4C": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3BucketF482197E" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + "Arn" + ] + }, + "Handler": "index.handler", + "Runtime": "nodejs12.x", + "Timeout": 120 + }, + "DependsOn": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleDefaultPolicyD28E1A5E", + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + ] + }, + "JSONTable00348F1D": { + "Type": "AWS::Glue::Table", + "Properties": { + "CatalogId": { + "Ref": "AWS::AccountId" + }, + "DatabaseName": { + "Ref": "MyDatabase1E2517DB" + }, + "TableInput": { + "Description": "json_table generated by CDK", + "Name": "json_table", + "Parameters": { + "classification": "json", + "has_encrypted_data": false + }, + "PartitionKeys": [ + { + "Name": "year", + "Type": "smallint" + }, + { + "Name": "month", + "Type": "bigint" + } + ], + "StorageDescriptor": { + "Columns": [ + { + "Name": "col1", + "Type": "string" + }, + { + "Name": "col2", + "Type": "string" + }, + { + "Name": "col3", + "Type": "string" + } + ], + "Compressed": false, + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "Location": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Ref": "DataBucketE3889A50" + }, + "/" + ] + ] + }, + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "SerdeInfo": { + "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe" + }, + "StoredAsSubDirectories": false + }, + "TableType": "EXTERNAL_TABLE" + } + } + }, + "JSONTablepartitionindexyearmonthawscdkglueJSONTable937C116BCustomResourcePolicy92B3C1AE": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "glue:CreatePartitionIndex", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "JSONTablepartitionindexyearmonthawscdkglueJSONTable937C116BCustomResourcePolicy92B3C1AE", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + } + }, + "JSONTablepartitionindexyearmonthawscdkglueJSONTable937C116B74A5990F": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"service\":\"Glue\",\"action\":\"createPartitionIndex\",\"parameters\":{\"DatabaseName\":\"", + { + "Ref": "MyDatabase1E2517DB" + }, + "\",\"TableName\":\"", + { + "Ref": "JSONTable00348F1D" + }, + "\",\"PartitionIndex\":{\"IndexName\":\"year-month-awscdkglueJSONTable937C116B\",\"Keys\":[\"year\",\"month\"]}},\"physicalResourceId\":{\"id\":\"year-month-awscdkglueJSONTable937C116B\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "JSONTablepartitionindexyearmonthawscdkglueJSONTable937C116BCustomResourcePolicy92B3C1AE" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + } + }, + "Parameters": { + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3BucketF482197E": { + "Type": "String", + "Description": "S3 bucket for asset \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + }, + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632": { + "Type": "String", + "Description": "S3 key for asset version \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + }, + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2ArtifactHash4BE92B79": { + "Type": "String", + "Description": "Artifact hash for asset \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + } + }, + "Outputs": { + "CatalogId": { + "Value": { + "Ref": "AWS::AccountId" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue/test/integ.partition-index.ts b/packages/@aws-cdk/aws-glue/test/integ.partition-index.ts new file mode 100644 index 0000000000000..e3a514bfadf6e --- /dev/null +++ b/packages/@aws-cdk/aws-glue/test/integ.partition-index.ts @@ -0,0 +1,75 @@ +#!/usr/bin/env node +import * as s3 from '@aws-cdk/aws-s3'; +import * as cdk from '@aws-cdk/core'; +import * as glue from '../lib'; + +/** + * Stack verification steps: + * * aws cloudformation describe-stacks --stack-name aws-cdk-glue --query Stacks[0].Outputs[0].OutputValue + * * aws glue get-partition-indexes --catalog-id --database-name database --table-name csv_table + * returns two indexes named 'index1' and 'index2' + * * aws glue get-partition-indexes --catalog-id --database-name database --table-name json_table + * returns an index with name 'year-month...' + */ + +const app = new cdk.App(); +const stack = new cdk.Stack(app, 'aws-cdk-glue'); +const bucket = new s3.Bucket(stack, 'DataBucket'); +const database = new glue.Database(stack, 'MyDatabase', { + databaseName: 'database', +}); + +const columns = [{ + name: 'col1', + type: glue.Schema.STRING, +}, { + name: 'col2', + type: glue.Schema.STRING, +}, { + name: 'col3', + type: glue.Schema.STRING, +}]; + +const partitionKeys = [{ + name: 'year', + type: glue.Schema.SMALL_INT, +}, { + name: 'month', + type: glue.Schema.BIG_INT, +}]; + +new glue.Table(stack, 'CSVTable', { + database, + bucket, + tableName: 'csv_table', + columns, + partitionKeys, + partitionIndexes: [{ + indexName: 'index1', + keyNames: ['month'], + }, { + indexName: 'index2', + keyNames: ['month', 'year'], + }], + dataFormat: glue.DataFormat.CSV, +}); + +const jsonTable = new glue.Table(stack, 'JSONTable', { + database, + bucket, + tableName: 'json_table', + columns, + partitionKeys, + dataFormat: glue.DataFormat.JSON, +}); + +jsonTable.addPartitionIndex({ + keyNames: ['year', 'month'], +}); + +// output necessary for stack verification +new cdk.CfnOutput(stack, 'CatalogId', { + value: database.catalogId, +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.ts b/packages/@aws-cdk/aws-glue/test/integ.table.ts index 59eede985e5ce..e9d54d659921e 100644 --- a/packages/@aws-cdk/aws-glue/test/integ.table.ts +++ b/packages/@aws-cdk/aws-glue/test/integ.table.ts @@ -81,10 +81,7 @@ const encryptedTable = new glue.Table(stack, 'MyEncryptedTable', { database, tableName: 'my_encrypted_table', columns, - partitionKeys: [{ - name: 'year', - type: glue.Schema.SMALL_INT, - }], + partitionKeys, dataFormat: glue.DataFormat.JSON, encryption: glue.TableEncryption.KMS, encryptionKey: new kms.Key(stack, 'MyKey'), diff --git a/packages/@aws-cdk/aws-glue/test/table.test.ts b/packages/@aws-cdk/aws-glue/test/table.test.ts index e4dac06728e19..a7aa71474724f 100644 --- a/packages/@aws-cdk/aws-glue/test/table.test.ts +++ b/packages/@aws-cdk/aws-glue/test/table.test.ts @@ -6,6 +6,7 @@ import { testFutureBehavior } from '@aws-cdk/cdk-build-tools/lib/feature-flag'; import * as cdk from '@aws-cdk/core'; import * as cxapi from '@aws-cdk/cx-api'; import * as glue from '../lib'; +import { PartitionIndex } from '../lib'; import { CfnTable } from '../lib/glue.generated'; const s3GrantWriteCtx = { [cxapi.S3_GRANT_WRITE_WITHOUT_ACL]: true }; @@ -933,501 +934,617 @@ test('explicit s3 bucket and with empty prefix', () => { }); }); -test('grants: custom', () => { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); +describe('add partition index', () => { + test('fails if no partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['part'], + })).toThrowError(/The table must have partition keys to create a partition index/); }); - table.grant(user, ['glue:UpdateTable']); + test('fails if partition index does not match partition keys', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: 'glue:UpdateTable', - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: 'my-part', + keyNames: ['not-part'], + })).toThrowError(/All index keys must also be partition keys/); }); -}); -test('grants: read only', () => { - const stack = new cdk.Stack(); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', + test('fails with index name < 1 character', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + dataFormat: glue.DataFormat.JSON, + }); + + expect(() => table.addPartitionIndex({ + indexName: '', + keyNames: ['part'], + })).toThrowError(/Index name must be between 1 and 255 characters, but got 0/); }); - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, + test('fails with > 3 indexes', () => { + const stack = new cdk.Stack(); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); + + const indexes: PartitionIndex[] = [{ + indexName: 'ind1', + keyNames: ['part'], + }, { + indexName: 'ind2', + keyNames: ['part'], + }, { + indexName: 'ind3', + keyNames: ['part'], + }, { + indexName: 'ind4', + keyNames: ['part'], + }]; + + expect(() => new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'part', + type: glue.Schema.SMALL_INT, + }], + partitionIndexes: indexes, + dataFormat: glue.DataFormat.JSON, + })).toThrowError('Maximum number of partition indexes allowed is 3'); }); +}); - table.grantRead(user); +describe('grants', () => { + test('custom permissions', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchGetPartition', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetTableVersion', - 'glue:GetTableVersions', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], - ], - }, - }, - { - Action: [ - 's3:GetObject*', - 's3:GetBucket*', - 's3:List*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); + + table.grant(user, ['glue:UpdateTable']); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'glue:UpdateTable', + Effect: 'Allow', + Resource: { 'Fn::Join': [ '', [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], + Ref: 'Table4C2D914F', }, - '/*', ], ], }, - ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', }, ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], + }); }); -}); -testFutureBehavior('grants: write only', s3GrantWriteCtx, cdk.App, (app) => { - const stack = new cdk.Stack(app); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); + test('read only', () => { + const stack = new cdk.Stack(); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); - table.grantWrite(user); + table.grantRead(user); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchCreatePartition', - 'glue:BatchDeletePartition', - 'glue:CreatePartition', - 'glue:DeletePartition', - 'glue:UpdatePartition', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', ], - }, - }, - { - Action: [ - 's3:DeleteObject*', - 's3:PutObject', - 's3:Abort*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { + Effect: 'Allow', + Resource: { 'Fn::Join': [ '', [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', }, - '/*', ], ], }, - ], + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', }, ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], + }); }); -}); -testFutureBehavior('grants: read and write', s3GrantWriteCtx, cdk.App, (app) => { - const stack = new cdk.Stack(app); - const user = new iam.User(stack, 'User'); - const database = new glue.Database(stack, 'Database', { - databaseName: 'database', - }); + testFutureBehavior('write only', s3GrantWriteCtx, cdk.App, (app) => { + const stack = new cdk.Stack(app); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', + }); - const table = new glue.Table(stack, 'Table', { - database, - tableName: 'table', - columns: [{ - name: 'col', - type: glue.Schema.STRING, - }], - compressed: true, - dataFormat: glue.DataFormat.JSON, - }); + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', + columns: [{ + name: 'col', + type: glue.Schema.STRING, + }], + compressed: true, + dataFormat: glue.DataFormat.JSON, + }); - table.grantReadWrite(user); + table.grantWrite(user); - Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'glue:BatchGetPartition', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetTableVersion', - 'glue:GetTableVersions', - 'glue:BatchCreatePartition', - 'glue:BatchDeletePartition', - 'glue:CreatePartition', - 'glue:DeletePartition', - 'glue:UpdatePartition', - ], - Effect: 'Allow', - Resource: { - 'Fn::Join': [ - '', - [ - 'arn:', - { - Ref: 'AWS::Partition', - }, - ':glue:', - { - Ref: 'AWS::Region', - }, - ':', - { - Ref: 'AWS::AccountId', - }, - ':table/', - { - Ref: 'DatabaseB269D8BB', - }, - '/', - { - Ref: 'Table4C2D914F', - }, - ], + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', ], - }, - }, - { - Action: [ - 's3:GetObject*', - 's3:GetBucket*', - 's3:List*', - 's3:DeleteObject*', - 's3:PutObject', - 's3:Abort*', - ], - Effect: 'Allow', - Resource: [ - { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], - }, - { + Effect: 'Allow', + Resource: { 'Fn::Join': [ '', [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', { - 'Fn::GetAtt': [ - 'TableBucketDA42407C', - 'Arn', - ], + Ref: 'Table4C2D914F', }, - '/*', ], ], }, - ], + }, + { + Action: [ + 's3:DeleteObject*', + 's3:PutObject', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', }, ], - Version: '2012-10-17', - }, - PolicyName: 'UserDefaultPolicy1F97781E', - Users: [ - { - Ref: 'User00B015A1', - }, - ], + }); }); -}); -test('validate: at least one column', () => { - expect(() => { - createTable({ - columns: [], - tableName: 'name', + testFutureBehavior('read and write', s3GrantWriteCtx, cdk.App, (app) => { + const stack = new cdk.Stack(app); + const user = new iam.User(stack, 'User'); + const database = new glue.Database(stack, 'Database', { + databaseName: 'database', }); - }).toThrowError('you must specify at least one column for the table'); - -}); -test('validate: unique column names', () => { - expect(() => { - createTable({ - tableName: 'name', + const table = new glue.Table(stack, 'Table', { + database, + tableName: 'table', columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }, { - name: 'col1', + name: 'col', type: glue.Schema.STRING, }], + compressed: true, + dataFormat: glue.DataFormat.JSON, }); - }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + table.grantReadWrite(user); + + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'glue:BatchGetPartition', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetTableVersion', + 'glue:GetTableVersions', + 'glue:BatchCreatePartition', + 'glue:BatchDeletePartition', + 'glue:CreatePartition', + 'glue:DeletePartition', + 'glue:UpdatePartition', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':glue:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':table/', + { + Ref: 'DatabaseB269D8BB', + }, + '/', + { + Ref: 'Table4C2D914F', + }, + ], + ], + }, + }, + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'TableBucketDA42407C', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'UserDefaultPolicy1F97781E', + Users: [ + { + Ref: 'User00B015A1', + }, + ], + }); + }); }); -test('validate: unique partition keys', () => { - expect(() => { - createTable({ +describe('validate', () => { + test('at least one column', () => { + expect(() => { + createTable({ + columns: [], + tableName: 'name', + }); + }).toThrowError('you must specify at least one column for the table'); + + }); + + test('unique column names', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }, { + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + + }); + + test('unique partition keys', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'p1', + type: glue.Schema.STRING, + }, { + name: 'p1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'p1' is duplicated"); + + }); + + test('column names and partition keys are all unique', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + partitionKeys: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + }); + }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); + + }); + + test('can not specify an explicit bucket and encryption', () => { + expect(() => { + createTable({ + tableName: 'name', + columns: [{ + name: 'col1', + type: glue.Schema.STRING, + }], + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: glue.TableEncryption.KMS, + }); + }).toThrowError('you can not specify encryption settings if you also provide a bucket'); + }); + + test('can explicitly pass bucket if Encryption undefined', () => { + expect(() => createTable({ tableName: 'name', columns: [{ name: 'col1', type: glue.Schema.STRING, }], - partitionKeys: [{ - name: 'p1', - type: glue.Schema.STRING, - }, { - name: 'p1', - type: glue.Schema.STRING, - }], - }); - }).toThrowError("column names and partition keys must be unique, but 'p1' is duplicated"); - -}); + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); -test('validate: column names and partition keys are all unique', () => { - expect(() => { - createTable({ + test('can explicitly pass bucket if Unencrypted', () => { + expect(() => createTable({ tableName: 'name', columns: [{ name: 'col1', type: glue.Schema.STRING, }], - partitionKeys: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - }); - }).toThrowError("column names and partition keys must be unique, but 'col1' is duplicated"); - -}); + bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), + encryption: undefined, + })).not.toThrow(); + }); -test('validate: can not specify an explicit bucket and encryption', () => { - expect(() => { - createTable({ + test('can explicitly pass bucket if ClientSideKms', () => { + expect(() => createTable({ tableName: 'name', columns: [{ name: 'col1', type: glue.Schema.STRING, }], bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.KMS, - }); - }).toThrowError('you can not specify encryption settings if you also provide a bucket'); -}); - -test('validate: can explicitly pass bucket if Encryption undefined', () => { - expect(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })).not.toThrow(); -}); - -test('validate: can explicitly pass bucket if Unencrypted', () => { - expect(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: undefined, - })).not.toThrow(); -}); - -test('validate: can explicitly pass bucket if ClientSideKms', () => { - expect(() => createTable({ - tableName: 'name', - columns: [{ - name: 'col1', - type: glue.Schema.STRING, - }], - bucket: new s3.Bucket(new cdk.Stack(), 'Bucket'), - encryption: glue.TableEncryption.CLIENT_SIDE_KMS, - })).not.toThrow(); + encryption: glue.TableEncryption.CLIENT_SIDE_KMS, + })).not.toThrow(); + }); }); test('Table.fromTableArn', () => { From dee732d063c0658b768bf7b9f24087b8baed2ed6 Mon Sep 17 00:00:00 2001 From: Julian Michel Date: Wed, 29 Dec 2021 19:42:22 +0100 Subject: [PATCH 5/8] feat(ec2): add Windows Server 2022 WindowsVersions (#18203) Add Windows Server 2022 versions to WindowsVersions. List of relevant versions: `aws ssm get-parameters-by-path --path "/aws/service/ami-windows-latest" --region us-east-1 | jq '.Parameters | .[] | .Name' | grep Windows_Server-2022` Logic for enum values: Convert value from CLI command to upper case and replace `-` by `_`. Closes #18199. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../@aws-cdk/aws-ec2/lib/windows-versions.ts | 37 +++++++++++++++++++ packages/@aws-cdk/aws-ec2/package.json | 37 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/packages/@aws-cdk/aws-ec2/lib/windows-versions.ts b/packages/@aws-cdk/aws-ec2/lib/windows-versions.ts index 20e959420168f..2d5a60c5f9c22 100644 --- a/packages/@aws-cdk/aws-ec2/lib/windows-versions.ts +++ b/packages/@aws-cdk/aws-ec2/lib/windows-versions.ts @@ -255,4 +255,41 @@ export enum WindowsVersion { WINDOWS_SERVER_2016_CHINESE_TRADITIONAL_FULL_BASE = 'Windows_Server-2016-Chinese_Traditional-Full-Base', WINDOWS_SERVER_2016_ENGLISH_FULL_SQL_2016_SP2_WEB = 'Windows_Server-2016-English-Full-SQL_2016_SP2_Web', WINDOWS_SERVER_2016_ENGLISH_FULL_SQL_2017_EXPRESS = 'Windows_Server-2016-English-Full-SQL_2017_Express', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_ENTERPRISE = 'Windows_Server-2022-Japanese-Full-SQL_2019_Enterprise', + WINDOWS_SERVER_2022_PORTUGUESE_BRAZIL_FULL_BASE = 'Windows_Server-2022-Portuguese_Brazil-Full-Base', + WINDOWS_SERVER_2022_ITALIAN_FULL_BASE = 'Windows_Server-2022-Italian-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_CONTAINERSLATEST = 'Windows_Server-2022-English-Full-ContainersLatest', + WINDOWS_SERVER_2022_RUSSIAN_FULL_BASE = 'Windows_Server-2022-Russian-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_EXPRESS = 'Windows_Server-2022-English-Full-SQL_2019_Express', + WINDOWS_SERVER_2022_POLISH_FULL_BASE = 'Windows_Server-2022-Polish-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_CORE_BASE = 'Windows_Server-2022-English-Core-Base', + WINDOWS_SERVER_2022_HUNGARIAN_FULL_BASE = 'Windows_Server-2022-Hungarian-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_EXPRESS = 'Windows_Server-2022-English-Full-SQL_2017_Express', + WINDOWS_SERVER_2022_GERMAN_FULL_BASE = 'Windows_Server-2022-German-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_CORE_CONTAINERSLATEST = 'Windows_Server-2022-English-Core-ContainersLatest', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_STANDARD = 'Windows_Server-2022-English-Full-SQL_2019_Standard', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_WEB = 'Windows_Server-2022-Japanese-Full-SQL_2017_Web', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_WEB = 'Windows_Server-2022-English-Full-SQL_2017_Web', + WINDOWS_SERVER_2022_JAPANESE_FULL_BASE = 'Windows_Server-2022-Japanese-Full-Base', + WINDOWS_SERVER_2022_KOREAN_FULL_BASE = 'Windows_Server-2022-Korean-Full-Base', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_ENTERPRISE = 'Windows_Server-2022-Japanese-Full-SQL_2017_Enterprise', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_STANDARD = 'Windows_Server-2022-Japanese-Full-SQL_2019_Standard', + WINDOWS_SERVER_2022_CHINESE_SIMPLIFIED_FULL_BASE = 'Windows_Server-2022-Chinese_Simplified-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_WEB = 'Windows_Server-2022-English-Full-SQL_2019_Web', + WINDOWS_SERVER_2022_SPANISH_FULL_BASE = 'Windows_Server-2022-Spanish-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_CORE_ECS_OPTIMIZED = 'Windows_Server-2022-English-Core-ECS_Optimized', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_STANDARD = 'Windows_Server-2022-English-Full-SQL_2017_Standard', + WINDOWS_SERVER_2022_CHINESE_TRADITIONAL_FULL_BASE = 'Windows_Server-2022-Chinese_Traditional-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_ENTERPRISE = 'Windows_Server-2022-English-Full-SQL_2019_Enterprise', + WINDOWS_SERVER_2022_FRENCH_FULL_BASE = 'Windows_Server-2022-French-Full-Base', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_STANDARD = 'Windows_Server-2022-Japanese-Full-SQL_2017_Standard', + WINDOWS_SERVER_2022_ENGLISH_FULL_BASE = 'Windows_Server-2022-English-Full-Base', + WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_WEB = 'Windows_Server-2022-Japanese-Full-SQL_2019_Web', + WINDOWS_SERVER_2022_TURKISH_FULL_BASE = 'Windows_Server-2022-Turkish-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_ENTERPRISE = 'Windows_Server-2022-English-Full-SQL_2017_Enterprise', + WINDOWS_SERVER_2022_PORTUGUESE_PORTUGAL_FULL_BASE = 'Windows_Server-2022-Portuguese_Portugal-Full-Base', + WINDOWS_SERVER_2022_CZECH_FULL_BASE = 'Windows_Server-2022-Czech-Full-Base', + WINDOWS_SERVER_2022_ENGLISH_FULL_ECS_OPTIMIZED = 'Windows_Server-2022-English-Full-ECS_Optimized', + WINDOWS_SERVER_2022_DUTCH_FULL_BASE = 'Windows_Server-2022-Dutch-Full-Base', + WINDOWS_SERVER_2022_SWEDISH_FULL_BASE = 'Windows_Server-2022-Swedish-Full-Base', } diff --git a/packages/@aws-cdk/aws-ec2/package.json b/packages/@aws-cdk/aws-ec2/package.json index 8ef7a50a8b3c3..f9007178afc8e 100644 --- a/packages/@aws-cdk/aws-ec2/package.json +++ b/packages/@aws-cdk/aws-ec2/package.json @@ -211,6 +211,43 @@ "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2012_RTM_JAPANESE_64BIT_SQL_2014_SP2_STANDARD", "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2012_RTM_ENGLISH_64BIT_SQL_2014_SP3_EXPRESS", "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2012_RTM_JAPANESE_64BIT_SQL_2016_SP2_EXPRESS", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_ENTERPRISE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_PORTUGUESE_BRAZIL_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ITALIAN_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_CONTAINERSLATEST", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_RUSSIAN_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_EXPRESS", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_POLISH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_CORE_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_HUNGARIAN_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_EXPRESS", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_GERMAN_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_CORE_CONTAINERSLATEST", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_STANDARD", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_WEB", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_WEB", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_KOREAN_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_ENTERPRISE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_STANDARD", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_CHINESE_SIMPLIFIED_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_WEB", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_SPANISH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_CORE_ECS_OPTIMIZED", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_STANDARD", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_CHINESE_TRADITIONAL_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2019_ENTERPRISE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_FRENCH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2017_STANDARD", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_JAPANESE_FULL_SQL_2019_WEB", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_TURKISH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_SQL_2017_ENTERPRISE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_PORTUGUESE_PORTUGAL_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_CZECH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_ENGLISH_FULL_ECS_OPTIMIZED", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_DUTCH_FULL_BASE", + "docs-public-apis:@aws-cdk/aws-ec2.WindowsVersion.WINDOWS_SERVER_2022_SWEDISH_FULL_BASE", "docs-public-apis:@aws-cdk/aws-ec2.AclCidr.toCidrConfig", "docs-public-apis:@aws-cdk/aws-ec2.AclTraffic.toTrafficConfig", "docs-public-apis:@aws-cdk/aws-ec2.Connections.connections", From 9e6f977d8ac4ad7ab2852536cc20c4469fe74f03 Mon Sep 17 00:00:00 2001 From: Nathan Walters Date: Wed, 29 Dec 2021 17:52:54 -0600 Subject: [PATCH 6/8] feat(aws-autoscaling): Add support for termination policies (#17936) This PR adds support for [termination policies](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-as-group.html#cfn-as-group-termpolicy) to the `AutoScalingGroup` resource. `aws autoscaling describe-termination-policy-types --region us-east-2` reports the existence of a `Lambda` termination type, but that isn't documented in the CloudFormation docs, so I opted to omit it. Closes #15654. --- packages/@aws-cdk/aws-autoscaling/README.md | 26 ++++++++++++ .../aws-autoscaling/lib/auto-scaling-group.ts | 12 ++++++ .../@aws-cdk/aws-autoscaling/lib/index.ts | 3 +- .../aws-autoscaling/lib/termination-policy.ts | 42 +++++++++++++++++++ .../test/auto-scaling-group.test.ts | 25 +++++++++++ 5 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 packages/@aws-cdk/aws-autoscaling/lib/termination-policy.ts diff --git a/packages/@aws-cdk/aws-autoscaling/README.md b/packages/@aws-cdk/aws-autoscaling/README.md index 11c67226afdb4..a939ddac4a182 100644 --- a/packages/@aws-cdk/aws-autoscaling/README.md +++ b/packages/@aws-cdk/aws-autoscaling/README.md @@ -400,6 +400,32 @@ new autoscaling.AutoScalingGroup(this, 'ASG', { }); ``` +## Termination policies + +Auto Scaling uses [termination policies](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-instance-termination.html) +to determine which instances it terminates first during scale-in events. You +can specify one or more termination policies with the `terminationPolicies` +property: + +```ts +declare const vpc: ec2.Vpc; +declare const instanceType: ec2.InstanceType; +declare const machineImage: ec2.IMachineImage; + +new autoscaling.AutoScalingGroup(this, 'ASG', { + vpc, + instanceType, + machineImage, + + // ... + + terminationPolicies: [ + autoscaling.TerminationPolicy.OLDEST_INSTANCE, + autoscaling.TerminationPolicy.DEFAULT, + ], +}); +``` + ## Protecting new instances from being terminated on scale-in By default, Auto Scaling can terminate an instance at any time after launch when diff --git a/packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts b/packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts index 45fd06c478dfc..e2ec8ed39f3b6 100644 --- a/packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts +++ b/packages/@aws-cdk/aws-autoscaling/lib/auto-scaling-group.ts @@ -21,6 +21,7 @@ import { BasicLifecycleHookProps, LifecycleHook } from './lifecycle-hook'; import { BasicScheduledActionProps, ScheduledAction } from './scheduled-action'; import { BasicStepScalingPolicyProps, StepScalingPolicy } from './step-scaling-policy'; import { BaseTargetTrackingProps, PredefinedMetric, TargetTrackingScalingPolicy } from './target-tracking-scaling-policy'; +import { TerminationPolicy } from './termination-policy'; import { BlockDevice, BlockDeviceVolume, EbsDeviceVolumeType } from './volume'; /** @@ -314,6 +315,16 @@ export interface CommonAutoScalingGroupProps { * @default - Auto generated by CloudFormation */ readonly autoScalingGroupName?: string; + + /** + * A policy or a list of policies that are used to select the instances to + * terminate. The policies are executed in the order that you list them. + * + * @see https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-instance-termination.html + * + * @default - `TerminationPolicy.DEFAULT` + */ + readonly terminationPolicies?: TerminationPolicy[]; } /** @@ -1052,6 +1063,7 @@ export class AutoScalingGroup extends AutoScalingGroupBase implements healthCheckGracePeriod: props.healthCheck && props.healthCheck.gracePeriod && props.healthCheck.gracePeriod.toSeconds(), maxInstanceLifetime: this.maxInstanceLifetime ? this.maxInstanceLifetime.toSeconds() : undefined, newInstancesProtectedFromScaleIn: Lazy.any({ produce: () => this.newInstancesProtectedFromScaleIn }), + terminationPolicies: props.terminationPolicies, }; if (!hasPublic && props.associatePublicIpAddress) { diff --git a/packages/@aws-cdk/aws-autoscaling/lib/index.ts b/packages/@aws-cdk/aws-autoscaling/lib/index.ts index 186d1a3058fae..8cfcb36d42497 100644 --- a/packages/@aws-cdk/aws-autoscaling/lib/index.ts +++ b/packages/@aws-cdk/aws-autoscaling/lib/index.ts @@ -7,7 +7,8 @@ export * from './scheduled-action'; export * from './step-scaling-action'; export * from './step-scaling-policy'; export * from './target-tracking-scaling-policy'; +export * from './termination-policy'; export * from './volume'; // AWS::AutoScaling CloudFormation Resources: -export * from './autoscaling.generated'; \ No newline at end of file +export * from './autoscaling.generated'; diff --git a/packages/@aws-cdk/aws-autoscaling/lib/termination-policy.ts b/packages/@aws-cdk/aws-autoscaling/lib/termination-policy.ts new file mode 100644 index 0000000000000..1fe6b4d139459 --- /dev/null +++ b/packages/@aws-cdk/aws-autoscaling/lib/termination-policy.ts @@ -0,0 +1,42 @@ +/** + * Specifies the termination criteria to apply before Amazon EC2 Auto Scaling + * chooses an instance for termination. + */ +export enum TerminationPolicy { + /** + * Terminate instances in the Auto Scaling group to align the remaining + * instances to the allocation strategy for the type of instance that is + * terminating (either a Spot Instance or an On-Demand Instance). + */ + ALLOCATION_STRATEGY = 'AllocationStrategy', + + /** + * Terminate instances that are closest to the next billing hour. + */ + CLOSEST_TO_NEXT_INSTANCE_HOUR = 'ClosestToNextInstanceHour', + + /** + * Terminate instances according to the default termination policy. + */ + DEFAULT = 'Default', + + /** + * Terminate the newest instance in the group. + */ + NEWEST_INSTANCE = 'NewestInstance', + + /** + * Terminate the oldest instance in the group. + */ + OLDEST_INSTANCE = 'OldestInstance', + + /** + * Terminate instances that have the oldest launch configuration. + */ + OLDEST_LAUNCH_CONFIGURATION = 'OldestLaunchConfiguration', + + /** + * Terminate instances that have the oldest launch template. + */ + OLDEST_LAUNCH_TEMPLATE = 'OldestLaunchTemplate', +} diff --git a/packages/@aws-cdk/aws-autoscaling/test/auto-scaling-group.test.ts b/packages/@aws-cdk/aws-autoscaling/test/auto-scaling-group.test.ts index 25e84eeacaecf..8ca98c19deeb8 100644 --- a/packages/@aws-cdk/aws-autoscaling/test/auto-scaling-group.test.ts +++ b/packages/@aws-cdk/aws-autoscaling/test/auto-scaling-group.test.ts @@ -1382,6 +1382,31 @@ describe('auto scaling group', () => { }, }); }); + + test('supports termination policies', () => { + // GIVEN + const stack = new cdk.Stack(); + const vpc = mockVpc(stack); + + // WHEN + new autoscaling.AutoScalingGroup(stack, 'MyASG', { + vpc, + instanceType: new ec2.InstanceType('t2.micro'), + machineImage: ec2.MachineImage.latestAmazonLinux(), + terminationPolicies: [ + autoscaling.TerminationPolicy.OLDEST_INSTANCE, + autoscaling.TerminationPolicy.DEFAULT, + ], + }); + + // THEN + expect(stack).toHaveResource('AWS::AutoScaling::AutoScalingGroup', { + TerminationPolicies: [ + 'OldestInstance', + 'Default', + ], + }); + }); }); function mockVpc(stack: cdk.Stack) { From 8cb6a768cd4310628925fa117b674ae456aa8474 Mon Sep 17 00:00:00 2001 From: hanukoon Date: Thu, 30 Dec 2021 20:57:08 +0900 Subject: [PATCH 7/8] feat(aws-ec2): add g4ad instance types (#17927) update EC2 instance type for `g4ad` series. AWS CFN docs have already been updated: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-instance.html#cfn-ec2-instance-instancetype closes #17565 --- packages/@aws-cdk/aws-ec2/lib/instance-types.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/@aws-cdk/aws-ec2/lib/instance-types.ts b/packages/@aws-cdk/aws-ec2/lib/instance-types.ts index 5f30f1493987a..029219ce9bb79 100644 --- a/packages/@aws-cdk/aws-ec2/lib/instance-types.ts +++ b/packages/@aws-cdk/aws-ec2/lib/instance-types.ts @@ -576,6 +576,16 @@ export enum InstanceClass { */ G4DN = 'g4dn', + /** + * Graphics-optimized instances based on AMD EPYC And Radeon Pro GPU (NAVI) with local NVME drive, 4th generation + */ + GRAPHICS4_AMD_NVME_DRIVE = 'g4ad', + + /** + * Graphics-optimized instances based on AMD EPYC And Radeon Pro GPU (NAVI) with local NVME drive, 4th generation + */ + G4AD = 'g4ad', + /** * Graphics-optimized instances, 5th generation */ From 7d0680a5a858633f92aeb78353cac22b9a391fa7 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 30 Dec 2021 14:44:12 +0100 Subject: [PATCH 8/8] feat(ssm): reference latest version of secure string parameters (#18187) Supported by CF since April 2021 but not yet ported to CDK. See https://aws.amazon.com/about-aws/whats-new/2021/04/now-reference-latest-aws-systems-manager-parameter-values-in-aws-cloudformation-templates-without-specifying-parameter-versions/ Close #17091 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-ssm/lib/parameter.ts | 9 ++++++--- packages/@aws-cdk/aws-ssm/test/parameter.test.ts | 13 +++++++++++++ packages/@aws-cdk/core/README.md | 4 +++- packages/@aws-cdk/core/lib/secret-value.ts | 8 ++++---- packages/@aws-cdk/core/test/secret-value.test.ts | 11 +++++++++++ packages/aws-cdk-lib/README.md | 4 +++- 6 files changed, 40 insertions(+), 9 deletions(-) diff --git a/packages/@aws-cdk/aws-ssm/lib/parameter.ts b/packages/@aws-cdk/aws-ssm/lib/parameter.ts index 29c0eb8a33f03..99afccb82fda3 100644 --- a/packages/@aws-cdk/aws-ssm/lib/parameter.ts +++ b/packages/@aws-cdk/aws-ssm/lib/parameter.ts @@ -311,9 +311,11 @@ export interface StringParameterAttributes extends CommonStringParameterAttribut */ export interface SecureStringParameterAttributes extends CommonStringParameterAttributes { /** - * The version number of the value you wish to retrieve. This is required for secure strings. + * The version number of the value you wish to retrieve. + * + * @default - AWS CloudFormation uses the latest version of the parameter */ - readonly version: number; + readonly version?: number; /** * The encryption key that is used to encrypt this parameter @@ -365,7 +367,8 @@ export class StringParameter extends ParameterBase implements IStringParameter { * Imports a secure string parameter from the SSM parameter store. */ public static fromSecureStringParameterAttributes(scope: Construct, id: string, attrs: SecureStringParameterAttributes): IStringParameter { - const stringValue = new CfnDynamicReference(CfnDynamicReferenceService.SSM_SECURE, `${attrs.parameterName}:${Tokenization.stringifyNumber(attrs.version)}`).toString(); + const version = attrs.version ? Tokenization.stringifyNumber(attrs.version) : ''; + const stringValue = new CfnDynamicReference(CfnDynamicReferenceService.SSM_SECURE, `${attrs.parameterName}:${version}`).toString(); class Import extends ParameterBase { public readonly parameterName = attrs.parameterName; diff --git a/packages/@aws-cdk/aws-ssm/test/parameter.test.ts b/packages/@aws-cdk/aws-ssm/test/parameter.test.ts index 45fbc70f9d926..272a477523623 100644 --- a/packages/@aws-cdk/aws-ssm/test/parameter.test.ts +++ b/packages/@aws-cdk/aws-ssm/test/parameter.test.ts @@ -589,6 +589,19 @@ test('StringParameter.fromSecureStringParameterAttributes with encryption key cr }); }); +test('StringParameter.fromSecureStringParameterAttributes without version', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + const param = ssm.StringParameter.fromSecureStringParameterAttributes(stack, 'MyParamName', { + parameterName: 'MyParamName', + }); + + // THEN + expect(stack.resolve(param.stringValue)).toEqual('{{resolve:ssm-secure:MyParamName:}}'); +}); + test('StringListParameter.fromName', () => { // GIVEN const stack = new cdk.Stack(); diff --git a/packages/@aws-cdk/core/README.md b/packages/@aws-cdk/core/README.md index d6b58901838c2..d2b1e2c5e5222 100644 --- a/packages/@aws-cdk/core/README.md +++ b/packages/@aws-cdk/core/README.md @@ -223,7 +223,9 @@ Using AWS Secrets Manager is the recommended way to reference secrets in a CDK a `SecretValue` also supports the following secret sources: - `SecretValue.plainText(secret)`: stores the secret as plain text in your app and the resulting template (not recommended). - - `SecretValue.ssmSecure(param, version)`: refers to a secret stored as a SecureString in the SSM Parameter Store. + - `SecretValue.ssmSecure(param, version)`: refers to a secret stored as a SecureString in the SSM + Parameter Store. If you don't specify the exact version, AWS CloudFormation uses the latest + version of the parameter. - `SecretValue.cfnParameter(param)`: refers to a secret passed through a CloudFormation parameter (must have `NoEcho: true`). - `SecretValue.cfnDynamicReference(dynref)`: refers to a secret described by a CloudFormation dynamic reference (used by `ssmSecure` and `secretsManager`). diff --git a/packages/@aws-cdk/core/lib/secret-value.ts b/packages/@aws-cdk/core/lib/secret-value.ts index b57b704bd9ed6..84e668a1b4306 100644 --- a/packages/@aws-cdk/core/lib/secret-value.ts +++ b/packages/@aws-cdk/core/lib/secret-value.ts @@ -67,11 +67,11 @@ export class SecretValue extends Intrinsic { * Parameter Store. The parameter name is case-sensitive. * * @param version An integer that specifies the version of the parameter to - * use. You must specify the exact version. You cannot currently specify that - * AWS CloudFormation use the latest version of a parameter. + * use. If you don't specify the exact version, AWS CloudFormation uses the + * latest version of the parameter. */ - public static ssmSecure(parameterName: string, version: string): SecretValue { - const parts = [parameterName, version]; + public static ssmSecure(parameterName: string, version?: string): SecretValue { + const parts = [parameterName, version ?? '']; return this.cfnDynamicReference(new CfnDynamicReference(CfnDynamicReferenceService.SSM_SECURE, parts.join(':'))); } diff --git a/packages/@aws-cdk/core/test/secret-value.test.ts b/packages/@aws-cdk/core/test/secret-value.test.ts index 2efcdffbe808d..a987cfaff0c87 100644 --- a/packages/@aws-cdk/core/test/secret-value.test.ts +++ b/packages/@aws-cdk/core/test/secret-value.test.ts @@ -119,6 +119,17 @@ describe('secret value', () => { }); + test('ssmSecure without version', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const v = SecretValue.ssmSecure('param-name'); + + // THEN + expect(stack.resolve(v)).toEqual('{{resolve:ssm-secure:param-name:}}'); + }); + test('cfnDynamicReference', () => { // GIVEN const stack = new Stack(); diff --git a/packages/aws-cdk-lib/README.md b/packages/aws-cdk-lib/README.md index d08d5b4495bb6..017a85437675c 100644 --- a/packages/aws-cdk-lib/README.md +++ b/packages/aws-cdk-lib/README.md @@ -254,7 +254,9 @@ Using AWS Secrets Manager is the recommended way to reference secrets in a CDK a `SecretValue` also supports the following secret sources: - `SecretValue.plainText(secret)`: stores the secret as plain text in your app and the resulting template (not recommended). - - `SecretValue.ssmSecure(param, version)`: refers to a secret stored as a SecureString in the SSM Parameter Store. + - `SecretValue.ssmSecure(param, version)`: refers to a secret stored as a SecureString in the SSM + Parameter Store. If you don't specify the exact version, AWS CloudFormation uses the latest + version of the parameter. - `SecretValue.cfnParameter(param)`: refers to a secret passed through a CloudFormation parameter (must have `NoEcho: true`). - `SecretValue.cfnDynamicReference(dynref)`: refers to a secret described by a CloudFormation dynamic reference (used by `ssmSecure` and `secretsManager`).