From 0d2b63397074abbde793b38f9b6d2637db602f15 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 27 Dec 2018 20:26:14 +0200 Subject: [PATCH] feat(assets): enable local tooling scenarios such as lambda debugging (#1433) Adds CloudFormation resource metadata which enables tools such as SAM CLI to find local assets used by resources in the template. The toolkit enables this feature by default, but it can be disabled using `--no-asset-metadata` or by setting `assetMetadata` to `false` in `cdk.json`. See design document under [design/code-asset-metadata.md](./design/code-asset-metadata.md) Fixes #1432 --- design/code-asset-metadata.md | 83 +++++++++++++++++++ packages/@aws-cdk/assets/README.md | 22 +++++ packages/@aws-cdk/assets/lib/asset.ts | 28 +++++++ packages/@aws-cdk/assets/test/test.asset.ts | 47 ++++++++++- packages/@aws-cdk/aws-lambda/lib/code.ts | 11 ++- packages/@aws-cdk/aws-lambda/lib/lambda.ts | 2 +- .../@aws-cdk/aws-lambda/test/test.code.ts | 26 ++++++ packages/@aws-cdk/cx-api/lib/cxapi.ts | 12 --- .../@aws-cdk/cx-api/lib/metadata/assets.ts | 26 ++++++ packages/aws-cdk/bin/cdk.ts | 1 + packages/aws-cdk/lib/api/cxapp/exec.ts | 11 ++- packages/aws-cdk/lib/settings.ts | 1 + tools/cdk-integ-tools/bin/cdk-integ-assert.ts | 2 +- tools/cdk-integ-tools/bin/cdk-integ.ts | 3 +- 14 files changed, 254 insertions(+), 21 deletions(-) create mode 100644 design/code-asset-metadata.md diff --git a/design/code-asset-metadata.md b/design/code-asset-metadata.md new file mode 100644 index 0000000000000..7a367b7d6ccb6 --- /dev/null +++ b/design/code-asset-metadata.md @@ -0,0 +1,83 @@ +# RFC: AWS Lambda - Metadata about Code Assets + +As described in [#1432](https://github.com/awslabs/aws-cdk/issues/1432), in order to support local debugging, +debuggers like [SAM CLI](https://github.com/awslabs/aws-sam-cli) need to be able to find the code of a Lambda +function locally. + +The current implementation of assets uses CloudFormation Parameters which represent the S3 bucket and key of the +uploaded asset, which makes it impossible for local tools to reason about (without traversing the cx protocol with +many heuristics). + +## Approach + +We will automatically embed CloudFormation metadata on `AWS::Lambda::Function` (and any other) resources which use +local assets for code. The metadata will allow tools like SAM CLI to find the code locally for local invocations. + +Given a CDK app with an AWS Lambda function defined like so: + +```ts +new lambda.Function(this, 'MyHandler', { + // ... + code: lambda.Code.asset('/path/to/handler') +}); +``` + +The synthesized `AWS::Lambda::Function` resource will include a "Metadata" entry as follows: + +```js +{ + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + // current asset magic + } + }, + "Metadata": { + "aws:asset:property": "Code", + "aws:asset:path": "/path/to/handler" + } +} +``` + +Local debugging tools like SAM CLI will be able to traverse the template and look up the `aws:asset` metadata +entries, and use them to process the template so it will be compatible with their inputs. + +## Design + +We will add a new method to the `Asset` class called `addResourceMetadata(resource, propName)`. This method will +take in an L1 resource (`cdk.Resource`) and a property name (`string`) and will add template metadata as +described above. + +This feature will be enabled by a context key `aws:cdk:enable-asset-metadata`, which will be enabled by default in +the CDK Toolkit. The switch `--asset-metadata` (or `--no-asset-metadata`) can be used to control this behavior, as +well as through the key `assetMetadata` in `cdk.json`. Very similar design to how path metadata is controlled. + +## Alternatives Considered + +We considered alternatives that will "enforce" the embedding of metadata when an asset is referenced by a resource. Since +a single asset can be referenced by multiple resources, it means that the _relationship_ is what should trigger the +metadata addition. There currently isn't support in the framework for such hooks, but there is a possiblility that +the changes in [#1436](https://github.com/awslabs/aws-cdk/pull/1436) might enable hooking into the relationnship, and then we might be able to use this mechanism to produce the metadata. + +Having said that, the need to embed asset metadata on resources is mainly confined to authors of L2 constructs, and not applicable for the general user population, so the value of automation is not high. + +## What about L1 resources? + +If users directly use L1 resources such as `lambda.CfnFunction` or `serverless.CfnFunction` and wish to use local CDK assets with tooling, they will have to explicitly add metadata: + +```ts +const asset = new assets.ZipDirectoryAsset(this, 'Foo', { + path: '/foo/boom' +}); + +const resource = new serverless.CfnFunction(this, 'Func', { + codeUri: { + bucket: asset.s3BucketName, + key: asset.s3ObjectKey + }, + runtime: 'nodejs8.10', + handler: 'index.handler' +}); + +resource.addResourceMetadata(resource, 'CodeUri'); +``` diff --git a/packages/@aws-cdk/assets/README.md b/packages/@aws-cdk/assets/README.md index 681a4d2e45dec..18adfea13ebda 100644 --- a/packages/@aws-cdk/assets/README.md +++ b/packages/@aws-cdk/assets/README.md @@ -59,3 +59,25 @@ the asset store, it is uploaded during deployment. Now, when the toolkit deploys the stack, it will set the relevant CloudFormation Parameters to point to the actual bucket and key for each asset. + +## CloudFormation Resource Metadata + +> NOTE: This section is relevant for authors of AWS Resource Constructs. + +In certain situations, it is desirable for tools to be able to know that a certain CloudFormation +resource is using a local asset. For example, SAM CLI can be used to invoke AWS Lambda functions +locally for debugging purposes. + +To enable such use cases, external tools will consult a set of metadata entries on AWS CloudFormation +resources: + +- `aws:asset:path` points to the local path of the asset. +- `aws:asset:property` is the name of the resource property where the asset is used + +Using these two metadata entries, tools will be able to identify that assets are used +by a certain resource, and enable advanced local experiences. + +To add these metadata entries to a resource, use the +`asset.addResourceMetadata(resource, property)` method. + +See https://github.com/awslabs/aws-cdk/issues/1432 for more details diff --git a/packages/@aws-cdk/assets/lib/asset.ts b/packages/@aws-cdk/assets/lib/asset.ts index 92af4aeba3588..c20f87265d35f 100644 --- a/packages/@aws-cdk/assets/lib/asset.ts +++ b/packages/@aws-cdk/assets/lib/asset.ts @@ -140,6 +140,34 @@ export class Asset extends cdk.Construct { } } + /** + * Adds CloudFormation template metadata to the specified resource with + * information that indicates which resource property is mapped to this local + * asset. This can be used by tools such as SAM CLI to provide local + * experience such as local invocation and debugging of Lambda functions. + * + * Asset metadata will only be included if the stack is synthesized with the + * "aws:cdk:enable-asset-metadata" context key defined, which is the default + * behavior when synthesizing via the CDK Toolkit. + * + * @see https://github.com/awslabs/aws-cdk/issues/1432 + * + * @param resource The CloudFormation resource which is using this asset. + * @param resourceProperty The property name where this asset is referenced + * (e.g. "Code" for AWS::Lambda::Function) + */ + public addResourceMetadata(resource: cdk.Resource, resourceProperty: string) { + if (!this.getContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT)) { + return; // not enabled + } + + // tell tools such as SAM CLI that the "Code" property of this resource + // points to a local path in order to enable local invocation of this function. + resource.options.metadata = resource.options.metadata || { }; + resource.options.metadata[cxapi.ASSET_RESOURCE_METADATA_PATH_KEY] = this.assetPath; + resource.options.metadata[cxapi.ASSET_RESOURCE_METADATA_PROPERTY_KEY] = resourceProperty; + } + /** * Grants read permissions to the principal on the asset's S3 object. */ diff --git a/packages/@aws-cdk/assets/test/test.asset.ts b/packages/@aws-cdk/assets/test/test.asset.ts index c7395deac8634..e299f46928485 100644 --- a/packages/@aws-cdk/assets/test/test.asset.ts +++ b/packages/@aws-cdk/assets/test/test.asset.ts @@ -1,6 +1,7 @@ -import { expect, haveResource } from '@aws-cdk/assert'; +import { expect, haveResource, ResourcePart } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import cdk = require('@aws-cdk/cdk'); +import cxapi = require('@aws-cdk/cx-api'); import { Test } from 'nodeunit'; import path = require('path'); import { FileAsset, ZipDirectoryAsset } from '../lib/asset'; @@ -139,6 +140,50 @@ export = { test.equal(zipDirectoryAsset.isZipArchive, true); test.equal(zipFileAsset.isZipArchive, true); test.equal(jarFileAsset.isZipArchive, true); + test.done(); + }, + + 'addResourceMetadata can be used to add CFN metadata to resources'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + stack.setContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT, true); + + const location = path.join(__dirname, 'sample-asset-directory'); + const resource = new cdk.Resource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: location }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + // THEN + expect(stack).to(haveResource('My::Resource::Type', { + Metadata: { + "aws:asset:path": location, + "aws:asset:property": "PropName" + } + }, ResourcePart.CompleteDefinition)); + test.done(); + }, + + 'asset metadata is only emitted if ASSET_RESOURCE_METADATA_ENABLED_CONTEXT is defined'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + + const location = path.join(__dirname, 'sample-asset-directory'); + const resource = new cdk.Resource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: location }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + // THEN + expect(stack).notTo(haveResource('My::Resource::Type', { + Metadata: { + "aws:asset:path": location, + "aws:asset:property": "PropName" + } + }, ResourcePart.CompleteDefinition)); + test.done(); } }; diff --git a/packages/@aws-cdk/aws-lambda/lib/code.ts b/packages/@aws-cdk/aws-lambda/lib/code.ts index 22237682d22ea..2c7f35f8af84c 100644 --- a/packages/@aws-cdk/aws-lambda/lib/code.ts +++ b/packages/@aws-cdk/aws-lambda/lib/code.ts @@ -54,7 +54,7 @@ export abstract class Code { * Called during stack synthesis to render the CodePropery for the * Lambda function. */ - public abstract toJSON(): CfnFunction.CodeProperty; + public abstract toJSON(resource: CfnFunction): CfnFunction.CodeProperty; /** * Called when the lambda is initialized to allow this object to @@ -81,7 +81,7 @@ export class S3Code extends Code { this.bucketName = bucket.bucketName; } - public toJSON(): CfnFunction.CodeProperty { + public toJSON(_: CfnFunction): CfnFunction.CodeProperty { return { s3Bucket: this.bucketName, s3Key: this.key, @@ -108,7 +108,7 @@ export class InlineCode extends Code { } } - public toJSON(): CfnFunction.CodeProperty { + public toJSON(_: CfnFunction): CfnFunction.CodeProperty { return { zipFile: this.code }; @@ -156,7 +156,10 @@ export class AssetCode extends Code { } } - public toJSON(): CfnFunction.CodeProperty { + public toJSON(resource: CfnFunction): CfnFunction.CodeProperty { + // https://github.com/awslabs/aws-cdk/issues/1432 + this.asset!.addResourceMetadata(resource, 'Code'); + return { s3Bucket: this.asset!.s3BucketName, s3Key: this.asset!.s3ObjectKey diff --git a/packages/@aws-cdk/aws-lambda/lib/lambda.ts b/packages/@aws-cdk/aws-lambda/lib/lambda.ts index 237e8a9e6b4e0..d2ec71d12d4b2 100644 --- a/packages/@aws-cdk/aws-lambda/lib/lambda.ts +++ b/packages/@aws-cdk/aws-lambda/lib/lambda.ts @@ -244,7 +244,7 @@ export class Function extends FunctionRef { const resource = new CfnFunction(this, 'Resource', { functionName: props.functionName, description: props.description, - code: new cdk.Token(() => props.code.toJSON()), + code: new cdk.Token(() => props.code.toJSON(resource)), handler: props.handler, timeout: props.timeout, runtime: props.runtime.name, diff --git a/packages/@aws-cdk/aws-lambda/test/test.code.ts b/packages/@aws-cdk/aws-lambda/test/test.code.ts index 7e2401f2e69bf..d9d58efd39b08 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.code.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.code.ts @@ -1,5 +1,7 @@ +import { expect, haveResource, ResourcePart } from '@aws-cdk/assert'; import assets = require('@aws-cdk/assets'); import cdk = require('@aws-cdk/cdk'); +import cxapi = require('@aws-cdk/cx-api'); import { Test } from 'nodeunit'; import path = require('path'); import lambda = require('../lib'); @@ -65,6 +67,30 @@ export = { test.deepEqual(synthesized.metadata['/MyStack/Func1/Code'][0].type, 'aws:cdk:asset'); test.deepEqual(synthesized.metadata['/MyStack/Func2/Code'], undefined); + test.done(); + }, + + 'adds code asset metadata'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + stack.setContext(cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT, true); + + const location = path.join(__dirname, 'my-lambda-handler'); + + // WHEN + new lambda.Function(stack, 'Func1', { + code: lambda.Code.asset(location), + runtime: lambda.Runtime.NodeJS810, + handler: 'foom', + }); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::Function', { + Metadata: { + [cxapi.ASSET_RESOURCE_METADATA_PATH_KEY]: location, + [cxapi.ASSET_RESOURCE_METADATA_PROPERTY_KEY]: 'Code' + } + }, ResourcePart.CompleteDefinition)); test.done(); } } diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index 51bf4388dff31..83c0fa6f4828a 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -128,15 +128,3 @@ export const PATH_METADATA_KEY = 'aws:cdk:path'; * Enables the embedding of the "aws:cdk:path" in CloudFormation template metadata. */ export const PATH_METADATA_ENABLE_CONTEXT = 'aws:cdk:enable-path-metadata'; - -/** - * Separator string that separates the prefix separator from the object key separator. - * - * Asset keys will look like: - * - * /assets/MyConstruct12345678/||abcdef12345.zip - * - * This allows us to encode both the prefix and the full location in a single - * CloudFormation Template Parameter. - */ -export const ASSET_PREFIX_SEPARATOR = '||'; diff --git a/packages/@aws-cdk/cx-api/lib/metadata/assets.ts b/packages/@aws-cdk/cx-api/lib/metadata/assets.ts index e3645d4302f22..01786d0c595da 100644 --- a/packages/@aws-cdk/cx-api/lib/metadata/assets.ts +++ b/packages/@aws-cdk/cx-api/lib/metadata/assets.ts @@ -1,5 +1,31 @@ export const ASSET_METADATA = 'aws:cdk:asset'; +/** + * If this is set in the context, the aws:asset:xxx metadata entries will not be + * added to the template. This is used, for example, when we run integrationt + * tests. + */ +export const ASSET_RESOURCE_METADATA_ENABLED_CONTEXT = 'aws:cdk:enable-asset-metadata'; + +/** + * Metadata added to the CloudFormation template entries that map local assets + * to resources. + */ +export const ASSET_RESOURCE_METADATA_PATH_KEY = 'aws:asset:path'; +export const ASSET_RESOURCE_METADATA_PROPERTY_KEY = 'aws:asset:property'; + +/** + * Separator string that separates the prefix separator from the object key separator. + * + * Asset keys will look like: + * + * /assets/MyConstruct12345678/||abcdef12345.zip + * + * This allows us to encode both the prefix and the full location in a single + * CloudFormation Template Parameter. + */ +export const ASSET_PREFIX_SEPARATOR = '||'; + export interface FileAssetMetadataEntry { /** * Requested packaging style diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 56e6fed5c3b86..c91f10123ab8e 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -45,6 +45,7 @@ async function parseCommandLineArguments() { .option('ec2creds', { type: 'boolean', alias: 'i', default: undefined, desc: 'Force trying to fetch EC2 instance credentials. Default: guess EC2 instance status.' }) .option('version-reporting', { type: 'boolean', desc: 'Include the "AWS::CDK::Metadata" resource in synthesized templates (enabled by default)', default: undefined }) .option('path-metadata', { type: 'boolean', desc: 'Include "aws:cdk:path" CloudFormation metadata for each resource (enabled by default)', default: true }) + .option('asset-metadata', { type: 'boolean', desc: 'Include "aws:asset:*" CloudFormation metadata for resources that user assets (enabled by default)', default: true }) .option('role-arn', { type: 'string', alias: 'r', desc: 'ARN of Role to use when invoking CloudFormation', default: undefined }) .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack' }) .command([ 'list', 'ls' ], 'Lists all stacks in the app', yargs => yargs diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index a597783573398..7bea9bf569be6 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -17,13 +17,22 @@ export async function execProgram(aws: SDK, config: Settings): Promise(); - // inject "--no-path-metadata" so aws:cdk:path entries are not added to CFN metadata + // don't inject cloudformation metadata into template args.push('--no-path-metadata'); + args.push('--no-asset-metadata'); // inject "--verbose" to the command line of "cdk" if we are in verbose mode if (argv.verbose) {