From df2ad6df56ff248b7e32f4ed2a982768bb802a74 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Tue, 5 Oct 2021 15:49:56 +0200 Subject: [PATCH 1/5] feat(lambda): docker platform for architecture Add a `dockerPlatform` property in `Architecture` and use it to pass the correct `platform` when bundling in a container in `aws-lambda-nodejs`, `aws-lambda-go` and `aws-lambda-python`. --- .../@aws-cdk/aws-lambda-go/lib/bundling.ts | 8 +++- .../@aws-cdk/aws-lambda-go/lib/function.ts | 2 + .../aws-lambda-go/test/bundling.test.ts | 35 ++++++++++++---- .../aws-lambda-nodejs/lib/bundling.ts | 8 +++- .../aws-lambda-nodejs/lib/function.ts | 3 ++ .../aws-lambda-nodejs/test/bundling.test.ts | 40 +++++++++++++++---- .../aws-lambda-python/lib/bundling.ts | 8 +++- .../aws-lambda-python/lib/function.ts | 2 + .../@aws-cdk/aws-lambda-python/lib/layer.ts | 13 +++++- .../aws-lambda-python/test/bundling.test.ts | 17 +++++++- .../test/integ.function.expected.json | 18 ++++----- .../@aws-cdk/aws-lambda/lib/architecture.ts | 16 +++++--- 12 files changed, 134 insertions(+), 36 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index fd320cce90aed..d7251688dfa10 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -1,6 +1,6 @@ import * as os from 'os'; import * as path from 'path'; -import { AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; +import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; import { BundlingOptions } from './types'; import { exec, findUp, getGoBuildVersion } from './util'; @@ -55,6 +55,11 @@ export interface BundlingProps extends BundlingOptions { * The runtime of the lambda function */ readonly runtime: Runtime; + + /** + * The system architecture of the lambda function + */ + readonly architecture: Architecture; } /** @@ -117,6 +122,7 @@ export class Bundling implements cdk.BundlingOptions { ...props.buildArgs ?? {}, IMAGE: Runtime.GO_1_X.bundlingImage.image, // always use the GO_1_X build image }, + platform: props.architecture.dockerPlatform, }) : cdk.DockerImage.fromRegistry('dummy'); // Do not build if we don't need to diff --git a/packages/@aws-cdk/aws-lambda-go/lib/function.ts b/packages/@aws-cdk/aws-lambda-go/lib/function.ts index 3b915e859cef3..4c7220ee27dce 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/function.ts @@ -104,6 +104,7 @@ export class GoFunction extends lambda.Function { } const runtime = props.runtime ?? lambda.Runtime.PROVIDED_AL2; + const architecture = props.architecture ?? lambda.Architecture.X86_64; super(scope, id, { ...props, @@ -112,6 +113,7 @@ export class GoFunction extends lambda.Function { ...props.bundling ?? {}, entry, runtime, + architecture, moduleDir, }), handler: 'bootstrap', // setting name to bootstrap so that the 'provided' runtime can also be used diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index 474657dff61b9..73dfec99b45c1 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -1,26 +1,28 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; -import { Code, Runtime } from '@aws-cdk/aws-lambda'; +import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; import { AssetHashType, DockerImage } from '@aws-cdk/core'; import { Bundling } from '../lib/bundling'; import * as util from '../lib/util'; -jest.spyOn(Code, 'fromAsset'); -const fromAssetMock = jest.spyOn(DockerImage, 'fromBuild'); let getGoBuildVersionMock = jest.spyOn(util, 'getGoBuildVersion'); beforeEach(() => { jest.clearAllMocks(); jest.resetAllMocks(); Bundling.clearRunsLocallyCache(); - getGoBuildVersionMock.mockReturnValue(true); - fromAssetMock.mockReturnValue({ + + jest.spyOn(Code, 'fromAsset'); + + jest.spyOn(DockerImage, 'fromBuild').mockReturnValue({ image: 'built-image', cp: () => 'built-image', run: () => {}, toJSON: () => 'build-image', }); + + getGoBuildVersionMock.mockReturnValue(true); }); const moduleDir = '/project/go.mod'; @@ -30,6 +32,7 @@ test('bundling', () => { Bundling.bundle({ entry, runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, forcedDockerBundling: true, environment: { @@ -55,12 +58,20 @@ test('bundling', () => { ], }), }); + + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/aws-lambda-go\/lib$/), expect.objectContaining({ + buildArgs: expect.objectContaining({ + IMAGE: expect.stringMatching(/build-go/), + }), + platform: 'linux/amd64', + })); }); test('bundling with file as entry', () => { Bundling.bundle({ entry: '/project/main.go', runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, }); @@ -81,6 +92,7 @@ test('bundling with file in subdirectory as entry', () => { Bundling.bundle({ entry: '/project/cmd/api/main.go', runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, }); @@ -101,6 +113,7 @@ test('bundling with file other than main.go in subdirectory as entry', () => { Bundling.bundle({ entry: '/project/cmd/api/api.go', runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, }); @@ -122,6 +135,7 @@ test('go with Windows paths', () => { Bundling.bundle({ entry: 'C:\\my-project\\cmd\\api', runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir: 'C:\\my-project\\go.mod', forcedDockerBundling: true, }); @@ -141,13 +155,14 @@ test('with Docker build args', () => { Bundling.bundle({ entry, runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, forcedDockerBundling: true, buildArgs: { HELLO: 'WORLD', }, }); - expect(fromAssetMock).toHaveBeenCalledWith(expect.stringMatching(/lib$/), expect.objectContaining({ + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/aws-lambda-go\/lib$/), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', }), @@ -171,6 +186,7 @@ test('Local bundling', () => { KEY: 'value', }, runtime: Runtime.PROVIDED_AL2, + architecture: Architecture.X86_64, }); expect(bundler.local).toBeDefined(); @@ -188,7 +204,7 @@ test('Local bundling', () => { ); // Docker image is not built - expect(fromAssetMock).not.toHaveBeenCalled(); + expect(DockerImage.fromBuild).not.toHaveBeenCalled(); }); test('Incorrect go version', () => { @@ -198,6 +214,7 @@ test('Incorrect go version', () => { entry, moduleDir, runtime: Runtime.PROVIDED_AL2, + architecture: Architecture.X86_64, }); const tryBundle = bundler.local?.tryBundle('/outdir', { image: Runtime.GO_1_X.bundlingDockerImage }); @@ -211,6 +228,7 @@ test('Custom bundling docker image', () => { entry, moduleDir, runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, forcedDockerBundling: true, dockerImage: DockerImage.fromRegistry('my-custom-image'), }); @@ -227,6 +245,7 @@ test('Go build flags can be passed', () => { Bundling.bundle({ entry, runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, environment: { KEY: 'value', @@ -258,6 +277,7 @@ test('AssetHashType can be specified', () => { Bundling.bundle({ entry, runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, moduleDir, environment: { KEY: 'value', @@ -291,6 +311,7 @@ test('with command hooks', () => { entry, moduleDir, runtime: Runtime.PROVIDED_AL2, + architecture: Architecture.X86_64, commandHooks: { beforeBundling(inputDir: string, outputDir: string): string[] { return [ diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 3c33ad74f2471..3db8b21c3f592 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -1,6 +1,6 @@ import * as os from 'os'; import * as path from 'path'; -import { AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; +import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; import { EsbuildInstallation } from './esbuild-installation'; import { PackageManager } from './package-manager'; @@ -28,6 +28,11 @@ export interface BundlingProps extends BundlingOptions { */ readonly runtime: Runtime; + /** + * The system architecture of the lambda function + */ + readonly architecture: Architecture; + /** * Path to project root */ @@ -99,6 +104,7 @@ export class Bundling implements cdk.BundlingOptions { IMAGE: props.runtime.bundlingImage.image, ESBUILD_VERSION: props.esbuildVersion ?? ESBUILD_MAJOR_VERSION, }, + platform: props.architecture.dockerPlatform, }) : cdk.DockerImage.fromRegistry('dummy'); // Do not build if we don't need to diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts index 09c4964fd610d..0bb00a2c35e5c 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/function.ts @@ -1,6 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as lambda from '@aws-cdk/aws-lambda'; +import { Architecture } from '@aws-cdk/aws-lambda'; import { Bundling } from './bundling'; import { PackageManager } from './package-manager'; import { BundlingOptions } from './types'; @@ -95,6 +96,7 @@ export class NodejsFunction extends lambda.Function { const entry = path.resolve(findEntry(id, props.entry)); const handler = props.handler ?? 'handler'; const runtime = props.runtime ?? lambda.Runtime.NODEJS_14_X; + const architecture = props.architecture ?? Architecture.X86_64; const depsLockFilePath = findLockFile(props.depsLockFilePath); const projectRoot = props.projectRoot ?? path.dirname(depsLockFilePath); @@ -105,6 +107,7 @@ export class NodejsFunction extends lambda.Function { ...props.bundling ?? {}, entry, runtime, + architecture, depsLockFilePath, projectRoot, }), diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index a38a6fa08d5bc..df70c1437d356 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -1,7 +1,7 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; -import { Code, Runtime } from '@aws-cdk/aws-lambda'; +import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; import { AssetHashType, DockerImage } from '@aws-cdk/core'; import { version as delayVersion } from 'delay/package.json'; import { Bundling } from '../lib/bundling'; @@ -9,10 +9,6 @@ import { EsbuildInstallation } from '../lib/esbuild-installation'; import { LogLevel, SourceMapMode } from '../lib/types'; import * as util from '../lib/util'; -jest.mock('@aws-cdk/aws-lambda'); - -// Mock DockerImage.fromAsset() to avoid building the image -let fromBuildMock: jest.SpyInstance; let detectEsbuildMock: jest.SpyInstance; beforeEach(() => { jest.clearAllMocks(); @@ -20,12 +16,14 @@ beforeEach(() => { jest.restoreAllMocks(); Bundling.clearEsbuildInstallationCache(); + jest.spyOn(Code, 'fromAsset'); + detectEsbuildMock = jest.spyOn(EsbuildInstallation, 'detect').mockReturnValue({ isLocal: true, version: '0.8.8', }); - fromBuildMock = jest.spyOn(DockerImage, 'fromBuild').mockReturnValue({ + jest.spyOn(DockerImage, 'fromBuild').mockReturnValue({ image: 'built-image', cp: () => 'dest-path', run: () => {}, @@ -44,6 +42,7 @@ test('esbuild bundling in Docker', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, environment: { KEY: 'value', }, @@ -67,6 +66,13 @@ test('esbuild bundling in Docker', () => { workingDirectory: '/', }), }); + + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/aws-lambda-nodejs\/lib$/), expect.objectContaining({ + buildArgs: expect.objectContaining({ + IMAGE: expect.stringMatching(/build-nodejs/), + }), + platform: 'linux/amd64', + })); }); test('esbuild bundling with handler named index.ts', () => { @@ -75,6 +81,7 @@ test('esbuild bundling with handler named index.ts', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, forceDockerBundling: true, }); @@ -96,6 +103,7 @@ test('esbuild bundling with tsx handler', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, forceDockerBundling: true, }); @@ -121,6 +129,7 @@ test('esbuild with Windows paths', () => { Bundling.bundle({ entry: 'C:\\my-project\\lib\\entry.ts', runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, projectRoot: 'C:\\my-project', depsLockFilePath: 'C:\\my-project\\package-lock.json', forceDockerBundling: true, @@ -144,6 +153,7 @@ test('esbuild bundling with externals and dependencies', () => { projectRoot: path.dirname(packageLock), depsLockFilePath: packageLock, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, externalModules: ['abc'], nodeModules: ['delay'], forceDockerBundling: true, @@ -173,6 +183,7 @@ test('esbuild bundling with esbuild options', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, minify: true, sourceMap: true, target: 'es2020', @@ -224,6 +235,7 @@ test('esbuild bundling source map default', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_14_X, + architecture: Architecture.X86_64, sourceMap: true, sourceMapMode: SourceMapMode.DEFAULT, }); @@ -249,6 +261,7 @@ test('esbuild bundling source map inline', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_14_X, + architecture: Architecture.X86_64, sourceMap: true, sourceMapMode: SourceMapMode.INLINE, }); @@ -274,6 +287,7 @@ test('esbuild bundling source map enabled when only source map mode exists', () projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_14_X, + architecture: Architecture.X86_64, sourceMapMode: SourceMapMode.INLINE, }); @@ -299,6 +313,7 @@ test('esbuild bundling throws when sourceMapMode used with false sourceMap', () projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_14_X, + architecture: Architecture.X86_64, sourceMap: false, sourceMapMode: SourceMapMode.INLINE, }); @@ -312,6 +327,7 @@ test('Detects yarn.lock', () => { projectRoot: path.dirname(yarnLock), depsLockFilePath: yarnLock, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, nodeModules: ['delay'], forceDockerBundling: true, }); @@ -334,6 +350,7 @@ test('Detects pnpm-lock.yaml', () => { projectRoot, depsLockFilePath: pnpmLock, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, nodeModules: ['delay'], forceDockerBundling: true, }); @@ -355,13 +372,14 @@ test('with Docker build args', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, buildArgs: { HELLO: 'WORLD', }, forceDockerBundling: true, }); - expect(fromBuildMock).toHaveBeenCalledWith(expect.stringMatching(/lib$/), expect.objectContaining({ + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/lib$/), expect.objectContaining({ buildArgs: expect.objectContaining({ HELLO: 'WORLD', }), @@ -383,6 +401,7 @@ test('Local bundling', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, environment: { KEY: 'value', }, @@ -403,7 +422,7 @@ test('Local bundling', () => { ); // Docker image is not built - expect(fromBuildMock).not.toHaveBeenCalled(); + expect(DockerImage.fromBuild).not.toHaveBeenCalled(); spawnSyncMock.mockRestore(); }); @@ -420,6 +439,7 @@ test('Incorrect esbuild version', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, }); expect(() => bundler.local?.tryBundle('/outdir', { @@ -433,6 +453,7 @@ test('Custom bundling docker image', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, dockerImage: DockerImage.fromRegistry('my-custom-image'), forceDockerBundling: true, }); @@ -451,6 +472,7 @@ test('with command hooks', () => { projectRoot, depsLockFilePath, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, commandHooks: { beforeBundling(inputDir: string, outputDir: string): string[] { return [ @@ -486,6 +508,7 @@ test('esbuild bundling with projectRoot', () => { depsLockFilePath, tsconfig, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, }); // Correctly bundles with esbuild @@ -508,6 +531,7 @@ test('esbuild bundling with projectRoot and externals and dependencies', () => { projectRoot: repoRoot, depsLockFilePath: packageLock, runtime: Runtime.NODEJS_12_X, + architecture: Architecture.X86_64, externalModules: ['abc'], nodeModules: ['delay'], forceDockerBundling: true, diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index 0c2b4bf624786..722cd2d062fb6 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -27,6 +27,11 @@ export interface BundlingOptions { */ readonly runtime: lambda.Runtime; + /** + * The system architecture of the lambda function + */ + readonly architecture: lambda.Architecture; + /** * Output path suffix ('python' for a layer, '.' otherwise) */ @@ -77,7 +82,7 @@ export interface BundlingOptions { * Produce bundled Lambda asset code */ export function bundle(options: BundlingOptions): lambda.Code { - const { entry, runtime, outputPathSuffix } = options; + const { entry, runtime, architecture, outputPathSuffix } = options; const stagedir = cdk.FileSystem.mkdtemp('python-bundling-'); const hasDeps = stageDependencies(entry, stagedir); @@ -102,6 +107,7 @@ export function bundle(options: BundlingOptions): lambda.Code { buildArgs: { IMAGE: runtime.bundlingImage.image, }, + platform: architecture.dockerPlatform, file: dockerfile, }); diff --git a/packages/@aws-cdk/aws-lambda-python/lib/function.ts b/packages/@aws-cdk/aws-lambda-python/lib/function.ts index 733c115c0383d..84d21b7564909 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/function.ts @@ -102,12 +102,14 @@ export class PythonFunction extends lambda.Function { const handler = props.handler ?? 'handler'; const runtime = props.runtime ?? lambda.Runtime.PYTHON_3_7; + const architecture = props.architecture ?? lambda.Architecture.X86_64; super(scope, id, { ...props, runtime, code: bundle({ runtime, + architecture, entry, outputPathSuffix: '.', assetHashType: props.assetHashType, diff --git a/packages/@aws-cdk/aws-lambda-python/lib/layer.ts b/packages/@aws-cdk/aws-lambda-python/lib/layer.ts index 1a9684e224580..4f247acc10bae 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/layer.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/layer.ts @@ -21,6 +21,12 @@ export interface PythonLayerVersionProps extends lambda.LayerVersionOptions { * @default - All runtimes are supported. */ readonly compatibleRuntimes?: lambda.Runtime[]; + + /** + * The system architectures compatible with this layer. + * @default [Architecture.X86_64] + */ + readonly compatibleArchitectures?: lambda.Architecture[]; } /** @@ -30,6 +36,7 @@ export interface PythonLayerVersionProps extends lambda.LayerVersionOptions { export class PythonLayerVersion extends lambda.LayerVersion { constructor(scope: Construct, id: string, props: PythonLayerVersionProps) { const compatibleRuntimes = props.compatibleRuntimes ?? [lambda.Runtime.PYTHON_3_7]; + const compatibleArchitectures = props.compatibleArchitectures ?? [lambda.Architecture.X86_64]; // Ensure that all compatible runtimes are python for (const runtime of compatibleRuntimes) { @@ -40,8 +47,9 @@ export class PythonLayerVersion extends lambda.LayerVersion { // Entry and defaults const entry = path.resolve(props.entry); - // Pick the first compatibleRuntime to use for bundling or PYTHON_3_7 - const runtime = compatibleRuntimes[0] ?? lambda.Runtime.PYTHON_3_7; + // Pick the first compatibleRuntime and compatibleArchitectures to use for bundling + const runtime = compatibleRuntimes[0]; + const architecture = compatibleArchitectures[0]; super(scope, id, { ...props, @@ -49,6 +57,7 @@ export class PythonLayerVersion extends lambda.LayerVersion { code: bundle({ entry, runtime, + architecture, outputPathSuffix: 'python', }), }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index 712852023a367..01449e0cadeaa 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -1,10 +1,11 @@ import * as fs from 'fs'; import * as path from 'path'; -import { Code, Runtime } from '@aws-cdk/aws-lambda'; -import { FileSystem } from '@aws-cdk/core'; +import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; +import { DockerImage, FileSystem } from '@aws-cdk/core'; import { stageDependencies, bundle } from '../lib/bundling'; jest.spyOn(Code, 'fromAsset'); +jest.spyOn(DockerImage, 'fromBuild'); jest.mock('child_process', () => ({ spawnSync: jest.fn(() => { @@ -28,6 +29,7 @@ test('Bundling a function without dependencies', () => { bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, + architecture: Architecture.X86_64, outputPathSuffix: '.', }); @@ -40,6 +42,13 @@ test('Bundling a function without dependencies', () => { ], }), })); + + expect(DockerImage.fromBuild).toHaveBeenCalledWith(expect.stringMatching(/python-bundling/), expect.objectContaining({ + buildArgs: expect.objectContaining({ + IMAGE: expect.stringMatching(/build-python/), + }), + platform: 'linux/amd64', + })); }); test('Bundling a function with requirements.txt installed', () => { @@ -47,6 +56,7 @@ test('Bundling a function with requirements.txt installed', () => { bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, + architecture: Architecture.X86_64, outputPathSuffix: '.', }); @@ -66,6 +76,7 @@ test('Bundling Python 2.7 with requirements.txt installed', () => { bundle({ entry: entry, runtime: Runtime.PYTHON_2_7, + architecture: Architecture.X86_64, outputPathSuffix: '.', }); @@ -86,6 +97,7 @@ test('Bundling a layer with dependencies', () => { bundle({ entry: entry, runtime: Runtime.PYTHON_2_7, + architecture: Architecture.X86_64, outputPathSuffix: 'python', }); @@ -105,6 +117,7 @@ test('Bundling a python code layer', () => { bundle({ entry: path.join(entry, '.'), runtime: Runtime.PYTHON_2_7, + architecture: Architecture.X86_64, outputPathSuffix: 'python', }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json index e48b9c2e0ad92..ed3d577fb40a3 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json @@ -36,7 +36,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2S3Bucket383ED51E" + "Ref": "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374S3Bucket07AE44EE" }, "S3Key": { "Fn::Join": [ @@ -49,7 +49,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2S3VersionKeyA520554C" + "Ref": "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374S3VersionKey01F8F2A1" } ] } @@ -62,7 +62,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2S3VersionKeyA520554C" + "Ref": "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374S3VersionKey01F8F2A1" } ] } @@ -87,17 +87,17 @@ } }, "Parameters": { - "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2S3Bucket383ED51E": { + "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374S3Bucket07AE44EE": { "Type": "String", - "Description": "S3 bucket for asset \"fc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2\"" + "Description": "S3 bucket for asset \"3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374\"" }, - "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2S3VersionKeyA520554C": { + "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374S3VersionKey01F8F2A1": { "Type": "String", - "Description": "S3 key for asset version \"fc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2\"" + "Description": "S3 key for asset version \"3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374\"" }, - "AssetParametersfc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2ArtifactHashB863A6ED": { + "AssetParameters3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374ArtifactHashDECBC32A": { "Type": "String", - "Description": "Artifact hash for asset \"fc7bfbf72c74b955f7bc25d2bb123c0eeec9557cda17481146d51672768907b2\"" + "Description": "Artifact hash for asset \"3dc2f7b8375fbf383f44eb8f798d324f60d516946c9f829fca3c5f747f973374\"" } }, "Outputs": { diff --git a/packages/@aws-cdk/aws-lambda/lib/architecture.ts b/packages/@aws-cdk/aws-lambda/lib/architecture.ts index 40edee1896755..a0f53ee2bea69 100644 --- a/packages/@aws-cdk/aws-lambda/lib/architecture.ts +++ b/packages/@aws-cdk/aws-lambda/lib/architecture.ts @@ -10,15 +10,15 @@ export class Architecture { /** * 64 bit architecture with the ARM instruction set. */ - public static readonly ARM_64 = new Architecture('arm64'); + public static readonly ARM_64 = new Architecture('arm64', 'linux/arm64'); /** * Used to specify a custom architecture name. * Use this if the architecture name is not yet supported by the CDK. * @param name the architecture name as recognized by AWS Lambda. */ - public static custom(name: string) { - return new Architecture(name); + public static custom(name: string, dockerPlatform?: string) { + return new Architecture(name, dockerPlatform); } /** @@ -26,7 +26,13 @@ export class Architecture { */ public readonly name: string; - private constructor(archName: string) { + /** + * The platform to use for this architecture when building with Docker. + */ + public readonly dockerPlatform: string; + + private constructor(archName: string, dockerPlatform?: string) { this.name = archName; + this.dockerPlatform = dockerPlatform ?? 'linux/amd64'; } -} \ No newline at end of file +} From 93dc02a88c094dc890dc26d798cb33489020de42 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 7 Oct 2021 21:42:24 +0200 Subject: [PATCH 2/5] README --- packages/@aws-cdk/aws-lambda-go/README.md | 2 +- packages/@aws-cdk/aws-lambda-nodejs/README.md | 3 ++- packages/@aws-cdk/aws-lambda-python/README.md | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 5fb2907b0bf7a..9f5ba745c83df 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -124,7 +124,7 @@ new lambda.GoFunction(this, 'handler', { ## Local Bundling -If `Go` is installed locally and the version is >= `go1.11` then it will be used to bundle your code in your environment. Otherwise, bundling will happen in a [Lambda compatible Docker container](https://hub.docker.com/layers/lambci/lambda/build-go1.x/images/sha256-e14dab718ed0bb06b2243825c5993e494a6969de7c01754ad7e80dacfce9b0cf?context=explore). +If `Go` is installed locally and the version is >= `go1.11` then it will be used to bundle your code in your environment. Otherwise, bundling will happen in a [Lambda compatible Docker container](https://gallery.ecr.aws/sam/build-go1.x) with the Docker platform based on the target architecture of the Lambda function. For macOS the recommended approach is to install `Go` as Docker volume performance is really poor. diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 22347b28b1dd9..2c0572231a8ae 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -99,7 +99,8 @@ used by your function. Otherwise bundling will fail. ## Local bundling If `esbuild` is available it will be used to bundle your code in your environment. Otherwise, -bundling will happen in a [Lambda compatible Docker container](https://gallery.ecr.aws/sam/build-nodejs12.x). +bundling will happen in a [Lambda compatible Docker container](https://gallery.ecr.aws/sam/build-nodejs12.x) +with the Docker platform based on the target architecture of the Lambda function. For macOS the recommendend approach is to install `esbuild` as Docker volume performance is really poor. diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index ffd19568aa5dc..4106b6210b871 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -42,11 +42,11 @@ All other properties of `lambda.Function` are supported, see also the [AWS Lambd If `requirements.txt` or `Pipfile` exists at the entry path, the construct will handle installing all required modules in a [Lambda compatible Docker container](https://gallery.ecr.aws/sam/build-python3.7) -according to the `runtime`. +according to the `runtime` and with the Docker platform based on the target architecture of the Lambda function. -Python bundles are only recreated and published when a file in a source directory has changed. +Python bundles are only recreated and published when a file in a source directory has changed. Therefore (and as a general best-practice), it is highly recommended to commit a lockfile with a -list of all transitive dependencies and their exact versions. +list of all transitive dependencies and their exact versions. This will ensure that when any dependency version is updated, the bundle asset is recreated and uploaded. To that end, we recommend using [`pipenv`] or [`poetry`] which has lockfile support. From 52dbdc3d361a7a0772e38d06b4e1b91c2ab8aa3d Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Thu, 7 Oct 2021 22:28:08 +0200 Subject: [PATCH 3/5] GOARCH --- packages/@aws-cdk/aws-lambda-go/README.md | 2 +- packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 9f5ba745c83df..16fcffee919ea 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -106,7 +106,7 @@ All other properties of `lambda.Function` are supported, see also the [AWS Lambd By default the following environment variables are set for you: * `GOOS=linux` -* `GOARCH=amd64` +* `GOARCH`: based on the target architecture of the Lambda function * `GO111MODULE=on` Use the `environment` prop to define additional environment variables when go runs: diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index d7251688dfa10..afc233479ef3b 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -109,7 +109,7 @@ export class Bundling implements cdk.BundlingOptions { const environment = { CGO_ENABLED: cgoEnabled, GO111MODULE: 'on', - GOARCH: 'amd64', + GOARCH: props.architecture.dockerPlatform.split('/')[1], GOOS: 'linux', ...props.environment, }; From b96e24ddf08d3f4f13b6f454776db7b898c0d4a4 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 11 Oct 2021 09:40:48 +0200 Subject: [PATCH 4/5] PR feedback --- packages/@aws-cdk/aws-lambda/lib/architecture.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/architecture.ts b/packages/@aws-cdk/aws-lambda/lib/architecture.ts index a0f53ee2bea69..db8cba5bede50 100644 --- a/packages/@aws-cdk/aws-lambda/lib/architecture.ts +++ b/packages/@aws-cdk/aws-lambda/lib/architecture.ts @@ -5,7 +5,7 @@ export class Architecture { /** * 64 bit architecture with x86 instruction set. */ - public static readonly X86_64 = new Architecture('x86_64'); + public static readonly X86_64 = new Architecture('x86_64', 'linux/amd64'); /** * 64 bit architecture with the ARM instruction set. @@ -16,9 +16,10 @@ export class Architecture { * Used to specify a custom architecture name. * Use this if the architecture name is not yet supported by the CDK. * @param name the architecture name as recognized by AWS Lambda. + * @param [dockerPlatform=linux/amd64] the docker platform to use when building a container for this architecture */ public static custom(name: string, dockerPlatform?: string) { - return new Architecture(name, dockerPlatform); + return new Architecture(name, dockerPlatform ?? 'linux/amd64'); } /** @@ -31,8 +32,8 @@ export class Architecture { */ public readonly dockerPlatform: string; - private constructor(archName: string, dockerPlatform?: string) { + private constructor(archName: string, dockerPlatform: string) { this.name = archName; - this.dockerPlatform = dockerPlatform ?? 'linux/amd64'; + this.dockerPlatform = dockerPlatform; } } From c9a53f2d3b5063db22c5dd0afa4c579c593024e8 Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Mon, 11 Oct 2021 09:42:36 +0200 Subject: [PATCH 5/5] JSDoc --- packages/@aws-cdk/aws-lambda/lib/architecture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/architecture.ts b/packages/@aws-cdk/aws-lambda/lib/architecture.ts index db8cba5bede50..ae86624ca006b 100644 --- a/packages/@aws-cdk/aws-lambda/lib/architecture.ts +++ b/packages/@aws-cdk/aws-lambda/lib/architecture.ts @@ -16,7 +16,7 @@ export class Architecture { * Used to specify a custom architecture name. * Use this if the architecture name is not yet supported by the CDK. * @param name the architecture name as recognized by AWS Lambda. - * @param [dockerPlatform=linux/amd64] the docker platform to use when building a container for this architecture + * @param [dockerPlatform=linux/amd64] the platform to use for this architecture when building with Docker */ public static custom(name: string, dockerPlatform?: string) { return new Architecture(name, dockerPlatform ?? 'linux/amd64');