From 98a6fbb9b4a4a2b4ba3d114e8d3ad959063514a0 Mon Sep 17 00:00:00 2001 From: Ryan Andonian Date: Mon, 30 Jan 2023 11:51:56 -0800 Subject: [PATCH] feat(aws-lambda-python): support for poetry bundle asset exclusion list --- packages/@aws-cdk/aws-lambda-python/README.md | 14 +++++ .../aws-lambda-python/lib/bundling.ts | 16 ++++- .../@aws-cdk/aws-lambda-python/lib/types.ts | 7 +++ .../aws-lambda-python/test/bundling.test.ts | 60 ++++++++++++++++--- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index f399d1559c81a..ac8c5780cbc37 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -109,6 +109,20 @@ Packaging is executed using the `Packaging` class, which: ├── poetry.lock # your poetry lock file has to be present at the entry path ``` +If using `poetry`, particularly with `virtualenvs.in-project = true`, you can exclude specific files from the copied files using the optional bundling string array parameter `poetryAssetExcludes` + +```ts +new python.PythonFunction(this, 'function', { + entry: '/path/to/poetry-function', + runtime: Runtime.PYTHON_3_8, + bundling: { + // translates to `rsync --exclude='.venv'` + poetryAssetExcludes: ['.venv'], + }, +}); +``` + + ## Custom Bundling Custom bundling can be performed by passing in additional build arguments that point to index URLs to private repos, or by using an entirely custom Docker images for bundling dependencies. The build args currently supported are: diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index fad11266bb0cb..dc73c0ea437f3 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -47,6 +47,12 @@ export interface BundlingProps extends BundlingOptions { * @default - BundlingFileAccess.BIND_MOUNT */ bundlingFileAccess?: BundlingFileAccess + + /** + * List of file patterns to exclude when copying assets for bundling + * @default - Empty list + */ + readonly assetExcludes?: string[]; } /** @@ -83,6 +89,7 @@ export class Bundling implements CdkBundlingOptions { image, poetryIncludeHashes, commandHooks, + assetExcludes = [], } = props; const outputPath = path.posix.join(AssetStaging.BUNDLING_OUTPUT_DIR, outputPathSuffix); @@ -93,6 +100,7 @@ export class Bundling implements CdkBundlingOptions { outputDir: outputPath, poetryIncludeHashes, commandHooks, + assetExcludes, }); this.image = image ?? DockerImage.fromBuild(path.join(__dirname, '../lib'), { @@ -118,7 +126,10 @@ export class Bundling implements CdkBundlingOptions { const packaging = Packaging.fromEntry(options.entry, options.poetryIncludeHashes); let bundlingCommands: string[] = []; bundlingCommands.push(...options.commandHooks?.beforeBundling(options.inputDir, options.outputDir) ?? []); - bundlingCommands.push(`cp -rTL ${options.inputDir}/ ${options.outputDir}`); + const exclusionStr = options.assetExcludes?.map(item => `--exclude='${item}'`).join(' '); + bundlingCommands.push([ + 'rsync', '-rLv', exclusionStr ?? '', `${options.inputDir}/`, options.outputDir, + ].filter(item => item).join(' ')); bundlingCommands.push(`cd ${options.outputDir}`); bundlingCommands.push(packaging.exportCommand ?? ''); if (packaging.dependenciesFile) { @@ -134,7 +145,8 @@ interface BundlingCommandOptions { readonly inputDir: string; readonly outputDir: string; readonly poetryIncludeHashes?: boolean; - readonly commandHooks?: ICommandHooks + readonly commandHooks?: ICommandHooks; + readonly assetExcludes?: string[]; } /** diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index e0d328e68c858..80003cab8a327 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -15,6 +15,13 @@ export interface BundlingOptions extends DockerRunOptions { */ readonly poetryIncludeHashes?: boolean; + /** + * When using Poetry bundler, list of file patterns to exclude when copying assets for bundling. + * + * @default - Empty list + */ + readonly poetryAssetExcludes?: string[]; + /** * Output path suffix: the suffix for the directory into which the bundled output is written. * diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index b51e8bb046acc..0069ed8c3b8c9 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -37,7 +37,7 @@ test('Bundling a function without dependencies', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output && cd /asset-output', + 'rsync -rLv /asset-input/ /asset-output && cd /asset-output', ], }), })); @@ -66,7 +66,7 @@ test('Bundling a function with requirements.txt', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output', + 'rsync -rLv /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output', ], }), })); @@ -89,7 +89,7 @@ test('Bundling Python 2.7 with requirements.txt installed', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output', + 'rsync -rLv /asset-input/ /asset-output && cd /asset-output && python -m pip install -r requirements.txt -t /asset-output', ], }), })); @@ -109,7 +109,7 @@ test('Bundling a layer with dependencies', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python', ], }), })); @@ -129,7 +129,7 @@ test('Bundling a python code layer', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python', ], }), })); @@ -149,7 +149,7 @@ test('Bundling a function with pipenv dependencies', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && PIPENV_VENV_IN_PROJECT=1 pipenv lock -r > requirements.txt && rm -rf .venv && python -m pip install -r requirements.txt -t /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && PIPENV_VENV_IN_PROJECT=1 pipenv lock -r > requirements.txt && rm -rf .venv && python -m pip install -r requirements.txt -t /asset-output/python', ], }), })); @@ -176,7 +176,7 @@ test('Bundling a function with poetry dependencies', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python', ], }), })); @@ -189,6 +189,48 @@ test('Bundling a function with poetry dependencies', () => { expect(files).toContain('.ignorefile'); }); +test('Bundling a function with poetry and assetExcludes', () => { + const entry = path.join(__dirname, 'lambda-handler-poetry'); + + Bundling.bundle({ + entry: path.join(entry, '.'), + runtime: Runtime.PYTHON_3_9, + architecture: Architecture.X86_64, + outputPathSuffix: 'python', + assetExcludes: ['.ignorefile'], + }); + + expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ + bundling: expect.objectContaining({ + command: [ + 'bash', '-c', + "rsync -rLv --exclude='.ignorefile' /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --without-hashes --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python", + ], + }), + })); + +}); + +test('Bundling a function with poetry and no assetExcludes', () => { + const entry = path.join(__dirname, 'lambda-handler-poetry'); + + Bundling.bundle({ + entry: path.join(entry, '.'), + runtime: Runtime.PYTHON_3_9, + architecture: Architecture.X86_64, + outputPathSuffix: 'python', + }); + + expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ + bundling: expect.objectContaining({ + command: [ + 'bash', '-c', + expect.not.stringContaining('--exclude'), + ], + }), + })); +}); + test('Bundling a function with poetry dependencies, with hashes', () => { const entry = path.join(__dirname, 'lambda-handler-poetry'); @@ -204,7 +246,7 @@ test('Bundling a function with poetry dependencies, with hashes', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && poetry export --with-credentials --format requirements.txt --output requirements.txt && python -m pip install -r requirements.txt -t /asset-output/python', ], }), })); @@ -234,7 +276,7 @@ test('Bundling a function with custom bundling image', () => { image, command: [ 'bash', '-c', - 'cp -rTL /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python', + 'rsync -rLv /asset-input/ /asset-output/python && cd /asset-output/python && python -m pip install -r requirements.txt -t /asset-output/python', ], }), }));