From 8527675e1cf83519a211c8b4cc43161ac29757f1 Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Wed, 11 Sep 2024 11:41:22 +0100 Subject: [PATCH] feat: experimental workers assets can be ignored by adding a .assetsignore file (#6640) * feat: experimental workers assets can be ignored by adding a .cfassetsignore file * fixup! feat: experimental workers assets can be ignored by adding a .cfassetsignore file --- .changeset/nasty-hats-rhyme.md | 10 +++++ .vscode/settings.json | 2 + .../wrangler/src/__tests__/deploy.test.ts | 41 +++++++++++++++++++ packages/wrangler/src/experimental-assets.ts | 36 ++++++++++++++++ packages/wrangler/src/sites.ts | 14 +------ packages/wrangler/src/utils/filesystem.ts | 16 ++++++++ 6 files changed, 106 insertions(+), 13 deletions(-) create mode 100644 .changeset/nasty-hats-rhyme.md diff --git a/.changeset/nasty-hats-rhyme.md b/.changeset/nasty-hats-rhyme.md new file mode 100644 index 000000000000..a6d754c6d60f --- /dev/null +++ b/.changeset/nasty-hats-rhyme.md @@ -0,0 +1,10 @@ +--- +"wrangler": minor +--- + +feat: experimental workers assets can be ignored by adding a .assetsignore file + +This file can be added to the root of the assets directory that is to be uploaded alongside the Worker +when using `experimental_assets`. + +The file follows the `.gitignore` syntax, and any matching paths will not be included in the upload. diff --git a/.vscode/settings.json b/.vscode/settings.json index 09b6e6507cd5..af3e13c4c8a8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "editor.defaultFormatter": "esbenp.prettier-vscode", "cSpell.words": [ "Abortable", + "assetsignore", "cfetch", "chatgpt", "clipboardy", @@ -11,6 +12,7 @@ "esbuild", "eslintcache", "execa", + "filestat", "haikunate", "haikunator", "httplogs", diff --git a/packages/wrangler/src/__tests__/deploy.test.ts b/packages/wrangler/src/__tests__/deploy.test.ts index 54b68b901b2b..bd94d079db57 100644 --- a/packages/wrangler/src/__tests__/deploy.test.ts +++ b/packages/wrangler/src/__tests__/deploy.test.ts @@ -4381,6 +4381,47 @@ addEventListener('fetch', event => {});` }); }); + it("should ignore assets that match patterns in an .assetsignore file in the root of the assets directory", async () => { + const assets = [ + { filePath: ".assetsignore", content: "*.bak\nsub-dir" }, + { filePath: "file-1.txt", content: "Content of file-1" }, + { filePath: "file-2.bak", content: "Content of file-2" }, + { filePath: "file-3.txt", content: "Content of file-3" }, + { filePath: "sub-dir/file-4.bak", content: "Content of file-4" }, + { filePath: "sub-dir/file-5.txt", content: "Content of file-5" }, + ]; + writeAssets(assets, "some/path/assets"); + writeWranglerToml( + { + experimental_assets: { directory: "assets" }, + }, + "some/path/wrangler.toml" + ); + const bodies: AssetManifest[] = []; + await mockAUSRequest(bodies); + mockSubDomainRequest(); + mockUploadWorkerRequest({ + expectedExperimentalAssets: true, + expectedType: "none", + }); + await runWrangler("deploy --config some/path/wrangler.toml"); + expect(bodies.length).toBe(1); + expect(bodies[0]).toMatchInlineSnapshot(` + Object { + "manifest": Object { + "/file-1.txt": Object { + "hash": "0de3dd5df907418e9730fd2bd747bd5e", + "size": 17, + }, + "/file-3.txt": Object { + "hash": "ff5016e92f039aa743a4ff7abb3180fa", + "size": 17, + }, + }, + } + `); + }); + it("should resolve assets directory relative to cwd if using cli", async () => { const assets = [{ filePath: "file-1.txt", content: "Content of file-1" }]; writeAssets(assets, "some/path/assets"); diff --git a/packages/wrangler/src/experimental-assets.ts b/packages/wrangler/src/experimental-assets.ts index 211a2f6adc06..47f587adb099 100644 --- a/packages/wrangler/src/experimental-assets.ts +++ b/packages/wrangler/src/experimental-assets.ts @@ -14,6 +14,7 @@ import { logger, LOGGER_LEVELS } from "./logger"; import { hashFile } from "./pages/hash"; import { isJwtExpired } from "./pages/upload"; import { APIError } from "./parse"; +import { createPatternMatcher } from "./utils/filesystem"; import type { Config } from "./config"; import type { ExperimentalAssets } from "./config/environment"; @@ -220,10 +221,20 @@ export const buildAssetsManifest = async (dir: string) => { const files = await readdir(dir, { recursive: true }); const manifest: AssetManifest = {}; let counter = 0; + + const ignoreFn = await createAssetIgnoreFunction(dir); + await Promise.all( files.map(async (file) => { const filepath = path.join(dir, file); const relativeFilepath = path.relative(dir, filepath); + + if (ignoreFn?.(relativeFilepath)) { + logger.debug("Ignoring asset:", relativeFilepath); + // This file should not be included in the manifest. + return; + } + const filestat = await stat(filepath); if (filestat.isSymbolicLink() || filestat.isDirectory()) { @@ -361,3 +372,28 @@ const decodeFilepath = (filePath: string) => { .map((segment) => decodeURIComponent(segment)) .join(path.sep); }; + +/** + * Create a function for filtering out ignored assets. + * + * The generated function takes an asset path, relative to the asset directory, + * and returns true if the asset should not be ignored. + */ +async function createAssetIgnoreFunction(dir: string) { + const CF_ASSETS_IGNORE_FILENAME = ".assetsignore"; + + const cfAssetIgnorePath = path.resolve(dir, CF_ASSETS_IGNORE_FILENAME); + + if (!existsSync(cfAssetIgnorePath)) { + return null; + } + + const ignorePatterns = ( + await readFile(cfAssetIgnorePath, { encoding: "utf8" }) + ).split("\n"); + + // Always ignore the `.assetsignore` file. + ignorePatterns.push(CF_ASSETS_IGNORE_FILENAME); + + return createPatternMatcher(ignorePatterns, true); +} diff --git a/packages/wrangler/src/sites.ts b/packages/wrangler/src/sites.ts index 649f9fb278bd..1f189c621191 100644 --- a/packages/wrangler/src/sites.ts +++ b/packages/wrangler/src/sites.ts @@ -2,7 +2,6 @@ import assert from "node:assert"; import { readdir, readFile, stat } from "node:fs/promises"; import * as path from "node:path"; import chalk from "chalk"; -import ignore from "ignore"; import xxhash from "xxhash-wasm"; import { UserError } from "./errors"; import { @@ -17,6 +16,7 @@ import { putKVKeyValue, } from "./kv/helpers"; import { logger, LOGGER_LEVELS } from "./logger"; +import { createPatternMatcher } from "./utils/filesystem"; import type { Config } from "./config"; import type { KeyValue } from "./kv/helpers"; import type { XXHashAPI } from "xxhash-wasm"; @@ -391,18 +391,6 @@ export async function syncLegacyAssets( return { manifest, namespace }; } -function createPatternMatcher( - patterns: string[], - exclude: boolean -): (filePath: string) => boolean { - if (patterns.length === 0) { - return (_filePath) => !exclude; - } else { - const ignorer = ignore().add(patterns); - return (filePath) => ignorer.test(filePath).ignored; - } -} - /** * validate that the passed-in file is below 25 MiB * **PRIOR** to base64 encoding. 25 MiB is a KV limit diff --git a/packages/wrangler/src/utils/filesystem.ts b/packages/wrangler/src/utils/filesystem.ts index 390eb7d24169..c0866d0a709f 100644 --- a/packages/wrangler/src/utils/filesystem.ts +++ b/packages/wrangler/src/utils/filesystem.ts @@ -1,6 +1,7 @@ import { mkdirSync } from "fs"; import { mkdir } from "fs/promises"; import path from "path"; +import ignore from "ignore"; export async function ensureDirectoryExists(filepath: string) { const dirpath = path.dirname(filepath); @@ -13,3 +14,18 @@ export function ensureDirectoryExistsSync(filepath: string) { mkdirSync(dirpath, { recursive: true }); } + +/** + * Generate a function that can match relative filepaths against a list of gitignore formatted patterns. + */ +export function createPatternMatcher( + patterns: string[], + exclude: boolean +): (filePath: string) => boolean { + if (patterns.length === 0) { + return (_filePath) => !exclude; + } else { + const ignorer = ignore().add(patterns); + return (filePath) => ignorer.test(filePath).ignored; + } +}