From 5205bc12bd2b3b41c03b86923a43736accc5a931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 30 Jun 2021 16:24:25 +0100 Subject: [PATCH 1/2] refactor: move function builders to functions lib --- src/function-builder-detectors/README.md | 17 -- .../runtimes/js/builders}/netlify-lambda.js | 2 +- .../js/builders}/tests/netlify-lambda.test.js | 0 .../functions/runtimes/js/builders/zisi.js | 251 ++++++++++++++++++ src/utils/detect-functions-builder.js | 5 +- 5 files changed, 255 insertions(+), 20 deletions(-) delete mode 100644 src/function-builder-detectors/README.md rename src/{function-builder-detectors => lib/functions/runtimes/js/builders}/netlify-lambda.js (96%) rename src/{function-builder-detectors => lib/functions/runtimes/js/builders}/tests/netlify-lambda.test.js (100%) create mode 100644 src/lib/functions/runtimes/js/builders/zisi.js diff --git a/src/function-builder-detectors/README.md b/src/function-builder-detectors/README.md deleted file mode 100644 index 52af1c62c9e..00000000000 --- a/src/function-builder-detectors/README.md +++ /dev/null @@ -1,17 +0,0 @@ -## function builder detectors - -similar to project detectors, each file here detects function builders. this is so that netlify dev never manages the -webpack or other config. the expected output is very simple: - -```js -module.exports = { - src: 'functions-source', // source for your functions - build: () => {}, // chokidar will call this to build and rebuild your function - npmScript: 'build:functions', // optional, the matching package.json script that calls your function builder -} -``` - -example - -- [src](https://github.com/netlify/cli/blob/f7b7c6adda3903fa02cf1b3fadcef026a4e56c13/src/function-builder-detectors/netlify-lambda.js#L22) -- [npmScript](https://github.com/netlify/cli/blob/f7b7c6adda3903fa02cf1b3fadcef026a4e56c13/src/function-builder-detectors/netlify-lambda.js#L23) diff --git a/src/function-builder-detectors/netlify-lambda.js b/src/lib/functions/runtimes/js/builders/netlify-lambda.js similarity index 96% rename from src/function-builder-detectors/netlify-lambda.js rename to src/lib/functions/runtimes/js/builders/netlify-lambda.js index ed5d728572b..f4889751846 100644 --- a/src/function-builder-detectors/netlify-lambda.js +++ b/src/lib/functions/runtimes/js/builders/netlify-lambda.js @@ -2,7 +2,7 @@ const execa = require('execa') const debounce = require('lodash/debounce') const minimist = require('minimist') -const { fileExistsAsync, readFileAsync } = require('../lib/fs') +const { fileExistsAsync, readFileAsync } = require('../../../../fs') const DEBOUNCE_WAIT = 300 diff --git a/src/function-builder-detectors/tests/netlify-lambda.test.js b/src/lib/functions/runtimes/js/builders/tests/netlify-lambda.test.js similarity index 100% rename from src/function-builder-detectors/tests/netlify-lambda.test.js rename to src/lib/functions/runtimes/js/builders/tests/netlify-lambda.test.js diff --git a/src/lib/functions/runtimes/js/builders/zisi.js b/src/lib/functions/runtimes/js/builders/zisi.js new file mode 100644 index 00000000000..7e964c9a215 --- /dev/null +++ b/src/lib/functions/runtimes/js/builders/zisi.js @@ -0,0 +1,251 @@ +const path = require('path') + +const { zipFunction, zipFunctions } = require('@netlify/zip-it-and-ship-it') +const del = require('del') +const makeDir = require('make-dir') +const pFilter = require('p-filter') +const sourceMapSupport = require('source-map-support') + +const { getFunctions } = require('../../../../../utils/get-functions') +const { NETLIFYDEVERR } = require('../../../../../utils/logo') +const { getPathInProject } = require('../../../../settings') +const { normalizeFunctionsConfig } = require('../../../config') +const { memoizedBuild } = require('../../../memoized-build') + +const ZIP_CONCURRENCY = 5 + +const addFunctionsConfigDefaults = (config) => ({ + ...config, + '*': { + nodeSourcemap: true, + ...config['*'], + }, +}) + +const addFunctionToTree = (func, fileTree) => { + // Transforming the inputs into a Set so that we can have a O(1) lookup. + const inputs = new Set(func.inputs) + + // The `mainFile` property returned from ZISI will point to the original main + // function file, but we want to serve the bundled version, which we set as + // the `bundleFile` property. + const bundleFile = path.join(func.path, `${func.name}.js`) + + fileTree.set(func.mainFile, { ...func, bundleFile, inputs }) +} + +const zipFunctionsAndUpdateTree = async ({ fileTree, functions, sourceDirectory, targetDirectory, zipOptions }) => { + if (functions !== undefined) { + await pFilter( + functions, + async ({ mainFile }) => { + const functionDirectory = path.dirname(mainFile) + + // If we have a function at `functions/my-func/index.js` and we pass + // that path to `zipFunction`, it will lack the context of the whole + // functions directory and will infer the name of the function to be + // `index`, not `my-func`. Instead, we need to pass the directory of + // the function. The exception is when the function is a file at the + // root of the functions directory (e.g. `functions/my-func.js`). In + // this case, we use `mainFile` as the function path of `zipFunction`. + const entryPath = functionDirectory === sourceDirectory ? mainFile : functionDirectory + const func = await memoizedBuild({ + cacheKey: `zisi-${entryPath}`, + command: () => zipFunction(entryPath, targetDirectory, zipOptions), + }) + + addFunctionToTree(func, fileTree) + }, + { concurrency: ZIP_CONCURRENCY }, + ) + + return + } + + const result = await memoizedBuild({ + cacheKey: 'zisi@all', + command: () => zipFunctions(sourceDirectory, targetDirectory, zipOptions), + }) + + result.forEach((func) => { + addFunctionToTree(func, fileTree) + }) +} + +// Bundles a set of functions based on the event type and the updated path. It +// returns an array with the paths of the functions that were rebundled as a +// result of the update event. +const bundleFunctions = async ({ + config, + eventType, + fileTree, + projectRoot, + sourceDirectory, + targetDirectory, + updatedPath, + zipCache, +}) => { + const zipOptions = { + archiveFormat: 'none', + basePath: projectRoot, + config, + } + + if (eventType === 'add') { + // We first check to see if the file being added is associated with any + // functions (e.g. restoring a file that has been previously deleted). + // If that's the case, we bundle just those functions. + const functionsWithPath = [...fileTree.entries()].filter(([, { inputs }]) => inputs.has(updatedPath)) + + if (functionsWithPath.length !== 0) { + await zipFunctionsAndUpdateTree({ + fileTree, + functions: functionsWithPath.map(([, func]) => func), + sourceDirectory, + targetDirectory, + zipCache, + zipOptions, + }) + + return functionsWithPath.map(([mainFile]) => mainFile) + } + + // We then check whether the newly-added file is itself a function. If so, + // we bundle it. + const functions = await getFunctions(sourceDirectory) + const matchingFunction = functions.find(({ mainFile }) => mainFile === updatedPath) + + if (matchingFunction !== undefined) { + await zipFunctionsAndUpdateTree({ + fileTree, + functions: [matchingFunction], + sourceDirectory, + targetDirectory, + zipCache, + zipOptions, + }) + + return [updatedPath] + } + + // At this point, the newly-added file is neither a function nor a file + // associated with a function, so we can discard the update. + return + } + + if (eventType === 'change' || eventType === 'unlink') { + // If the file matches a function's main file, we just need to operate on + // that one function. + if (fileTree.has(updatedPath)) { + const matchingFunction = fileTree.get(updatedPath) + + // We bundle the function if this is a `change` event, or delete it if + // the event is `unlink`. + if (eventType === 'change') { + await zipFunctionsAndUpdateTree({ + fileTree, + functions: [matchingFunction], + sourceDirectory, + targetDirectory, + zipCache, + zipOptions, + }) + } else { + const { path: functionPath } = matchingFunction + + fileTree.delete(updatedPath) + + await del(functionPath, { force: true }) + } + + return [updatedPath] + } + + // The update is in one of the supporting files. We bundle every function + // that uses it. + const functions = [...fileTree.entries()].filter(([, { inputs }]) => inputs.has(updatedPath)) + + await zipFunctionsAndUpdateTree({ + fileTree, + functions: functions.map(([, func]) => func), + sourceDirectory, + targetDirectory, + zipCache, + zipOptions, + }) + + return functions.map(([mainFile]) => mainFile) + } + + // Deleting the target directory so that we can start from a clean slate. + try { + await del(targetDirectory, { force: true }) + } catch (_) { + // no-op + } + + // Bundling all functions. + await zipFunctionsAndUpdateTree({ + fileTree, + sourceDirectory, + targetDirectory, + zipCache, + zipOptions, + }) +} + +const getFunctionByName = ({ cache, name }) => [...cache.values()].find((func) => func.name === name) + +const getTargetDirectory = async ({ errorExit }) => { + const targetDirectory = path.resolve(getPathInProject(['functions-serve'])) + + try { + await makeDir(targetDirectory) + } catch (error) { + errorExit(`${NETLIFYDEVERR} Could not create directory: ${targetDirectory}`) + } + + return targetDirectory +} + +module.exports = async function handler({ config, errorExit, functionsDirectory: sourceDirectory, projectRoot }) { + const functions = await getFunctions(sourceDirectory) + const hasTSFunction = functions.some(({ mainFile }) => path.extname(mainFile) === '.ts') + const functionsConfig = addFunctionsConfigDefaults( + normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }), + ) + const isUsingEsbuild = functionsConfig['*'].nodeBundler === 'esbuild_zisi' + + if (!hasTSFunction && !isUsingEsbuild) { + return false + } + + // Enable source map support. + sourceMapSupport.install() + + // Keeps track of which files are associated with each function. + const fileTree = new Map() + + // Used for memoizing calls to ZISI, such that we don't bundle the same + // function multiple times at the same time. + const zipCache = {} + const targetDirectory = await getTargetDirectory({ errorExit }) + + return { + build: (updatedPath, eventType) => + bundleFunctions({ + config: functionsConfig, + eventType, + fileTree, + projectRoot, + sourceDirectory, + targetDirectory, + updatedPath, + zipCache, + }), + builderName: 'zip-it-and-ship-it', + getFunctionByName: (name) => getFunctionByName({ cache: fileTree, name }), + src: sourceDirectory, + target: targetDirectory, + } +} diff --git a/src/utils/detect-functions-builder.js b/src/utils/detect-functions-builder.js index c3bf5054db3..860f2c29aef 100644 --- a/src/utils/detect-functions-builder.js +++ b/src/utils/detect-functions-builder.js @@ -2,14 +2,15 @@ const fs = require('fs') const path = require('path') const detectFunctionsBuilder = async function (parameters) { + const buildersPath = path.join(__dirname, '..', 'lib', 'functions', 'runtimes', 'js', 'builders') const detectors = fs - .readdirSync(path.join(__dirname, '..', 'function-builder-detectors')) + .readdirSync(buildersPath) // only accept .js detector files .filter((filename) => filename.endsWith('.js')) // Sorting by filename .sort() // eslint-disable-next-line node/global-require, import/no-dynamic-require - .map((det) => require(path.join(__dirname, '..', `function-builder-detectors/${det}`))) + .map((det) => require(path.join(buildersPath, det))) for (const detector of detectors) { // eslint-disable-next-line no-await-in-loop From 3f463adcd28455547732c979599edf8efe7e7f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 30 Jun 2021 16:27:42 +0100 Subject: [PATCH 2/2] chore: remove old zisi builder file --- src/function-builder-detectors/zisi.js | 251 ------------------------- 1 file changed, 251 deletions(-) delete mode 100644 src/function-builder-detectors/zisi.js diff --git a/src/function-builder-detectors/zisi.js b/src/function-builder-detectors/zisi.js deleted file mode 100644 index 9f16df06a11..00000000000 --- a/src/function-builder-detectors/zisi.js +++ /dev/null @@ -1,251 +0,0 @@ -const path = require('path') - -const { zipFunction, zipFunctions } = require('@netlify/zip-it-and-ship-it') -const del = require('del') -const makeDir = require('make-dir') -const pFilter = require('p-filter') -const sourceMapSupport = require('source-map-support') - -const { normalizeFunctionsConfig } = require('../lib/functions/config') -const { memoizedBuild } = require('../lib/functions/memoized-build') -const { getPathInProject } = require('../lib/settings') -const { getFunctions } = require('../utils/get-functions') -const { NETLIFYDEVERR } = require('../utils/logo') - -const ZIP_CONCURRENCY = 5 - -const addFunctionsConfigDefaults = (config) => ({ - ...config, - '*': { - nodeSourcemap: true, - ...config['*'], - }, -}) - -const addFunctionToTree = (func, fileTree) => { - // Transforming the inputs into a Set so that we can have a O(1) lookup. - const inputs = new Set(func.inputs) - - // The `mainFile` property returned from ZISI will point to the original main - // function file, but we want to serve the bundled version, which we set as - // the `bundleFile` property. - const bundleFile = path.join(func.path, `${func.name}.js`) - - fileTree.set(func.mainFile, { ...func, bundleFile, inputs }) -} - -const zipFunctionsAndUpdateTree = async ({ fileTree, functions, sourceDirectory, targetDirectory, zipOptions }) => { - if (functions !== undefined) { - await pFilter( - functions, - async ({ mainFile }) => { - const functionDirectory = path.dirname(mainFile) - - // If we have a function at `functions/my-func/index.js` and we pass - // that path to `zipFunction`, it will lack the context of the whole - // functions directory and will infer the name of the function to be - // `index`, not `my-func`. Instead, we need to pass the directory of - // the function. The exception is when the function is a file at the - // root of the functions directory (e.g. `functions/my-func.js`). In - // this case, we use `mainFile` as the function path of `zipFunction`. - const entryPath = functionDirectory === sourceDirectory ? mainFile : functionDirectory - const func = await memoizedBuild({ - cacheKey: `zisi-${entryPath}`, - command: () => zipFunction(entryPath, targetDirectory, zipOptions), - }) - - addFunctionToTree(func, fileTree) - }, - { concurrency: ZIP_CONCURRENCY }, - ) - - return - } - - const result = await memoizedBuild({ - cacheKey: 'zisi@all', - command: () => zipFunctions(sourceDirectory, targetDirectory, zipOptions), - }) - - result.forEach((func) => { - addFunctionToTree(func, fileTree) - }) -} - -// Bundles a set of functions based on the event type and the updated path. It -// returns an array with the paths of the functions that were rebundled as a -// result of the update event. -const bundleFunctions = async ({ - config, - eventType, - fileTree, - projectRoot, - sourceDirectory, - targetDirectory, - updatedPath, - zipCache, -}) => { - const zipOptions = { - archiveFormat: 'none', - basePath: projectRoot, - config, - } - - if (eventType === 'add') { - // We first check to see if the file being added is associated with any - // functions (e.g. restoring a file that has been previously deleted). - // If that's the case, we bundle just those functions. - const functionsWithPath = [...fileTree.entries()].filter(([, { inputs }]) => inputs.has(updatedPath)) - - if (functionsWithPath.length !== 0) { - await zipFunctionsAndUpdateTree({ - fileTree, - functions: functionsWithPath.map(([, func]) => func), - sourceDirectory, - targetDirectory, - zipCache, - zipOptions, - }) - - return functionsWithPath.map(([mainFile]) => mainFile) - } - - // We then check whether the newly-added file is itself a function. If so, - // we bundle it. - const functions = await getFunctions(sourceDirectory) - const matchingFunction = functions.find(({ mainFile }) => mainFile === updatedPath) - - if (matchingFunction !== undefined) { - await zipFunctionsAndUpdateTree({ - fileTree, - functions: [matchingFunction], - sourceDirectory, - targetDirectory, - zipCache, - zipOptions, - }) - - return [updatedPath] - } - - // At this point, the newly-added file is neither a function nor a file - // associated with a function, so we can discard the update. - return - } - - if (eventType === 'change' || eventType === 'unlink') { - // If the file matches a function's main file, we just need to operate on - // that one function. - if (fileTree.has(updatedPath)) { - const matchingFunction = fileTree.get(updatedPath) - - // We bundle the function if this is a `change` event, or delete it if - // the event is `unlink`. - if (eventType === 'change') { - await zipFunctionsAndUpdateTree({ - fileTree, - functions: [matchingFunction], - sourceDirectory, - targetDirectory, - zipCache, - zipOptions, - }) - } else { - const { path: functionPath } = matchingFunction - - fileTree.delete(updatedPath) - - await del(functionPath, { force: true }) - } - - return [updatedPath] - } - - // The update is in one of the supporting files. We bundle every function - // that uses it. - const functions = [...fileTree.entries()].filter(([, { inputs }]) => inputs.has(updatedPath)) - - await zipFunctionsAndUpdateTree({ - fileTree, - functions: functions.map(([, func]) => func), - sourceDirectory, - targetDirectory, - zipCache, - zipOptions, - }) - - return functions.map(([mainFile]) => mainFile) - } - - // Deleting the target directory so that we can start from a clean slate. - try { - await del(targetDirectory, { force: true }) - } catch (_) { - // no-op - } - - // Bundling all functions. - await zipFunctionsAndUpdateTree({ - fileTree, - sourceDirectory, - targetDirectory, - zipCache, - zipOptions, - }) -} - -const getFunctionByName = ({ cache, name }) => [...cache.values()].find((func) => func.name === name) - -const getTargetDirectory = async ({ errorExit }) => { - const targetDirectory = path.resolve(getPathInProject(['functions-serve'])) - - try { - await makeDir(targetDirectory) - } catch (error) { - errorExit(`${NETLIFYDEVERR} Could not create directory: ${targetDirectory}`) - } - - return targetDirectory -} - -module.exports = async function handler({ config, errorExit, functionsDirectory: sourceDirectory, projectRoot }) { - const functions = await getFunctions(sourceDirectory) - const hasTSFunction = functions.some(({ mainFile }) => path.extname(mainFile) === '.ts') - const functionsConfig = addFunctionsConfigDefaults( - normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }), - ) - const isUsingEsbuild = functionsConfig['*'].nodeBundler === 'esbuild_zisi' - - if (!hasTSFunction && !isUsingEsbuild) { - return false - } - - // Enable source map support. - sourceMapSupport.install() - - // Keeps track of which files are associated with each function. - const fileTree = new Map() - - // Used for memoizing calls to ZISI, such that we don't bundle the same - // function multiple times at the same time. - const zipCache = {} - const targetDirectory = await getTargetDirectory({ errorExit }) - - return { - build: (updatedPath, eventType) => - bundleFunctions({ - config: functionsConfig, - eventType, - fileTree, - projectRoot, - sourceDirectory, - targetDirectory, - updatedPath, - zipCache, - }), - builderName: 'zip-it-and-ship-it', - getFunctionByName: (name) => getFunctionByName({ cache: fileTree, name }), - src: sourceDirectory, - target: targetDirectory, - } -}