feat: add support for manifest file (#3091)

* feat: add support for functions manifest file

* chore: add tests

* chore: update docs

* refactor: update skip flag and user messages

* chore: update README

* chore: update test

* chore: add test

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

docs/commands/deploy.md

@@ -98,6 +98,7 @@ netlify deploy
 - `timeout` (*string*) - Timeout to wait for deployment to finish
 - `trigger` (*boolean*) - Trigger a new build of your site on Netlify without uploading local files
 - `build` (*boolean*) - Run build command before deploying
+- `skip-functions-cache` (*boolean*) - Ignore any functions created as part of a previous `build` or `deploy` commands, forcing them to be bundled again as part of the deployment
 - `debug` (*boolean*) - Print debugging information
 - `httpProxy` (*string*) - Proxy server address to route requests through.
 - `httpProxyCertificateFilename` (*string*) - Certificate file to use when connecting using a proxy server

src/commands/deploy.js

@@ -19,7 +19,7 @@ const Command = require('../utils/command')
 const { log, logJson, getToken } = require('../utils/command-helpers')
 const { deploySite } = require('../utils/deploy/deploy-site')
 const { deployEdgeHandlers } = require('../utils/edge-handlers')
-const { getInternalFunctionsDir } = require('../utils/functions')
+const { getFunctionsManifestPath, getInternalFunctionsDir } = require('../utils/functions')
 const { NETLIFYDEV, NETLIFYDEVLOG, NETLIFYDEVERR } = require('../utils/logo')
 const openBrowser = require('../utils/open-browser')
 
@@ -271,6 +271,8 @@ const runDeploy = async ({
     // functions from the rightmost directories. In this case, we want user
     // functions to take precedence over internal functions.
     const functionDirectories = [internalFunctionsFolder, functionsFolder].filter(Boolean)
+    const skipFunctionsCache = flags['skip-functions-cache'] === true
+    const manifestPath = skipFunctionsCache ? null : await getFunctionsManifestPath({ base: site.root })
 
     results = await deploySite(api, siteId, deployFolder, {
       configPath,
@@ -284,6 +286,8 @@ const runDeploy = async ({
       filter: getDeployFilesFilter({ site, deployFolder }),
       warn,
       rootDir: site.root,
+      manifestPath,
+      skipFunctionsCache,
     })
   } catch (error_) {
     if (deployId) {
@@ -633,6 +637,11 @@ DeployCommand.flags = {
   build: flagsLib.boolean({
     description: 'Run build command before deploying',
   }),
+  'skip-functions-cache': flagsLib.boolean({
+    description:
+      'Ignore any functions created as part of a previous `build` or `deploy` commands, forcing them to be bundled again as part of the deployment',
+    default: false,
+  }),
   ...DeployCommand.flags,
 }
 

src/lib/build.js

@@ -20,6 +20,7 @@ const getBuildOptions = ({
   buffer: json || silent,
   offline,
   featureFlags: {
+    functionsBundlingManifest: true,
     zisiEsbuildDynamicImports: true,
   },
 })

src/utils/deploy/deploy-site.js

@@ -32,9 +32,11 @@ const deploySite = async (
     fnDir = [],
     functionsConfig,
     hashAlgorithm,
+    manifestPath,
     maxRetry = DEFAULT_MAX_RETRY,
     // API calls this the 'title'
     message: title,
+    skipFunctionsCache,
     statusCb = () => {
       /* default to noop */
     },
@@ -52,7 +54,17 @@ const deploySite = async (
 
   const [{ files, filesShaMap }, { functions, functionsWithNativeModules, fnShaMap }] = await Promise.all([
     hashFiles(dir, configPath, { concurrentHash, hashAlgorithm, assetType, statusCb, filter }),
-    hashFns(fnDir, { functionsConfig, tmpDir, concurrentHash, hashAlgorithm, statusCb, assetType, rootDir }),
+    hashFns(fnDir, {
+      functionsConfig,
+      tmpDir,
+      concurrentHash,
+      hashAlgorithm,
+      statusCb,
+      assetType,
+      rootDir,
+      manifestPath,
+      skipFunctionsCache,
+    }),
   ])
 
   const filesCount = Object.keys(files).length

src/utils/deploy/hash-fns.js

@@ -7,9 +7,76 @@ const pump = promisify(require('pump'))
 
 const { hasherCtor, manifestCollectorCtor } = require('./hasher-segments')
 
+// Maximum age of functions manifest (2 minutes).
+const MANIFEST_FILE_TTL = 12e4
+
+const getFunctionZips = async ({
+  directories,
+  functionsConfig,
+  manifestPath,
+  rootDir,
+  skipFunctionsCache,
+  statusCb,
+  tmpDir,
+}) => {
+  statusCb({
+    type: 'functions-manifest',
+    msg: 'Looking for a functions cache...',
+    phase: 'start',
+  })
+
+  if (manifestPath) {
+    try {
+      // eslint-disable-next-line import/no-dynamic-require, node/global-require
+      const { functions, timestamp } = require(manifestPath)
+      const manifestAge = Date.now() - timestamp
+
+      if (manifestAge > MANIFEST_FILE_TTL) {
+        throw new Error('Manifest expired')
+      }
+
+      statusCb({
+        type: 'functions-manifest',
+        msg: 'Deploying functions from cache (use --skip-functions-cache to override)',
+        phase: 'stop',
+      })
+
+      return functions
+    } catch (error) {
+      statusCb({
+        type: 'functions-manifest',
+        msg: 'Ignored invalid or expired functions cache',
+        phase: 'stop',
+      })
+    }
+  } else {
+    const msg = skipFunctionsCache
+      ? 'Ignoring functions cache (use without --skip-functions-cache to change)'
+      : 'No cached functions were found'
+
+    statusCb({
+      type: 'functions-manifest',
+      msg,
+      phase: 'stop',
+    })
+  }
+
+  return await zipIt.zipFunctions(directories, tmpDir, { basePath: rootDir, config: functionsConfig })
+}
+
 const hashFns = async (
   directories,
-  { tmpDir, concurrentHash, functionsConfig, hashAlgorithm = 'sha256', assetType = 'function', statusCb, rootDir },
+  {
+    tmpDir,
+    concurrentHash,
+    functionsConfig,
+    hashAlgorithm = 'sha256',
+    assetType = 'function',
+    skipFunctionsCache,
+    statusCb,
+    rootDir,
+    manifestPath,
+  },
 ) => {
   // Early out if no functions directories are configured.
   if (directories.length === 0) {
@@ -20,7 +87,15 @@ const hashFns = async (
     throw new Error('Missing tmpDir directory for zipping files')
   }
 
-  const functionZips = await zipIt.zipFunctions(directories, tmpDir, { basePath: rootDir, config: functionsConfig })
+  const functionZips = await getFunctionZips({
+    directories,
+    functionsConfig,
+    manifestPath,
+    rootDir,
+    skipFunctionsCache,
+    statusCb,
+    tmpDir,
+  })
   const fileObjs = functionZips.map(({ path: functionPath, runtime }) => ({
     filepath: functionPath,
     root: tmpDir,

src/utils/functions/index.js

@@ -1,6 +1,6 @@
 const { resolve } = require('path')
 
-const { isDirectoryAsync } = require('../../lib/fs')
+const { isDirectoryAsync, isFileAsync } = require('../../lib/fs')
 const { getPathInProject } = require('../../lib/settings')
 
 const getFunctionsDir = ({ flags, config }, defaultValue) =>
@@ -10,11 +10,18 @@ const getFunctionsDir = ({ flags, config }, defaultValue) =>
   (config.dev && config.dev.Functions) ||
   defaultValue
 
+const getFunctionsManifestPath = async ({ base }) => {
+  const path = resolve(base, getPathInProject(['functions', 'manifest.json']))
+  const isFile = await isFileAsync(path)
+
+  return isFile ? path : null
+}
+
 const getInternalFunctionsDir = async ({ base }) => {
   const path = resolve(base, getPathInProject(['functions-internal']))
   const isDirectory = await isDirectoryAsync(path)
 
   return isDirectory ? path : null
 }
 
-module.exports = { getFunctionsDir, getInternalFunctionsDir }
+module.exports = { getFunctionsDir, getInternalFunctionsDir, getFunctionsManifestPath }