Fix requests hanging when function not found

src/commands/dev/index.js

@@ -15,7 +15,7 @@ const proxyMiddleware = require('http-proxy-middleware')
 const cookie = require('cookie')
 const get = require('lodash.get')
 const isEmpty = require('lodash.isempty')
-const { serveFunctions, handleFormSubmission } = require('../../utils/serve-functions')
+const { serveFunctions } = require('../../utils/serve-functions')
 const { serverSettings } = require('../../utils/detect-server')
 const { detectFunctionsBuilder } = require('../../utils/detect-functions-builder')
 const Command = require('../../utils/command')
@@ -29,6 +29,7 @@ const { createRewriter } = require('../../utils/rules-proxy')
 const { onChanges } = require('../../utils/rules-proxy')
 const { parseHeadersFile, objectForPath } = require('../../utils/headers')
 const { getEnvSettings } = require('../../utils/env')
+const { createStreamPromise } = require('../../utils/create-stream-promise')
 
 const stat = util.promisify(fs.stat)
 
@@ -46,7 +47,7 @@ function addonUrl(addonUrls, req) {
 }
 
 async function getStatic(pathname, publicFolder) {
-  const alternatives = alternativePathsFor(pathname).map(p => path.resolve(publicFolder, p.substr(1)))
+  const alternatives = [pathname, ...alternativePathsFor(pathname)].map(p => path.resolve(publicFolder, p.substr(1)))
 
   for (const i in alternatives) {
     const p = alternatives[i]
@@ -78,7 +79,7 @@ function render404(publicFolder) {
 const assetExtensionRegExp = /\.(html?|png|jpg|js|css|svg|gif|ico|woff|woff2)$/
 
 function alternativePathsFor(url) {
-  const paths = [url]
+  const paths = []
   if (url[url.length - 1] === '/') {
     const end = url.length - 1
     if (url !== '/') {
@@ -118,6 +119,11 @@ function initializeProxy(port, distDir, projectDir) {
   })
 
   proxy.on('error', err => console.error('error while proxying request:', err.message))
+  proxy.on('proxyReq', (proxyReq, req) => {
+    if (req.originalBody) {
+      proxyReq.write(req.originalBody)
+    }
+  })
   proxy.on('proxyRes', (proxyRes, req, res) => {
     if (proxyRes.statusCode === 404) {
       if (req.alternativePaths && req.alternativePaths.length > 0) {
@@ -157,7 +163,7 @@ function initializeProxy(port, distDir, projectDir) {
   return handlers
 }
 
-async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projectDir, functionsDir, exit) {
+async function startProxy(settings = {}, addonUrls, configPath, projectDir, functionsDir, exit) {
   try {
     await waitPort({ port: settings.frameworkPort, output: 'silent' })
   } catch (err) {
@@ -181,6 +187,8 @@ async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projec
   })
 
   const server = http.createServer(async function(req, res) {
+    req.originalBody = ['GET', 'OPTIONS', 'HEAD'].includes(req.method) ? null : await createStreamPromise(req, 30)
+
     if (isFunction(settings.functionsPort, req.url)) {
       return proxy.web(req, res, { target: functionsServer })
     }
@@ -199,13 +207,12 @@ async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projec
         functionsPort: settings.functionsPort,
         jwtRolePath: settings.jwtRolePath,
         framework: settings.framework,
-        siteInfo: siteInfo,
       }
 
       if (match) return serveRedirect(req, res, proxy, match, options)
 
       if (req.method === 'POST' && !isInternal(req.url)) {
-        return handleFormSubmission(req, res, proxy, siteInfo, functionsServer)
+        return proxy.web(req, res, { target: functionsServer })
       }
 
       proxy.web(req, res, options)
@@ -323,7 +330,7 @@ async function serveRedirect(req, res, proxy, match, options) {
     }
 
     if (req.method === 'POST' && !isInternal(req.url) && !isInternal(destURL)) {
-      return handleFormSubmission(req, res, proxy, options.siteInfo, options.functionsServer)
+      return proxy.web(req, res, { target: options.functionsServer })
     }
 
     const destStaticFile = await getStatic(dest.pathname, options.publicFolder)
@@ -478,7 +485,7 @@ class DevCommand extends Command {
         functionWatcher.on('unlink', functionBuilder.build)
       }
 
-      const functionsServer = await serveFunctions(settings.functions)
+      const functionsServer = await serveFunctions(settings.functions, this.netlify.cachedConfig.siteInfo)
       functionsServer.listen(settings.functionsPort, function(err) {
         if (err) {
           errorExit(`${NETLIFYDEVERR} Unable to start lambda server: ${err}`)
@@ -489,15 +496,7 @@ class DevCommand extends Command {
       })
     }
 
-    let { url } = await startProxy(
-      settings,
-      this.netlify.cachedConfig.siteInfo,
-      addonUrls,
-      site.configPath,
-      site.root,
-      settings.functions,
-      this.exit
-    )
+    let { url } = await startProxy(settings, addonUrls, site.configPath, site.root, settings.functions, this.exit)
     if (!url) {
       throw new Error('Unable to start proxy server')
     }

src/utils/create-stream-promise.js

@@ -0,0 +1,42 @@
+function createStreamPromise(stream, timeoutSeconds, bytesLimit = 1024 * 1024 * 6) {
+  return new Promise(function(resolve, reject) {
+    let data = []
+    let dataLength = 0
+
+    let timeoutId = null
+    if (timeoutSeconds != null && Number.isFinite(timeoutSeconds)) {
+      timeoutId = setTimeout(() => {
+        data = null
+        reject(new Error('Request timed out waiting for body'))
+      }, timeoutSeconds * 1000)
+    }
+
+    stream.on('data', function(chunk) {
+      if (!Array.isArray(data)) {
+        // Stream harvesting closed
+        return
+      }
+      dataLength += chunk.length
+      if (dataLength > bytesLimit) {
+        data = null
+        reject(new Error('Stream body too big'))
+      } else {
+        data.push(chunk)
+      }
+    })
+
+    stream.on('error', function(error) {
+      data = null
+      reject(error)
+      clearTimeout(timeoutId)
+    })
+    stream.on('end', function() {
+      clearTimeout(timeoutId)
+      if (data) {
+        resolve(Buffer.concat(data))
+      }
+    })
+  })
+}
+
+module.exports = { createStreamPromise }

src/utils/serve-functions.js

@@ -178,117 +178,126 @@ function createHandler(dir) {
   }
 }
 
-async function handleFormSubmission(req, res, proxy, siteInfo, functionsServer) {
-  const originalUrl = new URL(req.url, 'http://localhost')
-  req.url = '/.netlify/functions/submission-created' + originalUrl.search
-  const ct = contentType.parse(req)
-  let fields = {}
-  let files = {}
-  if (ct.type.endsWith('/x-www-form-urlencoded')) {
-    const bodyData = await getRawBody(req, {
-      length: req.headers['content-length'],
-      limit: '10mb',
-      encoding: ct.parameters.charset,
+function createFormSubmissionHandler(siteInfo) {
+  return async function(req, res, next) {
+    if (req.url.startsWith('/.netlify/') || req.method !== 'POST') return next()
+
+    const fakeRequest = new Readable({
+      read(size) {
+        this.push(req.body)
+        this.push(null)
+      },
     })
-    fields = querystring.parse(bodyData.toString())
-  } else if (ct.type === 'multipart/form-data') {
-    try {
-      ;[fields, files] = await new Promise((resolve, reject) => {
-        const form = new multiparty.Form({ encoding: ct.parameters.charset || 'utf8' })
-        form.parse(req, (err, Fields, Files) => {
-          if (err) return reject(err)
-          Files = Object.entries(Files).reduce(
-            (prev, [name, values]) => ({
-              ...prev,
-              [name]: values.map(v => ({
-                filename: v['originalFilename'],
-                size: v['size'],
-                type: v['headers'] && v['headers']['content-type'],
-                url: v['path'],
-              })),
-            }),
-            {}
-          )
-          return resolve([
-            Object.entries(Fields).reduce(
-              (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }),
-              {}
-            ),
-            Object.entries(Files).reduce(
-              (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }),
+    fakeRequest.headers = req.headers
+
+    const originalUrl = new URL(req.url, 'http://localhost')
+    req.url = '/.netlify/functions/submission-created' + originalUrl.search
+
+    const ct = contentType.parse(req)
+    let fields = {}
+    let files = {}
+    if (ct.type.endsWith('/x-www-form-urlencoded')) {
+      const bodyData = await getRawBody(fakeRequest, {
+        length: req.headers['content-length'],
+        limit: '10mb',
+        encoding: ct.parameters.charset,
+      })
+      fields = querystring.parse(bodyData.toString())
+    } else if (ct.type === 'multipart/form-data') {
+      try {
+        ;[fields, files] = await new Promise((resolve, reject) => {
+          const form = new multiparty.Form({ encoding: ct.parameters.charset || 'utf8' })
+          form.parse(fakeRequest, (err, Fields, Files) => {
+            if (err) return reject(err)
+            Files = Object.entries(Files).reduce(
+              (prev, [name, values]) => ({
+                ...prev,
+                [name]: values.map(v => ({
+                  filename: v['originalFilename'],
+                  size: v['size'],
+                  type: v['headers'] && v['headers']['content-type'],
+                  url: v['path'],
+                })),
+              }),
               {}
-            ),
-          ])
+            )
+            return resolve([
+              Object.entries(Fields).reduce(
+                (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }),
+                {}
+              ),
+              Object.entries(Files).reduce(
+                (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }),
+                {}
+              ),
+            ])
+          })
         })
-      })
-    } catch (err) {
-      return console.error(err)
+      } catch (err) {
+        return console.error(err)
+      }
+    } else {
+      return console.error('Invalid Content-Type for Netlify Dev forms request')
     }
-  } else {
-    return console.error('Invalid Content-Type for Netlify Dev forms request')
-  }
-  const data = JSON.stringify({
-    payload: {
-      company:
-        fields[Object.keys(fields).find(name => ['company', 'business', 'employer'].includes(name.toLowerCase()))],
-      last_name:
-        fields[Object.keys(fields).find(name => ['lastname', 'surname', 'byname'].includes(name.toLowerCase()))],
-      first_name:
-        fields[Object.keys(fields).find(name => ['firstname', 'givenname', 'forename'].includes(name.toLowerCase()))],
-      name: fields[Object.keys(fields).find(name => ['name', 'fullname'].includes(name.toLowerCase()))],
-      email:
-        fields[
-          Object.keys(fields).find(name => ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase()))
-        ],
-      title: fields[Object.keys(fields).find(name => ['title', 'subject'].includes(name.toLowerCase()))],
-      data: {
-        ...fields,
-        ...files,
-        ip: req.connection.remoteAddress,
-        user_agent: req.headers['user-agent'],
-        referrer: req.headers['referer'],
+    const data = JSON.stringify({
+      payload: {
+        company:
+          fields[Object.keys(fields).find(name => ['company', 'business', 'employer'].includes(name.toLowerCase()))],
+        last_name:
+          fields[Object.keys(fields).find(name => ['lastname', 'surname', 'byname'].includes(name.toLowerCase()))],
+        first_name:
+          fields[Object.keys(fields).find(name => ['firstname', 'givenname', 'forename'].includes(name.toLowerCase()))],
+        name: fields[Object.keys(fields).find(name => ['name', 'fullname'].includes(name.toLowerCase()))],
+        email:
+          fields[
+            Object.keys(fields).find(name =>
+              ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase())
+            )
+          ],
+        title: fields[Object.keys(fields).find(name => ['title', 'subject'].includes(name.toLowerCase()))],
+        data: {
+          ...fields,
+          ...files,
+          ip: req.connection.remoteAddress,
+          user_agent: req.headers['user-agent'],
+          referrer: req.headers['referer'],
+        },
+        created_at: new Date().toISOString(),
+        human_fields: Object.entries({
+          ...fields,
+          ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}),
+        }).reduce((prev, [key, val]) => ({ ...prev, [capitalize(key)]: val }), {}),
+        ordered_human_fields: Object.entries({
+          ...fields,
+          ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}),
+        }).map(([key, val]) => ({ title: capitalize(key), name: key, value: val })),
+        site_url: siteInfo['ssl_url'],
       },
-      created_at: new Date().toISOString(),
-      human_fields: Object.entries({
-        ...fields,
-        ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}),
-      }).reduce((prev, [key, val]) => ({ ...prev, [capitalize(key)]: val }), {}),
-      ordered_human_fields: Object.entries({
-        ...fields,
-        ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}),
-      }).map(([key, val]) => ({ title: capitalize(key), name: key, value: val })),
-      site_url: siteInfo['ssl_url'],
-    },
-    site: siteInfo,
-  })
-  const buff = new Readable({
-    read(size) {
-      this.push(data)
-      this.push(null)
-    },
-  })
-  return proxy.web(req, res, {
-    target: functionsServer,
-    buffer: buff,
-    headers: {
+      site: siteInfo,
+    })
+    req.body = data
+    req.headers = {
       ...req.headers,
       'content-length': data.length,
       'content-type': 'application/json',
       'x-netlify-original-pathname': originalUrl.pathname,
-    },
-  })
+    }
+
+    next()
+  }
 }
 
-async function serveFunctions(dir) {
+async function serveFunctions(dir, siteInfo = {}) {
   const app = express()
 
   app.use(
     bodyParser.text({
       limit: '6mb',
-      type: ['text/*', 'application/json', 'multipart/form-data'],
+      type: ['text/*', 'application/json'],
     })
   )
   app.use(bodyParser.raw({ limit: '6mb', type: '*/*' }))
+  app.use(createFormSubmissionHandler(siteInfo))
   app.use(
     expressLogging(console, {
       blacklist: ['/favicon.ico'],
@@ -304,4 +313,4 @@ async function serveFunctions(dir) {
   return app
 }
 
-module.exports = { serveFunctions, handleFormSubmission }
+module.exports = { serveFunctions }

tests/dev.test.js

@@ -134,6 +134,15 @@ test('functions rewrite echo with Form body', async t => {
   t.regex(response.body, new RegExp(formBoundary))
 })
 
+test('functions: not found', async t => {
+  const response = await fetch(`http://${host}/api/none`, {
+    method: 'POST',
+    body: 'nothing',
+  })
+
+  t.is(response.status, 404)
+})
+
 test('Netlify Forms support', async t => {
   const form = new FormData()
   form.append('some', 'thing')