From e7ade1234b93f0914cc3b4ba0d218c78bca6dd3a Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Fri, 26 Jun 2020 15:41:43 +0500 Subject: [PATCH 1/8] alternativePathsFor: Dont include original URL --- src/commands/dev/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/dev/index.js b/src/commands/dev/index.js index 0e3bba6ae87..b661cd2519e 100644 --- a/src/commands/dev/index.js +++ b/src/commands/dev/index.js @@ -46,7 +46,7 @@ function addonUrl(addonUrls, req) { } async function getStatic(pathname, publicFolder) { - const alternatives = alternativePathsFor(pathname).map(p => path.resolve(publicFolder, p.substr(1))) + const alternatives = [pathname, ...alternativePathsFor(pathname)].map(p => path.resolve(publicFolder, p.substr(1))) for (const i in alternatives) { const p = alternatives[i] @@ -78,7 +78,7 @@ function render404(publicFolder) { const assetExtensionRegExp = /\.(html?|png|jpg|js|css|svg|gif|ico|woff|woff2)$/ function alternativePathsFor(url) { - const paths = [url] + const paths = [] if (url[url.length - 1] === '/') { const end = url.length - 1 if (url !== '/') { From 0f72a44d4fe193c37c5008af34c3f3953acb9921 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 15:43:39 +0500 Subject: [PATCH 2/8] Add util: createStreamPromise --- src/utils/create-stream-promise.js | 42 ++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/utils/create-stream-promise.js diff --git a/src/utils/create-stream-promise.js b/src/utils/create-stream-promise.js new file mode 100644 index 00000000000..47d4012d9e2 --- /dev/null +++ b/src/utils/create-stream-promise.js @@ -0,0 +1,42 @@ +function createStreamPromise(stream, timeoutSeconds, bytesLimit = 1024 * 1024 * 6) { + return new Promise(function(resolve, reject) { + let data = [] + let dataLength = 0 + + let timeoutId = null + if (timeoutSeconds != null && Number.isFinite(timeoutSeconds)) { + timeoutId = setTimeout(() => { + data = null + reject(new Error('Request timed out waiting for body')) + }, timeoutSeconds * 1000) + } + + stream.on('data', function(chunk) { + if (!Array.isArray(data)) { + // Stream harvesting closed + return + } + dataLength += chunk.length + if (dataLength > bytesLimit) { + data = null + reject(new Error('Stream body too big')) + } else { + data.push(chunk) + } + }) + + stream.on('error', function(error) { + data = null + reject(error) + clearTimeout(timeoutId) + }) + stream.on('end', function() { + clearTimeout(timeoutId) + if (data) { + resolve(Buffer.concat(data)) + } + }) + }) +} + +module.exports = {createStreamPromise} From 6fa1dd333ddee9e405cc804c19c831aafac58d5b Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 15:44:07 +0500 Subject: [PATCH 3/8] Replay request body on alternative URL's --- src/commands/dev/index.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/commands/dev/index.js b/src/commands/dev/index.js index b661cd2519e..80f86938ba8 100644 --- a/src/commands/dev/index.js +++ b/src/commands/dev/index.js @@ -29,6 +29,7 @@ const { createRewriter } = require('../../utils/rules-proxy') const { onChanges } = require('../../utils/rules-proxy') const { parseHeadersFile, objectForPath } = require('../../utils/headers') const { getEnvSettings } = require('../../utils/env') +const { createStreamPromise } = require('../../utils/create-stream-promise') const stat = util.promisify(fs.stat) @@ -118,6 +119,11 @@ function initializeProxy(port, distDir, projectDir) { }) proxy.on('error', err => console.error('error while proxying request:', err.message)) + proxy.on('proxyReq', (proxyReq, req) => { + if (req.originalBody) { + proxyReq.write(req.originalBody) + } + }) proxy.on('proxyRes', (proxyRes, req, res) => { if (proxyRes.statusCode === 404) { if (req.alternativePaths && req.alternativePaths.length > 0) { @@ -181,6 +187,8 @@ async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projec }) const server = http.createServer(async function(req, res) { + req.originalBody = ['GET', 'OPTIONS', 'HEAD'].includes(req.method) ? null : await createStreamPromise(req, 30) + if (isFunction(settings.functionsPort, req.url)) { return proxy.web(req, res, { target: functionsServer }) } From 5d24d9309408774f6cbee04c359eddc538793640 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 15:48:21 +0500 Subject: [PATCH 4/8] Add test for not found function --- tests/dev.test.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/dev.test.js b/tests/dev.test.js index 5576f7e92cf..b5c430f5292 100644 --- a/tests/dev.test.js +++ b/tests/dev.test.js @@ -134,6 +134,15 @@ test('functions rewrite echo with Form body', async t => { t.regex(response.body, new RegExp(formBoundary)) }) +test('functions: not found', async t => { + const response = await fetch(`http://${host}/api/none`, { + method: 'POST', + body: 'nothing', + }) + + t.is(response.status, 404) +}) + test('Netlify Forms support', async t => { const form = new FormData() form.append('some', 'thing') From 0a76d766afc5b8b3aa67f6ebc5e9e2d0deeda157 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 16:11:19 +0500 Subject: [PATCH 5/8] Serve Functions: Handle preread request body --- src/utils/serve-functions.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/utils/serve-functions.js b/src/utils/serve-functions.js index c0d8b94d723..07d477e4c0d 100644 --- a/src/utils/serve-functions.js +++ b/src/utils/serve-functions.js @@ -184,8 +184,15 @@ async function handleFormSubmission(req, res, proxy, siteInfo, functionsServer) const ct = contentType.parse(req) let fields = {} let files = {} + const requestClone = new Readable({ + read() { + this.push(req.originalBody) + this.push(null) + } + }) + requestClone.headers = req.headers if (ct.type.endsWith('/x-www-form-urlencoded')) { - const bodyData = await getRawBody(req, { + const bodyData = await getRawBody(requestClone, { length: req.headers['content-length'], limit: '10mb', encoding: ct.parameters.charset, @@ -195,7 +202,7 @@ async function handleFormSubmission(req, res, proxy, siteInfo, functionsServer) try { ;[fields, files] = await new Promise((resolve, reject) => { const form = new multiparty.Form({ encoding: ct.parameters.charset || 'utf8' }) - form.parse(req, (err, Fields, Files) => { + form.parse(requestClone, (err, Fields, Files) => { if (err) return reject(err) Files = Object.entries(Files).reduce( (prev, [name, values]) => ({ From 7c5b49fff236dc2db59d829e26f7f358064050d0 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 16:59:09 +0500 Subject: [PATCH 6/8] Remove a redundant param --- src/utils/serve-functions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/serve-functions.js b/src/utils/serve-functions.js index 07d477e4c0d..aab9622f8ed 100644 --- a/src/utils/serve-functions.js +++ b/src/utils/serve-functions.js @@ -292,7 +292,7 @@ async function serveFunctions(dir) { app.use( bodyParser.text({ limit: '6mb', - type: ['text/*', 'application/json', 'multipart/form-data'], + type: ['text/*', 'application/json'], }) ) app.use(bodyParser.raw({ limit: '6mb', type: '*/*' })) From 78969a32e59c7f4aed71864eddd1d635e885c417 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 16:59:34 +0500 Subject: [PATCH 7/8] Move form handling to Express --- src/commands/dev/index.js | 12 +-- src/utils/serve-functions.js | 198 +++++++++++++++++------------------ 2 files changed, 104 insertions(+), 106 deletions(-) diff --git a/src/commands/dev/index.js b/src/commands/dev/index.js index 95f778e9ad0..27ec77445a3 100644 --- a/src/commands/dev/index.js +++ b/src/commands/dev/index.js @@ -15,7 +15,7 @@ const proxyMiddleware = require('http-proxy-middleware') const cookie = require('cookie') const get = require('lodash.get') const isEmpty = require('lodash.isempty') -const { serveFunctions, handleFormSubmission } = require('../../utils/serve-functions') +const { serveFunctions } = require('../../utils/serve-functions') const { serverSettings } = require('../../utils/detect-server') const { detectFunctionsBuilder } = require('../../utils/detect-functions-builder') const Command = require('../../utils/command') @@ -163,7 +163,7 @@ function initializeProxy(port, distDir, projectDir) { return handlers } -async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projectDir, functionsDir, exit) { +async function startProxy(settings = {}, addonUrls, configPath, projectDir, functionsDir, exit) { try { await waitPort({ port: settings.frameworkPort, output: 'silent' }) } catch (err) { @@ -207,13 +207,12 @@ async function startProxy(settings, siteInfo = {}, addonUrls, configPath, projec functionsPort: settings.functionsPort, jwtRolePath: settings.jwtRolePath, framework: settings.framework, - siteInfo: siteInfo, } if (match) return serveRedirect(req, res, proxy, match, options) if (req.method === 'POST' && !isInternal(req.url)) { - return handleFormSubmission(req, res, proxy, siteInfo, functionsServer) + return proxy.web(req, res, { target: functionsServer }) } proxy.web(req, res, options) @@ -331,7 +330,7 @@ async function serveRedirect(req, res, proxy, match, options) { } if (req.method === 'POST' && !isInternal(req.url) && !isInternal(destURL)) { - return handleFormSubmission(req, res, proxy, options.siteInfo, options.functionsServer) + return proxy.web(req, res, { target: options.functionsServer }) } const destStaticFile = await getStatic(dest.pathname, options.publicFolder) @@ -486,7 +485,7 @@ class DevCommand extends Command { functionWatcher.on('unlink', functionBuilder.build) } - const functionsServer = await serveFunctions(settings.functions) + const functionsServer = await serveFunctions(settings.functions, this.netlify.cachedConfig.siteInfo) functionsServer.listen(settings.functionsPort, function(err) { if (err) { errorExit(`${NETLIFYDEVERR} Unable to start lambda server: ${err}`) @@ -499,7 +498,6 @@ class DevCommand extends Command { let { url } = await startProxy( settings, - this.netlify.cachedConfig.siteInfo, addonUrls, site.configPath, site.root, diff --git a/src/utils/serve-functions.js b/src/utils/serve-functions.js index aab9622f8ed..82245b5dd41 100644 --- a/src/utils/serve-functions.js +++ b/src/utils/serve-functions.js @@ -178,115 +178,114 @@ function createHandler(dir) { } } -async function handleFormSubmission(req, res, proxy, siteInfo, functionsServer) { - const originalUrl = new URL(req.url, 'http://localhost') - req.url = '/.netlify/functions/submission-created' + originalUrl.search - const ct = contentType.parse(req) - let fields = {} - let files = {} - const requestClone = new Readable({ - read() { - this.push(req.originalBody) - this.push(null) - } - }) - requestClone.headers = req.headers - if (ct.type.endsWith('/x-www-form-urlencoded')) { - const bodyData = await getRawBody(requestClone, { - length: req.headers['content-length'], - limit: '10mb', - encoding: ct.parameters.charset, +function createFormSubmissionHandler(siteInfo) { + return async function(req, res, next) { + if (req.url.startsWith('/.netlify/') || req.method !== 'POST') return next() + + const fakeRequest = new Readable({ + read(size) { + this.push(req.body) + this.push(null) + } }) - fields = querystring.parse(bodyData.toString()) - } else if (ct.type === 'multipart/form-data') { - try { - ;[fields, files] = await new Promise((resolve, reject) => { - const form = new multiparty.Form({ encoding: ct.parameters.charset || 'utf8' }) - form.parse(requestClone, (err, Fields, Files) => { - if (err) return reject(err) - Files = Object.entries(Files).reduce( - (prev, [name, values]) => ({ - ...prev, - [name]: values.map(v => ({ - filename: v['originalFilename'], - size: v['size'], - type: v['headers'] && v['headers']['content-type'], - url: v['path'], - })), - }), - {} - ) - return resolve([ - Object.entries(Fields).reduce( - (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }), - {} - ), - Object.entries(Files).reduce( - (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }), + fakeRequest.headers = req.headers + + const originalUrl = new URL(req.url, 'http://localhost') + req.url = '/.netlify/functions/submission-created' + originalUrl.search + + const ct = contentType.parse(req) + let fields = {} + let files = {} + if (ct.type.endsWith('/x-www-form-urlencoded')) { + const bodyData = await getRawBody(fakeRequest, { + length: req.headers['content-length'], + limit: '10mb', + encoding: ct.parameters.charset, + }) + fields = querystring.parse(bodyData.toString()) + } else if (ct.type === 'multipart/form-data') { + try { + ;[fields, files] = await new Promise((resolve, reject) => { + const form = new multiparty.Form({ encoding: ct.parameters.charset || 'utf8' }) + form.parse(fakeRequest, (err, Fields, Files) => { + if (err) return reject(err) + Files = Object.entries(Files).reduce( + (prev, [name, values]) => ({ + ...prev, + [name]: values.map(v => ({ + filename: v['originalFilename'], + size: v['size'], + type: v['headers'] && v['headers']['content-type'], + url: v['path'], + })), + }), {} - ), - ]) + ) + return resolve([ + Object.entries(Fields).reduce( + (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }), + {} + ), + Object.entries(Files).reduce( + (prev, [name, values]) => ({ ...prev, [name]: values.length > 1 ? values : values[0] }), + {} + ), + ]) + }) }) - }) - } catch (err) { - return console.error(err) + } catch (err) { + return console.error(err) + } + } else { + return console.error('Invalid Content-Type for Netlify Dev forms request') } - } else { - return console.error('Invalid Content-Type for Netlify Dev forms request') - } - const data = JSON.stringify({ - payload: { - company: - fields[Object.keys(fields).find(name => ['company', 'business', 'employer'].includes(name.toLowerCase()))], - last_name: - fields[Object.keys(fields).find(name => ['lastname', 'surname', 'byname'].includes(name.toLowerCase()))], - first_name: - fields[Object.keys(fields).find(name => ['firstname', 'givenname', 'forename'].includes(name.toLowerCase()))], - name: fields[Object.keys(fields).find(name => ['name', 'fullname'].includes(name.toLowerCase()))], - email: - fields[ - Object.keys(fields).find(name => ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase())) - ], - title: fields[Object.keys(fields).find(name => ['title', 'subject'].includes(name.toLowerCase()))], - data: { - ...fields, - ...files, - ip: req.connection.remoteAddress, - user_agent: req.headers['user-agent'], - referrer: req.headers['referer'], + const data = JSON.stringify({ + payload: { + company: + fields[Object.keys(fields).find(name => ['company', 'business', 'employer'].includes(name.toLowerCase()))], + last_name: + fields[Object.keys(fields).find(name => ['lastname', 'surname', 'byname'].includes(name.toLowerCase()))], + first_name: + fields[Object.keys(fields).find(name => ['firstname', 'givenname', 'forename'].includes(name.toLowerCase()))], + name: fields[Object.keys(fields).find(name => ['name', 'fullname'].includes(name.toLowerCase()))], + email: + fields[ + Object.keys(fields).find(name => ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase())) + ], + title: fields[Object.keys(fields).find(name => ['title', 'subject'].includes(name.toLowerCase()))], + data: { + ...fields, + ...files, + ip: req.connection.remoteAddress, + user_agent: req.headers['user-agent'], + referrer: req.headers['referer'], + }, + created_at: new Date().toISOString(), + human_fields: Object.entries({ + ...fields, + ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}), + }).reduce((prev, [key, val]) => ({ ...prev, [capitalize(key)]: val }), {}), + ordered_human_fields: Object.entries({ + ...fields, + ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}), + }).map(([key, val]) => ({ title: capitalize(key), name: key, value: val })), + site_url: siteInfo['ssl_url'], }, - created_at: new Date().toISOString(), - human_fields: Object.entries({ - ...fields, - ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}), - }).reduce((prev, [key, val]) => ({ ...prev, [capitalize(key)]: val }), {}), - ordered_human_fields: Object.entries({ - ...fields, - ...Object.entries(files).reduce((prev, [name, data]) => ({ ...prev, [name]: data['url'] }), {}), - }).map(([key, val]) => ({ title: capitalize(key), name: key, value: val })), - site_url: siteInfo['ssl_url'], - }, - site: siteInfo, - }) - const buff = new Readable({ - read(size) { - this.push(data) - this.push(null) - }, - }) - return proxy.web(req, res, { - target: functionsServer, - buffer: buff, - headers: { + site: siteInfo, + }) + req.body = data + req.headers = { ...req.headers, 'content-length': data.length, 'content-type': 'application/json', 'x-netlify-original-pathname': originalUrl.pathname, - }, - }) + } + + next() + } } -async function serveFunctions(dir) { +async function serveFunctions(dir, siteInfo = {}) { const app = express() app.use( @@ -296,6 +295,7 @@ async function serveFunctions(dir) { }) ) app.use(bodyParser.raw({ limit: '6mb', type: '*/*' })) + app.use(createFormSubmissionHandler(siteInfo)) app.use( expressLogging(console, { blacklist: ['/favicon.ico'], @@ -311,4 +311,4 @@ async function serveFunctions(dir) { return app } -module.exports = { serveFunctions, handleFormSubmission } +module.exports = { serveFunctions } From 23742d7512a14b87c11d23ed4049e4e10d8b3b93 Mon Sep 17 00:00:00 2001 From: Raees Iqbal Date: Sun, 28 Jun 2020 17:04:13 +0500 Subject: [PATCH 8/8] Formatting --- src/commands/dev/index.js | 9 +-------- src/utils/create-stream-promise.js | 2 +- src/utils/serve-functions.js | 8 +++++--- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/commands/dev/index.js b/src/commands/dev/index.js index 27ec77445a3..a9ac063f21a 100644 --- a/src/commands/dev/index.js +++ b/src/commands/dev/index.js @@ -496,14 +496,7 @@ class DevCommand extends Command { }) } - let { url } = await startProxy( - settings, - addonUrls, - site.configPath, - site.root, - settings.functions, - this.exit - ) + let { url } = await startProxy(settings, addonUrls, site.configPath, site.root, settings.functions, this.exit) if (!url) { throw new Error('Unable to start proxy server') } diff --git a/src/utils/create-stream-promise.js b/src/utils/create-stream-promise.js index 47d4012d9e2..e7e067a7379 100644 --- a/src/utils/create-stream-promise.js +++ b/src/utils/create-stream-promise.js @@ -39,4 +39,4 @@ function createStreamPromise(stream, timeoutSeconds, bytesLimit = 1024 * 1024 * }) } -module.exports = {createStreamPromise} +module.exports = { createStreamPromise } diff --git a/src/utils/serve-functions.js b/src/utils/serve-functions.js index 82245b5dd41..4f4f6351fbc 100644 --- a/src/utils/serve-functions.js +++ b/src/utils/serve-functions.js @@ -186,7 +186,7 @@ function createFormSubmissionHandler(siteInfo) { read(size) { this.push(req.body) this.push(null) - } + }, }) fakeRequest.headers = req.headers @@ -250,8 +250,10 @@ function createFormSubmissionHandler(siteInfo) { name: fields[Object.keys(fields).find(name => ['name', 'fullname'].includes(name.toLowerCase()))], email: fields[ - Object.keys(fields).find(name => ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase())) - ], + Object.keys(fields).find(name => + ['email', 'mail', 'from', 'twitter', 'sender'].includes(name.toLowerCase()) + ) + ], title: fields[Object.keys(fields).find(name => ['title', 'subject'].includes(name.toLowerCase()))], data: { ...fields,