From c4151467d78c00464944571a36f1f95a9b088050 Mon Sep 17 00:00:00 2001 From: "U-DESKTOP-07IKGBD\\Szymon Marczak" Date: Thu, 2 Aug 2018 17:40:38 +0200 Subject: [PATCH 01/10] Increase normalize-arguments.js coverage --- source/normalize-arguments.js | 6 +++--- test/retry.js | 40 +++++++++++++++++++++++++++++++++++ test/unix-socket.js | 4 ++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/source/normalize-arguments.js b/source/normalize-arguments.js index d22436649..96d578139 100644 --- a/source/normalize-arguments.js +++ b/source/normalize-arguments.js @@ -165,10 +165,10 @@ module.exports = (url, options, defaults) => { if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) { let after = Number(error.headers['retry-after']); - if (is.number(after)) { - after *= 1000; + if (is.nan(after)) { + after = Math.max(Date.parse(error.headers['retry-after']) - Date.now(), 0) || 0; } else { - after = Math.max(Date.parse(error.headers['retry-after']) - Date.now(), 0); + after *= 1000; } if (after > options.gotRetry.maxRetryAfter) { diff --git a/test/retry.js b/test/retry.js index 72807d2ec..4b99997b6 100644 --- a/test/retry.js +++ b/test/retry.js @@ -8,6 +8,7 @@ let trys = 0; let knocks = 0; let fifth = 0; let lastTried413access = Date.now(); +let lastTried413TimestampAccess; const retryAfterOn413 = 2; const socketTimeout = 200; @@ -54,6 +55,16 @@ test.before('setup', async () => { response.end(); }); + s.on('/413withTimestamp', (request, response) => { + const date = (new Date(Date.now() + (retryAfterOn413 * 1000))).toUTCString(); + + response.writeHead(413, { + 'Retry-After': date + }); + response.end(lastTried413TimestampAccess); + lastTried413TimestampAccess = date; + }); + s.on('/413withoutRetryAfter', (request, response) => { response.statusCode = 413; response.end(); @@ -149,6 +160,15 @@ test('respect 413 Retry-After', async t => { t.true(Number(body) >= retryAfterOn413 * 1000); }); +test('respect 413 Retry-After with RFC-1123 timestamp', async t => { + const {statusCode, body} = await got(`${s.url}/413withTimestamp`, { + throwHttpErrors: false, + retry: 1 + }); + t.is(statusCode, 413); + t.true(Date.now() >= Date.parse(body)); +}); + test('doesn\'t retry on 413 with empty statusCodes and methods', async t => { const {statusCode, retryCount} = await got(`${s.url}/413`, { throwHttpErrors: false, @@ -207,3 +227,23 @@ test('doesn\'t retry if Retry-After header is greater than maxRetryAfter', async }); t.is(retryCount, 0); }); + +test('doesn\'t retry when set to false', async t => { + const {statusCode, retryCount} = await got(`${s.url}/413`, { + throwHttpErrors: false, + retry: false + }); + t.is(statusCode, 413); + t.is(retryCount, 0); +}); + +test('works when defaults.options.retry is not an object', async t => { + const instance = got.extend({ + retry: 2 + }); + + const {retryCount} = await instance(`${s.url}/413`, { + throwHttpErrors: false + }); + t.is(retryCount, 0); +}); diff --git a/test/unix-socket.js b/test/unix-socket.js index 769b1b1c2..b3bba76cb 100644 --- a/test/unix-socket.js +++ b/test/unix-socket.js @@ -41,4 +41,8 @@ if (process.platform !== 'win32') { const url = format('unix:%s:%s', socketPath, '/foo:bar'); t.is((await got(url)).body, 'ok'); }); + + test('throws on invalid URL', async t => { + await t.throws(got('unix:')); + }); } From 7148bef872be7e0b5e0cbd4d037469120a39eabb Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 17:45:46 +0200 Subject: [PATCH 02/10] Update normalize-arguments.js --- source/normalize-arguments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/normalize-arguments.js b/source/normalize-arguments.js index 96d578139..1eaeeb9bf 100644 --- a/source/normalize-arguments.js +++ b/source/normalize-arguments.js @@ -166,7 +166,7 @@ module.exports = (url, options, defaults) => { if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) { let after = Number(error.headers['retry-after']); if (is.nan(after)) { - after = Math.max(Date.parse(error.headers['retry-after']) - Date.now(), 0) || 0; + after = Math.max(Date.parse(error.headers['retry-after']) - Date.now(), 0); } else { after *= 1000; } From 94fac092b3c8635440dee3b4b63412a3746432c4 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 17:51:15 +0200 Subject: [PATCH 03/10] Update normalize-arguments.js --- source/normalize-arguments.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/normalize-arguments.js b/source/normalize-arguments.js index 1eaeeb9bf..f96ab554f 100644 --- a/source/normalize-arguments.js +++ b/source/normalize-arguments.js @@ -166,7 +166,7 @@ module.exports = (url, options, defaults) => { if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) { let after = Number(error.headers['retry-after']); if (is.nan(after)) { - after = Math.max(Date.parse(error.headers['retry-after']) - Date.now(), 0); + after = Date.parse(error.headers['retry-after']) - Date.now(); } else { after *= 1000; } From c77c01128f0037489d1de74fe224e0b3602c7327 Mon Sep 17 00:00:00 2001 From: "U-DESKTOP-07IKGBD\\Szymon Marczak" Date: Thu, 2 Aug 2018 18:00:51 +0200 Subject: [PATCH 04/10] Increase errors.js coverage --- test/error.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/error.js b/test/error.js index f9326e9b2..b1b8239a1 100644 --- a/test/error.js +++ b/test/error.js @@ -27,6 +27,11 @@ test.before('setup', async () => { response.end('body'); }); + s.on('/no-status-message', (request, response) => { + response.writeHead(400, ''); + response.end('body'); + }); + s.on('/body', async (request, response) => { const body = await getStream(request); response.end(body); @@ -102,6 +107,12 @@ test('custom status message', async t => { t.is(error.statusMessage, 'Something Exploded'); }); +test('no status message is overriden by the default one', async t => { + const error = await t.throws(got(`${s.url}/no-status-message`)); + t.is(error.statusCode, 400); + t.is(error.statusMessage, http.STATUS_CODES[400]); +}); + test.serial('http.request error', async t => { const stub = sinon.stub(http, 'request').callsFake(() => { throw new TypeError('The header content contains invalid characters'); From ec09eef2e75a57565a6eb1037003f7da694df466 Mon Sep 17 00:00:00 2001 From: "U-DESKTOP-07IKGBD\\Szymon Marczak" Date: Thu, 2 Aug 2018 18:30:34 +0200 Subject: [PATCH 05/10] Increase coverage for as-stream.js --- test/stream.js | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/test/stream.js b/test/stream.js index 5e1defaf6..dad77feb4 100644 --- a/test/stream.js +++ b/test/stream.js @@ -2,6 +2,7 @@ import test from 'ava'; import toReadableStream from 'to-readable-stream'; import getStream from 'get-stream'; import pEvent from 'p-event'; +import delay from 'delay'; import is from '@sindresorhus/is'; import got from '../source'; import {createServer} from './helpers/server'; @@ -13,9 +14,10 @@ test.before('setup', async () => { s.on('/', (request, response) => { response.writeHead(200, { - unicorn: 'rainbow' + unicorn: 'rainbow', + 'content-encoding': 'gzip' }); - response.end('ok'); + response.end(Buffer.from('H4sIAAAAAAAA/8vPBgBH3dx5AgAAAA==', 'base64')); // 'ok' }); s.on('/post', (request, response) => { @@ -129,6 +131,38 @@ test('proxying headers works', async t => { const {headers} = await got(server.url); t.is(headers.unicorn, 'rainbow'); + t.is(headers['content-encoding'], undefined); await server.close(); }); + +test('skips proxying headers after server has sent them already', async t => { + const server = await createServer(); + + server.on('/', (request, response) => { + response.writeHead(200); + got.stream(s.url).pipe(response); + }); + + await server.listen(server.port); + + const {headers} = await got(server.url); + t.is(headers.unicorn, undefined); + + await server.close(); +}); + +test('throws when trying to proxy through a closed stream', async t => { + const server = await createServer(); + + server.on('/', async (request, response) => { + const stream = got.stream(s.url); + await delay(1000); + t.throws(() => stream.pipe(response)); + response.end(); + }); + + await server.listen(server.port); + await got(server.url); + await server.close(); +}); From ed7489e792d4c3efdb6cdd233b6f9e1e464fb564 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 18:39:01 +0200 Subject: [PATCH 06/10] Update request-as-event-emitter.js --- source/request-as-event-emitter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/request-as-event-emitter.js b/source/request-as-event-emitter.js index b0c32996d..37e47bc8f 100644 --- a/source/request-as-event-emitter.js +++ b/source/request-as-event-emitter.js @@ -15,7 +15,7 @@ const {GotError, CacheError, UnsupportedProtocolError, MaxRedirectsError, Reques const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]); const allMethodRedirectCodes = new Set([300, 303, 307, 308]); -module.exports = (options = {}) => { +module.exports = options => { const emitter = new EventEmitter(); const requestUrl = options.href || (new URLGlobal(options.path, urlLib.format(options))).toString(); const redirects = []; From a86e47e3d6eb3b238b9f17f6a29db30ce036c86d Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 18:48:04 +0200 Subject: [PATCH 07/10] Remove not needed try/catch --- source/request-as-event-emitter.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/source/request-as-event-emitter.js b/source/request-as-event-emitter.js index 37e47bc8f..0a2011c44 100644 --- a/source/request-as-event-emitter.js +++ b/source/request-as-event-emitter.js @@ -142,13 +142,7 @@ module.exports = options => { if (backoff) { retryCount++; - setTimeout(options => { - try { - get(options); - } catch (error2) { - emitter.emit('error', error2); - } - }, backoff, options); + setTimeout(get, backoff, options); cb(true); return; } From 93f44081425ac59174af4f836f91ff2d7e7cb1bf Mon Sep 17 00:00:00 2001 From: "U-DESKTOP-07IKGBD\\Szymon Marczak" Date: Thu, 2 Aug 2018 21:42:34 +0200 Subject: [PATCH 08/10] Catch the retry function --- source/request-as-event-emitter.js | 8 +++++++- test/retry.js | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/source/request-as-event-emitter.js b/source/request-as-event-emitter.js index 0a2011c44..ace016dd7 100644 --- a/source/request-as-event-emitter.js +++ b/source/request-as-event-emitter.js @@ -138,7 +138,13 @@ module.exports = options => { }; emitter.on('retry', (error, cb) => { - const backoff = options.gotRetry.retries(++retryTries, error); + let backoff; + try { + backoff = options.gotRetry.retries(++retryTries, error); + } catch (error) { + emitter.emit('error', error); + return; + } if (backoff) { retryCount++; diff --git a/test/retry.js b/test/retry.js index 4b99997b6..a3d26d2e5 100644 --- a/test/retry.js +++ b/test/retry.js @@ -213,8 +213,10 @@ test('retries on 503 without Retry-After header', async t => { test('doesn\'t retry on streams', async t => { const stream = got.stream(s.url, { timeout: 1, - retries: () => { - t.fail('Retries on streams'); + retry: { + retries: () => { + t.fail('Retries on streams'); + } } }); await t.throws(pEvent(stream, 'response')); @@ -247,3 +249,13 @@ test('works when defaults.options.retry is not an object', async t => { }); t.is(retryCount, 0); }); + +test('retry function can throw', async t => { + await t.throws(got(`${s.url}/413`, { + retry: { + retries: () => { + throw new Error('Simple error'); + } + } + }), 'Simple error'); +}); From 59b6f165811365c754fb9fe6e613a185ea3f328d Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 22:01:18 +0200 Subject: [PATCH 09/10] Update retry.js --- test/retry.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/retry.js b/test/retry.js index a3d26d2e5..511e04c64 100644 --- a/test/retry.js +++ b/test/retry.js @@ -251,11 +251,12 @@ test('works when defaults.options.retry is not an object', async t => { }); test('retry function can throw', async t => { + const error = 'Simple error'; await t.throws(got(`${s.url}/413`, { retry: { retries: () => { - throw new Error('Simple error'); + throw new Error(error); } } - }), 'Simple error'); + }), error); }); From 3609221f966cd36aaa5dc4c60ef8e14a83a4ff68 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 2 Aug 2018 22:24:24 +0200 Subject: [PATCH 10/10] Update timeout.js --- test/timeout.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/timeout.js b/test/timeout.js index 667556bdc..67ba3ae1d 100644 --- a/test/timeout.js +++ b/test/timeout.js @@ -24,8 +24,8 @@ const slowDataStream = () => { return slowStream; }; -const requestDelay = 250; -const requestTimeout = requestDelay - 10; +const requestDelay = 500; +const requestTimeout = requestDelay - 20; const errorMatcher = { instanceOf: got.TimeoutError,