Merge branch 'master' into got-forward

readme.md

@@ -258,6 +258,24 @@ Determines if a `got.HTTPError` is thrown for error responses (non-2xx status co
 
 If this is disabled, requests that encounter an error status code will be resolved with the `response` instead of throwing. This may be useful if you are checking for resource availability and are expecting error responses.
 
+###### hooks
+
+Type: `Object<string, Array<Function>>`<br>
+Default: `{ beforeRequest: [] }`
+
+Hooks allow modifications during the request lifecycle. Hook functions may be async and are run serially.
+
+###### hooks.beforeRequest
+
+Type: `Array<Function>`<br>
+Default: `[]`
+
+Called with the normalized request options. Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) and [`got.create()`](advanced-creation.md) when you want to create an API client that uses HMAC-signing.
+
+See the [AWS section](#aws) for an example.
+
+**Note**: Modifying the `body` is not recommended because the `content-length` header has already been computed and assigned.
+
 #### Streams
 
 **Note**: Progress events, redirect events and request/response events can also be used with promises.
@@ -626,47 +644,35 @@ got('http://unix:/var/run/docker.sock:/containers/json');
 got('unix:/var/run/docker.sock:/containers/json');
 ```
 
+
 ## AWS
 
-Requests to AWS services need to have their headers signed. This can be accomplished by using the [`aws4`](https://www.npmjs.com/package/aws4) package. This is an example for querying an ["Elasticsearch Service"](https://aws.amazon.com/elasticsearch-service/) host with a signed request.
+Requests to AWS services need to have their headers signed. This can be accomplished by using the [`aws4`](https://www.npmjs.com/package/aws4) package. This is an example for querying an ["API Gateway"](https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/) with a signed request.
 
 ```js
-const url = require('url');
 const AWS = require('aws-sdk');
 const aws4 = require('aws4');
 const got = require('got');
-const config = require('./config');
-
-// Reads keys from the environment or `~/.aws/credentials`. Could be a plain object.
-const awsConfig = new AWS.Config({ region: config.region });
 
-function request(url, options) {
-	const awsOpts = {
-		region: awsConfig.region,
-		headers: {
-			accept: 'application/json',
-			'content-type': 'application/json'
-		},
-		method: 'GET',
-		json: true
-	};
-
-	// We need to parse the URL before passing it to `got` so `aws4` can sign the request
-	options = {
-		...url.parse(url),
-		...awsOpts,
-		...options
-	};
-
-	aws4.sign(options, awsConfig.credentials);
-
-	return got(options);
-}
-
-request(`https://${config.host}/production/users/1`);
+const credentials = await new AWS.CredentialProviderChain().resolvePromise();
+
+// Create a Got instance to use relative paths and signed requests
+const awsClient = got.extend(
+	{
+		baseUrl: 'https://<api-id>.execute-api.<api-region>.amazonaws.com/<stage>/',
+		hooks: {
+			beforeRequest: [
+				async options => {
+					await credentials.getPromise();
+					aws4.sign(options, credentials);
+				}
+			]
+		}
+	}
+);
 
-request(`https://${config.host}/production/`, {
-	// All usual `got` options
+const response = await awsClient('endpoint/path', {
+	// Request-specific options
 });
 ```
 

source/as-stream.js

@@ -9,6 +9,10 @@ module.exports = options => {
 	const input = new PassThrough();
 	const output = new PassThrough();
 	const proxy = duplexer3(input, output);
+	const piped = new Set();
+	let isFinished = false;
+
+	options.gotRetry.retries = () => 0;
 
 	options.gotRetry.retries = () => 0;
 
@@ -52,13 +56,31 @@ module.exports = options => {
 			proxy.emit('error', new ReadError(error, options));
 		});
 
-		response.pipe(output);
-
 		if (options.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
 			proxy.emit('error', new HTTPError(statusCode, response.statusMessage, response.headers, options), null, response);
 			return;
 		}
 
+		isFinished = true;
+
+		response.pipe(output);
+
+		for (const destination of piped) {
+			if (destination.headersSent) {
+				continue;
+			}
+
+			for (const [key, value] of Object.entries(response.headers)) {
+				// Got gives *uncompressed* data. Overriding `content-encoding` header would result in an error.
+				// It's not possible to decompress uncompressed data, is it?
+				if (key.toLowerCase() !== 'content-encoding') {
+					destination.setHeader(key, value);
+				}
+			}
+
+			destination.statusCode = response.statusCode;
+		}
+
 		proxy.emit('response', response);
 	});
 
@@ -69,5 +91,25 @@ module.exports = options => {
 		'downloadProgress'
 	].forEach(event => emitter.on(event, (...args) => proxy.emit(event, ...args)));
 
+	const pipe = proxy.pipe.bind(proxy);
+	const unpipe = proxy.unpipe.bind(proxy);
+	proxy.pipe = (destination, options) => {
+		if (isFinished) {
+			throw new Error('Failed to pipe. The response has been emitted already.');
+		}
+
+		const result = pipe(destination, options);
+
+		if (Reflect.has(destination, 'setHeader')) {
+			piped.add(destination);
+		}
+
+		return result;
+	};
+	proxy.unpipe = stream => {
+		piped.delete(stream);
+		return unpipe(stream);
+	};
+
 	return proxy;
 };

source/index.js

@@ -23,6 +23,9 @@ const defaults = {
 		throwHttpErrors: true,
 		headers: {
 			'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`
+		},
+		hooks: {
+			beforeRequest: []
 		}
 	}
 };

source/normalize-arguments.js

@@ -8,7 +8,8 @@ const isRetryOnNetworkErrorAllowed = require('./is-retry-on-network-error-allowe
 const urlToOptions = require('./url-to-options');
 const isFormData = require('./is-form-data');
 
-const RETRY_AFTER_STATUS_CODES = new Set([413, 429, 503]);
+const retryAfterStatusCodes = new Set([413, 429, 503]);
+const knownHookEvents = ['beforeRequest'];
 
 module.exports = (url, options, defaults) => {
 	if (Reflect.has(options, 'url') || (is.object(url) && Reflect.has(url, 'url'))) {
@@ -67,25 +68,26 @@ module.exports = (url, options, defaults) => {
 		options.method = (options.method || 'GET').toUpperCase();
 	} else {
 		const {headers} = options;
+		const isObject = is.object(body) && !Buffer.isBuffer(body) && !is.nodeStream(body);
 		if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(options.form || options.json)) {
 			throw new TypeError('The `body` option must be a stream.Readable, string or Buffer');
 		}
 
-		if (options.json && !(is.plainObject(body) || is.array(body))) {
-			throw new TypeError('The `body` option must be a plain Object or Array when the `json` option is used');
+		if (options.json && !(isObject || is.array(body))) {
+			throw new TypeError('The `body` option must be an Object or Array when the `json` option is used');
 		}
 
-		if (options.form && !is.plainObject(body)) {
-			throw new TypeError('The `body` option must be a plain Object when the `form` option is used');
+		if (options.form && !isObject) {
+			throw new TypeError('The `body` option must be an Object when the `form` option is used');
 		}
 
 		if (isFormData(body)) {
 			// Special case for https://github.com/form-data/form-data
 			headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
-		} else if (options.form && is.plainObject(body)) {
+		} else if (options.form) {
 			headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
 			options.body = (new URLSearchParamsGlobal(body)).toString();
-		} else if (options.json && (is.plainObject(body) || is.array(body))) {
+		} else if (options.json) {
 			headers['content-type'] = headers['content-type'] || 'application/json';
 			options.body = JSON.stringify(body);
 		}
@@ -151,7 +153,7 @@ module.exports = (url, options, defaults) => {
 				return 0;
 			}
 
-			if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && RETRY_AFTER_STATUS_CODES.has(error.statusCode)) {
+			if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) {
 				let after = Number(error.headers['retry-after']);
 				if (is.number(after)) {
 					after *= 1000;
@@ -189,5 +191,31 @@ module.exports = (url, options, defaults) => {
 		delete options.timeout;
 	}
 
+	if (is.nullOrUndefined(options.hooks)) {
+		options.hooks = {};
+	}
+	if (is.object(options.hooks)) {
+		for (const hookEvent of knownHookEvents) {
+			const hooks = options.hooks[hookEvent];
+			if (is.nullOrUndefined(hooks)) {
+				options.hooks[hookEvent] = [];
+			} else if (is.array(hooks)) {
+				hooks.forEach(
+					(hook, index) => {
+						if (!is.function_(hook)) {
+							throw new TypeError(
+								`Parameter \`hooks.${hookEvent}[${index}]\` must be a function, not ${is(hook)}`
+							);
+						}
+					}
+				);
+			} else {
+				throw new TypeError(`Parameter \`hooks.${hookEvent}\` must be an array, not ${is(hooks)}`);
+			}
+		}
+	} else {
+		throw new TypeError(`Parameter \`hooks\` must be an object, not ${is(options.hooks)}`);
+	}
+
 	return options;
 };

source/request-as-event-emitter.js

@@ -241,6 +241,11 @@ module.exports = (options = {}) => {
 				options.headers['content-length'] = uploadBodySize;
 			}
 
+			for (const hook of options.hooks.beforeRequest) {
+				// eslint-disable-next-line no-await-in-loop
+				await hook(options);
+			}
+
 			get(options);
 		} catch (error) {
 			emitter.emit('error', error);

test/arguments.js

@@ -101,6 +101,40 @@ test('throws TypeError when `url` is passed as an option', async t => {
 	await t.throws(got({url: 'example.com'}), {instanceOf: TypeError});
 });
 
+test('throws TypeError when `hooks` is not an object', async t => {
+	await t.throws(
+		() => got(s.url, {hooks: 'not object'}),
+		{
+			instanceOf: TypeError,
+			message: 'Parameter `hooks` must be an object, not string'
+		}
+	);
+});
+
+test('throws TypeError when known `hooks` value is not an array', async t => {
+	await t.throws(
+		() => got(s.url, {hooks: {beforeRequest: {}}}),
+		{
+			instanceOf: TypeError,
+			message: 'Parameter `hooks.beforeRequest` must be an array, not Object'
+		}
+	);
+});
+
+test('throws TypeError when known `hooks` array item is not a function', async t => {
+	await t.throws(
+		() => got(s.url, {hooks: {beforeRequest: [{}]}}),
+		{
+			instanceOf: TypeError,
+			message: 'Parameter `hooks.beforeRequest[0]` must be a function, not Object'
+		}
+	);
+});
+
+test('allows extra keys in `hooks`', async t => {
+	await t.notThrows(() => got(`${s.url}/test`, {hooks: {extra: {}}}));
+});
+
 test.after('cleanup', async () => {
 	await s.close();
 });

test/error.js

@@ -1,6 +1,7 @@
 import http from 'http';
 import test from 'ava';
 import sinon from 'sinon';
+import getStream from 'get-stream';
 import proxyquire from 'proxyquire';
 import got from '../source';
 import {createServer} from './helpers/server';
@@ -26,6 +27,11 @@ test.before('setup', async () => {
 		res.end('body');
 	});
 
+	s.on('/body', async (req, res) => {
+		const body = await getStream(req);
+		res.end(body);
+	});
+
 	await s.listen(s.port);
 });
 
@@ -53,18 +59,31 @@ test('dns message', async t => {
 });
 
 test('options.body error message', async t => {
-	const err = await t.throws(got(s.url, {body: {}}));
-	t.regex(err.message, /The `body` option must be a stream\.Readable, string or Buffer/);
+	await t.throws(got(s.url, {body: {}}), {
+		message: 'The `body` option must be a stream.Readable, string or Buffer'
+	});
 });
 
 test('options.body json error message', async t => {
-	const err = await t.throws(got(s.url, {body: Buffer.from('test'), json: true}));
-	t.regex(err.message, /The `body` option must be a plain Object or Array when the `json` option is used/);
+	await t.throws(got(s.url, {body: Buffer.from('test'), json: true}), {
+		message: 'The `body` option must be an Object or Array when the `json` option is used'
+	});
 });
 
 test('options.body form error message', async t => {
-	const err = await t.throws(got(s.url, {body: Buffer.from('test'), form: true}));
-	t.regex(err.message, /The `body` option must be a plain Object when the `form` option is used/);
+	await t.throws(got(s.url, {body: Buffer.from('test'), form: true}), {
+		message: 'The `body` option must be an Object when the `form` option is used'
+	});
+});
+
+test('no plain object restriction on body', async t => {
+	function CustomObject() {
+		this.a = 123;
+	}
+
+	const {body} = await got(`${s.url}/body`, {body: new CustomObject(), json: true});
+
+	t.deepEqual(body, {a: 123});
 });
 
 test('default status message', async t => {
@@ -83,9 +102,7 @@ test.serial('http.request error', async t => {
 	const stub = sinon.stub(http, 'request').callsFake(() => {
 		throw new TypeError('The header content contains invalid characters');
 	});
-	const err = await t.throws(got(s.url));
-	t.true(err instanceof got.RequestError);
-	t.is(err.message, 'The header content contains invalid characters');
+	await t.throws(got(s.url), {instanceOf: got.RequestError, message: 'The header content contains invalid characters'});
 	stub.restore();
 });
 
@@ -99,8 +116,7 @@ test.serial('catch error in mimicResponse', async t => {
 		'mimic-response': mimicResponse
 	});
 
-	const err = await t.throws(proxiedGot(s.url));
-	t.is(err.message, 'Error in mimic-response');
+	await t.throws(proxiedGot(s.url), {message: 'Error in mimic-response'});
 });
 
 test.after('cleanup', async () => {