diff --git a/README.md b/README.md index 4102bae..f2fc9d4 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ pooling, proxies, retries, [and more](#features)! * [`opts.retry`](#opts-retry) * [`opts.onRetry`](#opts-onretry) * [`opts.integrity`](#opts-integrity) + * [`opts.dns`](#opts-dns) * [Message From Our Sponsors](#wow) ### Example @@ -67,6 +68,7 @@ fetch('https://registry.npmjs.org/make-fetch-happen').then(res => { * Transparent gzip and deflate support * [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) support * Literally punches nazis +* Built in DNS cache * (PENDING) Range request caching and resuming ### Contributing @@ -146,6 +148,7 @@ make-fetch-happen augments the `minipass-fetch` API with additional features ava * [`opts.retry`](#opts-retry) - Request retry settings * [`opts.onRetry`](#opts-onretry) - a function called whenever a retry is attempted * [`opts.integrity`](#opts-integrity) - [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) metadata. +* [`opts.dns`](#opts-dns) - DNS cache options #### `> opts.cachePath` @@ -387,3 +390,12 @@ fetch('https://malicious-registry.org/make-fetch-happen/-/make-fetch-happen-1.0. integrity: 'sha1-o47j7zAYnedYFn1dF/fR9OV3z8Q=' }) // Error: EINTEGRITY ``` + +#### `> opts.dns` + +An object that provides options for the built-in DNS cache. The following options are available: + +Note: Due to limitations in the current proxy agent implementation, users of proxies will not benefit from the DNS cache. + +* `ttl`: Milliseconds to keep cached DNS responses for. Defaults to `5 * 60 * 1000` (5 minutes) +* `lookup`: A custom lookup function, see [`dns.lookup()`](https://nodejs.org/api/dns.html#dnslookuphostname-options-callback) for implementation details. Defaults to `require('dns').lookup`. diff --git a/lib/agent.js b/lib/agent.js index d28a31b..f64644f 100644 --- a/lib/agent.js +++ b/lib/agent.js @@ -2,6 +2,7 @@ const LRU = require('lru-cache') const url = require('url') const isLambda = require('is-lambda') +const dns = require('./dns.js') const AGENT_CACHE = new LRU({ max: 50 }) const HttpAgent = require('agentkeepalive') @@ -77,11 +78,13 @@ function getAgent (uri, opts) { rejectUnauthorized: opts.rejectUnauthorized, timeout: agentTimeout, freeSocketTimeout: 15000, + lookup: dns.getLookup(opts.dns), }) : new HttpAgent({ maxSockets: agentMaxSockets, localAddress: opts.localAddress, timeout: agentTimeout, freeSocketTimeout: 15000, + lookup: dns.getLookup(opts.dns), }) AGENT_CACHE.set(key, agent) return agent @@ -171,6 +174,8 @@ const HttpsProxyAgent = require('https-proxy-agent') const SocksProxyAgent = require('socks-proxy-agent') module.exports.getProxy = getProxy function getProxy (proxyUrl, opts, isHttps) { + // our current proxy agents do not support an overridden dns lookup method, so will not + // benefit from the dns cache const popts = { host: proxyUrl.hostname, port: proxyUrl.port, diff --git a/lib/dns.js b/lib/dns.js new file mode 100644 index 0000000..f817c59 --- /dev/null +++ b/lib/dns.js @@ -0,0 +1,49 @@ +const LRUCache = require('lru-cache') +const dns = require('dns') + +const defaultOptions = exports.defaultOptions = { + family: undefined, + hints: dns.ADDRCONFIG, + all: false, + verbatim: true, +} + +const lookupCache = exports.lookupCache = new LRUCache({ max: 50 }) + +// this is a factory so that each request can have its own opts (i.e. ttl) +// while still sharing the cache across all requests +exports.getLookup = (dnsOptions) => { + return (hostname, options, callback) => { + if (typeof options === 'function') { + callback = options + options = null + } else if (typeof options === 'number') { + options = { family: options } + } + + options = { ...defaultOptions, ...options } + + const key = JSON.stringify({ + hostname, + family: options.family, + hints: options.hints, + all: options.all, + verbatim: options.verbatim, + }) + + if (lookupCache.has(key)) { + const [address, family] = lookupCache.get(key) + process.nextTick(callback, null, address, family) + return + } + + dnsOptions.lookup(hostname, options, (err, address, family) => { + if (err) { + return callback(err) + } + + lookupCache.set(key, [address, family], { ttl: dnsOptions.ttl }) + return callback(null, address, family) + }) + } +} diff --git a/lib/options.js b/lib/options.js index a0c8664..daa9ecd 100644 --- a/lib/options.js +++ b/lib/options.js @@ -1,3 +1,5 @@ +const dns = require('dns') + const conditionalHeaders = [ 'if-modified-since', 'if-none-match', @@ -26,6 +28,8 @@ const configureOptions = (opts) => { options.retry = { retries: 0, ...options.retry } } + options.dns = { ttl: 5 * 60 * 1000, lookup: dns.lookup, ...options.dns } + options.cache = options.cache || 'default' if (options.cache === 'default') { const hasConditionalHeader = Object.keys(options.headers || {}).some((name) => { diff --git a/test/agent.js b/test/agent.js index f8a3f23..2626fa8 100644 --- a/test/agent.js +++ b/test/agent.js @@ -47,7 +47,7 @@ t.test('agent: false returns false', async t => { }) t.test('all expected options passed down to HttpAgent', async t => { - t.same(agent('http://foo.com/bar', OPTS), { + t.match(agent('http://foo.com/bar', OPTS), { __type: 'http', maxSockets: 5, localAddress: 'localAddress', @@ -57,7 +57,7 @@ t.test('all expected options passed down to HttpAgent', async t => { }) t.test('timeout 0 keeps timeout 0', async t => { - t.same(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), { + t.match(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), { __type: 'http', maxSockets: 5, localAddress: 'localAddress', @@ -67,7 +67,7 @@ t.test('timeout 0 keeps timeout 0', async t => { }) t.test('no max sockets gets 15 max sockets', async t => { - t.same(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), { + t.match(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), { __type: 'http', maxSockets: 15, localAddress: 'localAddress', @@ -77,7 +77,7 @@ t.test('no max sockets gets 15 max sockets', async t => { }) t.test('no timeout gets timeout 0', async t => { - t.same(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), { + t.match(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), { __type: 'http', maxSockets: 5, localAddress: 'localAddress', @@ -87,7 +87,7 @@ t.test('no timeout gets timeout 0', async t => { }) t.test('all expected options passed down to HttpsAgent', async t => { - t.same(agent('https://foo.com/bar', OPTS), { + t.match(agent('https://foo.com/bar', OPTS), { __type: 'https', ca: 'ca', cert: 'cert', diff --git a/test/dns.js b/test/dns.js new file mode 100644 index 0000000..560cb71 --- /dev/null +++ b/test/dns.js @@ -0,0 +1,221 @@ +const t = require('tap') + +const dns = require('../lib/dns.js') +const DEFAULT_OPTS = { ttl: 5 * 60 * 1000 } + +t.afterEach(() => dns.lookupCache.clear()) + +t.test('supports no options passed', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + t.match(options, dns.defaultOptions, 'applied default options') + process.nextTick(callback, null, '127.0.0.1', 4) + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }) +}) + +t.test('supports family passed directly as options', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + t.match(options, { ...dns.defaultOptions, family: 4 }, 'kept family setting') + process.nextTick(callback, null, '127.0.0.1', 4) + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', 4, (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }) +}) + +t.test('reads from cache', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + t.match(options, dns.defaultOptions, 'applied default options') + process.nextTick(callback, null, '127.0.0.1', 4) + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }).then(() => new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was still only called once') + resolve() + }) + })) +}) + +t.test('does not cache errors', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + if (lookupCalled === 1) { + process.nextTick(callback, new Error('failed')) + return + } + + t.match(options, dns.defaultOptions, 'applied default options') + process.nextTick(callback, null, '127.0.0.1', 4) + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.match(err, { message: 'failed' }, 'got the error') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }).then(() => new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 2, 'lookup was now called twice') + resolve() + }) + })).then(() => new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 2, 'lookup was still only called twice') + resolve() + }) + })) +}) + +t.test('varies when options change', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + if (lookupCalled === 1) { + t.match(options, dns.defaultOptions, 'applied default options') + process.nextTick(callback, null, '127.0.0.1', 4) + } else { + t.match(options, { ...dns.defaultOptions, family: 6 }, 'kept family from second lookup') + process.nextTick(callback, null, '::1', 6) + } + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }).then(() => new Promise((resolve) => { + lookup('localhost', { family: 6 }, (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '::1', 'got address') + t.equal(family, 6, 'got family') + t.equal(lookupCalled, 2, 'lookup was called twice') + resolve() + }) + })) +}) + +t.test('lookup can return all results', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + t.match(options, { ...dns.defaultOptions, all: true }, 'applied default options') + process.nextTick(callback, null, [{ + address: '127.0.0.1', family: 4, + }, { + address: '::1', family: 6, + }]) + } + const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', { all: true }, (err, addresses) => { + t.equal(err, null, 'no error') + t.match(addresses, [{ + address: '127.0.0.1', family: 4, + }, { + address: '::1', family: 6, + }], 'got all addresses') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }).then(() => new Promise((resolve) => { + lookup('localhost', { all: true }, (err, addresses) => { + t.equal(err, null, 'no error') + t.match(addresses, [{ + address: '127.0.0.1', family: 4, + }, { + address: '::1', family: 6, + }], 'got all addresses') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + })) +}) + +t.test('respects ttl option', async (t) => { + let lookupCalled = 0 + const fakeLookup = (hostname, options, callback) => { + lookupCalled += 1 + t.match(options, dns.defaultOptions, 'applied default options') + process.nextTick(callback, null, '127.0.0.1', 4) + } + const lookup = dns.getLookup({ ttl: 10, lookup: fakeLookup }) + + return new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was called once') + resolve() + }) + }).then(() => new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 1, 'lookup was still only called once') + // delay before the next request to allow the ttl to invalidate + setTimeout(resolve, 15) + }) + })).then(() => new Promise((resolve) => { + lookup('localhost', (err, address, family) => { + t.equal(err, null, 'no error') + t.equal(address, '127.0.0.1', 'got address') + t.equal(family, 4, 'got family') + t.equal(lookupCalled, 2, 'lookup was now called twice') + resolve() + }) + })) +}) diff --git a/test/options.js b/test/options.js index df6f9a0..5fc32ec 100644 --- a/test/options.js +++ b/test/options.js @@ -1,40 +1,67 @@ 'use strict' - +const dns = require('dns') const configureOptions = require('../lib/options.js') const { test } = require('tap') +const defaultDns = { ttl: 5 * 60 * 1000, lookup: dns.lookup } + test('configure options', async (t) => { test('supplied with no value', async (t) => { const opts = configureOptions() - const expectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return default opts') }) test('supplied with empty object', async (t) => { const opts = configureOptions({}) - const expectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return default opts') }) test('changes method to upper case', async (t) => { const actualOpts = { method: 'post' } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'POST', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'POST', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return upper cased method') }) test('copies strictSSL to rejectUnauthorized', async (t) => { const trueOpts = configureOptions({ strictSSL: true }) - const trueExpectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const trueExpectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(trueOpts, trueExpectedObject, 'should return default opts and copy strictSSL') const falseOpts = configureOptions({ strictSSL: false }) - const falseExpectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: false } + const falseExpectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: false, + dns: defaultDns, + } t.same(falseOpts, falseExpectedObject, 'should return default opts and copy strictSSL') const undefinedOpts = configureOptions({ strictSSL: undefined }) @@ -50,44 +77,111 @@ test('configure options', async (t) => { 'should treat strictSSL: null as true just like tls.connect') }) + test('should set dns property correctly', async (t) => { + t.test('no property given', async (t) => { + const actualOpts = { method: 'GET' } + const opts = configureOptions(actualOpts) + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } + t.same(opts, expectedObject, 'should return default retry property') + }) + + t.test('ttl property given', async (t) => { + const actualOpts = { method: 'GET', dns: { ttl: 100 } } + const opts = configureOptions(actualOpts) + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: { ...defaultDns, ttl: 100 }, + } + t.same(opts, expectedObject, 'should extend default dns with custom ttl') + }) + + t.test('lookup property given', async (t) => { + const lookup = () => {} + const actualOpts = { method: 'GET', dns: { lookup } } + const opts = configureOptions(actualOpts) + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: { ...defaultDns, lookup }, + } + t.same(opts, expectedObject, 'should extend default dns with custom lookup') + }) + }) + test('should set retry property correctly', async (t) => { t.test('no property given', async (t) => { const actualOpts = { method: 'GET' } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return default retry property') }) t.test('invalid property give', async (t) => { const actualOpts = { method: 'GET', retry: 'one' } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return default retry property') }) t.test('number value for retry given', async (t) => { const actualOpts = { method: 'GET', retry: 10 } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'GET', retry: { retries: 10 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 10 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should set retry value, if number') }) t.test('string number value for retry given', async (t) => { const actualOpts = { method: 'GET', retry: '10' } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'GET', retry: { retries: 10 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 10 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should set retry value') }) t.test('truthy value for retry given', async (t) => { const actualOpts = { method: 'GET', retry: {} } const opts = configureOptions(actualOpts) - const expectedObject = - { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true } + const expectedObject = { + method: 'GET', + retry: { retries: 0 }, + cache: 'default', + rejectUnauthorized: true, + dns: defaultDns, + } t.same(opts, expectedObject, 'should return default retry property') }) }) @@ -101,6 +195,7 @@ test('configure options', async (t) => { rejectUnauthorized: true, retry: { retries: 0 }, cache: 'default', + dns: defaultDns, } t.same(opts, expectedObject, 'should set the default cache') }) @@ -113,6 +208,7 @@ test('configure options', async (t) => { rejectUnauthorized: true, retry: { retries: 0 }, cache: 'something', + dns: defaultDns, } t.same(opts, expectedObject, 'should keep the provided cache') })