diff --git a/README.md b/README.md
index 4102bae..f2fc9d4 100644
--- a/README.md
+++ b/README.md
@@ -29,6 +29,7 @@ pooling, proxies, retries, [and more](#features)!
* [`opts.retry`](#opts-retry)
* [`opts.onRetry`](#opts-onretry)
* [`opts.integrity`](#opts-integrity)
+ * [`opts.dns`](#opts-dns)
* [Message From Our Sponsors](#wow)
### Example
@@ -67,6 +68,7 @@ fetch('https://registry.npmjs.org/make-fetch-happen').then(res => {
* Transparent gzip and deflate support
* [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) support
* Literally punches nazis
+* Built in DNS cache
* (PENDING) Range request caching and resuming
### Contributing
@@ -146,6 +148,7 @@ make-fetch-happen augments the `minipass-fetch` API with additional features ava
* [`opts.retry`](#opts-retry) - Request retry settings
* [`opts.onRetry`](#opts-onretry) - a function called whenever a retry is attempted
* [`opts.integrity`](#opts-integrity) - [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) metadata.
+* [`opts.dns`](#opts-dns) - DNS cache options
#### `> opts.cachePath`
@@ -387,3 +390,12 @@ fetch('https://malicious-registry.org/make-fetch-happen/-/make-fetch-happen-1.0.
integrity: 'sha1-o47j7zAYnedYFn1dF/fR9OV3z8Q='
}) // Error: EINTEGRITY
```
+
+#### `> opts.dns`
+
+An object that provides options for the built-in DNS cache. The following options are available:
+
+Note: Due to limitations in the current proxy agent implementation, users of proxies will not benefit from the DNS cache.
+
+* `ttl`: Milliseconds to keep cached DNS responses for. Defaults to `5 * 60 * 1000` (5 minutes)
+* `lookup`: A custom lookup function, see [`dns.lookup()`](https://nodejs.org/api/dns.html#dnslookuphostname-options-callback) for implementation details. Defaults to `require('dns').lookup`.
diff --git a/lib/agent.js b/lib/agent.js
index d28a31b..f64644f 100644
--- a/lib/agent.js
+++ b/lib/agent.js
@@ -2,6 +2,7 @@
const LRU = require('lru-cache')
const url = require('url')
const isLambda = require('is-lambda')
+const dns = require('./dns.js')
const AGENT_CACHE = new LRU({ max: 50 })
const HttpAgent = require('agentkeepalive')
@@ -77,11 +78,13 @@ function getAgent (uri, opts) {
rejectUnauthorized: opts.rejectUnauthorized,
timeout: agentTimeout,
freeSocketTimeout: 15000,
+ lookup: dns.getLookup(opts.dns),
}) : new HttpAgent({
maxSockets: agentMaxSockets,
localAddress: opts.localAddress,
timeout: agentTimeout,
freeSocketTimeout: 15000,
+ lookup: dns.getLookup(opts.dns),
})
AGENT_CACHE.set(key, agent)
return agent
@@ -171,6 +174,8 @@ const HttpsProxyAgent = require('https-proxy-agent')
const SocksProxyAgent = require('socks-proxy-agent')
module.exports.getProxy = getProxy
function getProxy (proxyUrl, opts, isHttps) {
+ // our current proxy agents do not support an overridden dns lookup method, so will not
+ // benefit from the dns cache
const popts = {
host: proxyUrl.hostname,
port: proxyUrl.port,
diff --git a/lib/dns.js b/lib/dns.js
new file mode 100644
index 0000000..f817c59
--- /dev/null
+++ b/lib/dns.js
@@ -0,0 +1,49 @@
+const LRUCache = require('lru-cache')
+const dns = require('dns')
+
+const defaultOptions = exports.defaultOptions = {
+ family: undefined,
+ hints: dns.ADDRCONFIG,
+ all: false,
+ verbatim: true,
+}
+
+const lookupCache = exports.lookupCache = new LRUCache({ max: 50 })
+
+// this is a factory so that each request can have its own opts (i.e. ttl)
+// while still sharing the cache across all requests
+exports.getLookup = (dnsOptions) => {
+ return (hostname, options, callback) => {
+ if (typeof options === 'function') {
+ callback = options
+ options = null
+ } else if (typeof options === 'number') {
+ options = { family: options }
+ }
+
+ options = { ...defaultOptions, ...options }
+
+ const key = JSON.stringify({
+ hostname,
+ family: options.family,
+ hints: options.hints,
+ all: options.all,
+ verbatim: options.verbatim,
+ })
+
+ if (lookupCache.has(key)) {
+ const [address, family] = lookupCache.get(key)
+ process.nextTick(callback, null, address, family)
+ return
+ }
+
+ dnsOptions.lookup(hostname, options, (err, address, family) => {
+ if (err) {
+ return callback(err)
+ }
+
+ lookupCache.set(key, [address, family], { ttl: dnsOptions.ttl })
+ return callback(null, address, family)
+ })
+ }
+}
diff --git a/lib/options.js b/lib/options.js
index a0c8664..daa9ecd 100644
--- a/lib/options.js
+++ b/lib/options.js
@@ -1,3 +1,5 @@
+const dns = require('dns')
+
const conditionalHeaders = [
'if-modified-since',
'if-none-match',
@@ -26,6 +28,8 @@ const configureOptions = (opts) => {
options.retry = { retries: 0, ...options.retry }
}
+ options.dns = { ttl: 5 * 60 * 1000, lookup: dns.lookup, ...options.dns }
+
options.cache = options.cache || 'default'
if (options.cache === 'default') {
const hasConditionalHeader = Object.keys(options.headers || {}).some((name) => {
diff --git a/test/agent.js b/test/agent.js
index f8a3f23..2626fa8 100644
--- a/test/agent.js
+++ b/test/agent.js
@@ -47,7 +47,7 @@ t.test('agent: false returns false', async t => {
})
t.test('all expected options passed down to HttpAgent', async t => {
- t.same(agent('http://foo.com/bar', OPTS), {
+ t.match(agent('http://foo.com/bar', OPTS), {
__type: 'http',
maxSockets: 5,
localAddress: 'localAddress',
@@ -57,7 +57,7 @@ t.test('all expected options passed down to HttpAgent', async t => {
})
t.test('timeout 0 keeps timeout 0', async t => {
- t.same(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), {
+ t.match(agent('http://foo.com/bar', { ...OPTS, timeout: 0 }), {
__type: 'http',
maxSockets: 5,
localAddress: 'localAddress',
@@ -67,7 +67,7 @@ t.test('timeout 0 keeps timeout 0', async t => {
})
t.test('no max sockets gets 15 max sockets', async t => {
- t.same(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), {
+ t.match(agent('http://foo.com/bar', { ...OPTS, maxSockets: undefined }), {
__type: 'http',
maxSockets: 15,
localAddress: 'localAddress',
@@ -77,7 +77,7 @@ t.test('no max sockets gets 15 max sockets', async t => {
})
t.test('no timeout gets timeout 0', async t => {
- t.same(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), {
+ t.match(agent('http://foo.com/bar', { ...OPTS, timeout: undefined }), {
__type: 'http',
maxSockets: 5,
localAddress: 'localAddress',
@@ -87,7 +87,7 @@ t.test('no timeout gets timeout 0', async t => {
})
t.test('all expected options passed down to HttpsAgent', async t => {
- t.same(agent('https://foo.com/bar', OPTS), {
+ t.match(agent('https://foo.com/bar', OPTS), {
__type: 'https',
ca: 'ca',
cert: 'cert',
diff --git a/test/dns.js b/test/dns.js
new file mode 100644
index 0000000..560cb71
--- /dev/null
+++ b/test/dns.js
@@ -0,0 +1,221 @@
+const t = require('tap')
+
+const dns = require('../lib/dns.js')
+const DEFAULT_OPTS = { ttl: 5 * 60 * 1000 }
+
+t.afterEach(() => dns.lookupCache.clear())
+
+t.test('supports no options passed', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ t.match(options, dns.defaultOptions, 'applied default options')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ })
+})
+
+t.test('supports family passed directly as options', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ t.match(options, { ...dns.defaultOptions, family: 4 }, 'kept family setting')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', 4, (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ })
+})
+
+t.test('reads from cache', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ t.match(options, dns.defaultOptions, 'applied default options')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }).then(() => new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was still only called once')
+ resolve()
+ })
+ }))
+})
+
+t.test('does not cache errors', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ if (lookupCalled === 1) {
+ process.nextTick(callback, new Error('failed'))
+ return
+ }
+
+ t.match(options, dns.defaultOptions, 'applied default options')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.match(err, { message: 'failed' }, 'got the error')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }).then(() => new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 2, 'lookup was now called twice')
+ resolve()
+ })
+ })).then(() => new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 2, 'lookup was still only called twice')
+ resolve()
+ })
+ }))
+})
+
+t.test('varies when options change', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ if (lookupCalled === 1) {
+ t.match(options, dns.defaultOptions, 'applied default options')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ } else {
+ t.match(options, { ...dns.defaultOptions, family: 6 }, 'kept family from second lookup')
+ process.nextTick(callback, null, '::1', 6)
+ }
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }).then(() => new Promise((resolve) => {
+ lookup('localhost', { family: 6 }, (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '::1', 'got address')
+ t.equal(family, 6, 'got family')
+ t.equal(lookupCalled, 2, 'lookup was called twice')
+ resolve()
+ })
+ }))
+})
+
+t.test('lookup can return all results', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ t.match(options, { ...dns.defaultOptions, all: true }, 'applied default options')
+ process.nextTick(callback, null, [{
+ address: '127.0.0.1', family: 4,
+ }, {
+ address: '::1', family: 6,
+ }])
+ }
+ const lookup = dns.getLookup({ ...DEFAULT_OPTS, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', { all: true }, (err, addresses) => {
+ t.equal(err, null, 'no error')
+ t.match(addresses, [{
+ address: '127.0.0.1', family: 4,
+ }, {
+ address: '::1', family: 6,
+ }], 'got all addresses')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }).then(() => new Promise((resolve) => {
+ lookup('localhost', { all: true }, (err, addresses) => {
+ t.equal(err, null, 'no error')
+ t.match(addresses, [{
+ address: '127.0.0.1', family: 4,
+ }, {
+ address: '::1', family: 6,
+ }], 'got all addresses')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }))
+})
+
+t.test('respects ttl option', async (t) => {
+ let lookupCalled = 0
+ const fakeLookup = (hostname, options, callback) => {
+ lookupCalled += 1
+ t.match(options, dns.defaultOptions, 'applied default options')
+ process.nextTick(callback, null, '127.0.0.1', 4)
+ }
+ const lookup = dns.getLookup({ ttl: 10, lookup: fakeLookup })
+
+ return new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was called once')
+ resolve()
+ })
+ }).then(() => new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 1, 'lookup was still only called once')
+ // delay before the next request to allow the ttl to invalidate
+ setTimeout(resolve, 15)
+ })
+ })).then(() => new Promise((resolve) => {
+ lookup('localhost', (err, address, family) => {
+ t.equal(err, null, 'no error')
+ t.equal(address, '127.0.0.1', 'got address')
+ t.equal(family, 4, 'got family')
+ t.equal(lookupCalled, 2, 'lookup was now called twice')
+ resolve()
+ })
+ }))
+})
diff --git a/test/options.js b/test/options.js
index df6f9a0..5fc32ec 100644
--- a/test/options.js
+++ b/test/options.js
@@ -1,40 +1,67 @@
'use strict'
-
+const dns = require('dns')
const configureOptions = require('../lib/options.js')
const { test } = require('tap')
+const defaultDns = { ttl: 5 * 60 * 1000, lookup: dns.lookup }
+
test('configure options', async (t) => {
test('supplied with no value', async (t) => {
const opts = configureOptions()
- const expectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return default opts')
})
test('supplied with empty object', async (t) => {
const opts = configureOptions({})
- const expectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return default opts')
})
test('changes method to upper case', async (t) => {
const actualOpts = { method: 'post' }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'POST', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'POST',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return upper cased method')
})
test('copies strictSSL to rejectUnauthorized', async (t) => {
const trueOpts = configureOptions({ strictSSL: true })
- const trueExpectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const trueExpectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(trueOpts, trueExpectedObject, 'should return default opts and copy strictSSL')
const falseOpts = configureOptions({ strictSSL: false })
- const falseExpectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: false }
+ const falseExpectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: false,
+ dns: defaultDns,
+ }
t.same(falseOpts, falseExpectedObject, 'should return default opts and copy strictSSL')
const undefinedOpts = configureOptions({ strictSSL: undefined })
@@ -50,44 +77,111 @@ test('configure options', async (t) => {
'should treat strictSSL: null as true just like tls.connect')
})
+ test('should set dns property correctly', async (t) => {
+ t.test('no property given', async (t) => {
+ const actualOpts = { method: 'GET' }
+ const opts = configureOptions(actualOpts)
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
+ t.same(opts, expectedObject, 'should return default retry property')
+ })
+
+ t.test('ttl property given', async (t) => {
+ const actualOpts = { method: 'GET', dns: { ttl: 100 } }
+ const opts = configureOptions(actualOpts)
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: { ...defaultDns, ttl: 100 },
+ }
+ t.same(opts, expectedObject, 'should extend default dns with custom ttl')
+ })
+
+ t.test('lookup property given', async (t) => {
+ const lookup = () => {}
+ const actualOpts = { method: 'GET', dns: { lookup } }
+ const opts = configureOptions(actualOpts)
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: { ...defaultDns, lookup },
+ }
+ t.same(opts, expectedObject, 'should extend default dns with custom lookup')
+ })
+ })
+
test('should set retry property correctly', async (t) => {
t.test('no property given', async (t) => {
const actualOpts = { method: 'GET' }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return default retry property')
})
t.test('invalid property give', async (t) => {
const actualOpts = { method: 'GET', retry: 'one' }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return default retry property')
})
t.test('number value for retry given', async (t) => {
const actualOpts = { method: 'GET', retry: 10 }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'GET', retry: { retries: 10 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 10 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should set retry value, if number')
})
t.test('string number value for retry given', async (t) => {
const actualOpts = { method: 'GET', retry: '10' }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'GET', retry: { retries: 10 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 10 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should set retry value')
})
t.test('truthy value for retry given', async (t) => {
const actualOpts = { method: 'GET', retry: {} }
const opts = configureOptions(actualOpts)
- const expectedObject =
- { method: 'GET', retry: { retries: 0 }, cache: 'default', rejectUnauthorized: true }
+ const expectedObject = {
+ method: 'GET',
+ retry: { retries: 0 },
+ cache: 'default',
+ rejectUnauthorized: true,
+ dns: defaultDns,
+ }
t.same(opts, expectedObject, 'should return default retry property')
})
})
@@ -101,6 +195,7 @@ test('configure options', async (t) => {
rejectUnauthorized: true,
retry: { retries: 0 },
cache: 'default',
+ dns: defaultDns,
}
t.same(opts, expectedObject, 'should set the default cache')
})
@@ -113,6 +208,7 @@ test('configure options', async (t) => {
rejectUnauthorized: true,
retry: { retries: 0 },
cache: 'something',
+ dns: defaultDns,
}
t.same(opts, expectedObject, 'should keep the provided cache')
})