diff --git a/packages/metascraper-audio/index.js b/packages/metascraper-audio/index.js index fccc3f35a..a3c2fba87 100644 --- a/packages/metascraper-audio/index.js +++ b/packages/metascraper-audio/index.js @@ -11,7 +11,6 @@ const { toRule } = require('@metascraper/helpers') -const reachableUrl = require('reachable-url') const memoize = require('@keyvhq/memoize') const pReflect = require('p-reflect') const cheerio = require('cheerio') @@ -49,16 +48,11 @@ const _getIframe = async (url, { src }) => { const createGetPlayer = ({ gotOpts, keyvOpts }) => { const getPlayer = async playerUrl => { - const response = await reachableUrl(playerUrl, gotOpts) - if (!reachableUrl.isReachable(response)) return + const { value: response } = await pReflect(got(playerUrl, gotOpts)) const contentType = response.headers['content-type'] if (!contentType || !contentType.startsWith('text')) return - const { value: html } = await pReflect( - got(playerUrl, { resolveBodyOnly: true, ...gotOpts }) - ) - return html + return response.body } - return memoize(getPlayer, keyvOpts) } diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json index 352dbf463..209b760fa 100644 --- a/packages/metascraper-audio/package.json +++ b/packages/metascraper-audio/package.json @@ -26,8 +26,7 @@ "@metascraper/helpers": "^5.26.0", "cheerio": "~1.0.0-rc.10", "got": "~11.8.3", - "p-reflect": "~2.1.0", - "reachable-url": "~1.6.7" + "p-reflect": "~2.1.0" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-audio/test/index.js b/packages/metascraper-audio/test/index.js index 33ff546b3..f54981aaa 100644 --- a/packages/metascraper-audio/test/index.js +++ b/packages/metascraper-audio/test/index.js @@ -7,7 +7,7 @@ const createMetascraper = (...args) => require('metascraper')([require('..')(...args)]) describe('metascraper-audio', () => { - it('allow to customize keyOpts', async () => { + it('allow to customize keyv options', async () => { const cache = new Map() const html = '' diff --git a/packages/metascraper-spotify/test/index.js b/packages/metascraper-spotify/test/index.js index 50584e55f..e8a01453a 100644 --- a/packages/metascraper-spotify/test/index.js +++ b/packages/metascraper-spotify/test/index.js @@ -45,6 +45,16 @@ const spotifyUrls = [ ] describe('metascraper-spotify', () => { + it('allow to customize keyv options', async () => { + const url = 'https://open.spotify.com/playlist/0Lt5S4hGarhtZmtz7BNTeX' + const cache = new Map() + const metascraper = createMetascraper({ keyvOpts: { store: cache } }) + await metascraper({ url }) + await metascraper({ url }) + await metascraper({ url }) + should(cache.size > 0).be.true() + }) + describe('.isvalidUrl', () => { describe('true', () => { spotifyUrls.forEach(url => { @@ -53,6 +63,7 @@ describe('metascraper-spotify', () => { }) }) }) + describe('false', () => { ;[ 'https://soundcloud.com/beautybrainsp/beauty-brain-swag-bandicoot' @@ -73,14 +84,4 @@ describe('metascraper-spotify', () => { }) }) }) - - it('allow to customise keyv options', async () => { - const url = 'https://open.spotify.com/playlist/0Lt5S4hGarhtZmtz7BNTeX' - const cache = new Map() - const metascraper = createMetascraper({ keyvOpts: { store: cache } }) - await metascraper({ url }) - await metascraper({ url }) - await metascraper({ url }) - should(cache.size > 0).be.true() - }) }) diff --git a/packages/metascraper-video/index.js b/packages/metascraper-video/index.js index c0c914153..e10801e2c 100644 --- a/packages/metascraper-video/index.js +++ b/packages/metascraper-video/index.js @@ -9,7 +9,6 @@ const { video } = require('@metascraper/helpers') -const reachableUrl = require('reachable-url') const memoize = require('@keyvhq/memoize') const pReflect = require('p-reflect') const { chain } = require('lodash') @@ -43,16 +42,11 @@ const videoRules = [ const createGetPlayer = ({ gotOpts, keyvOpts }) => { const getPlayer = async playerUrl => { - const response = await reachableUrl(playerUrl, gotOpts) - if (!reachableUrl.isReachable(response)) return + const { value: response } = await pReflect(got(playerUrl, gotOpts)) const contentType = response.headers['content-type'] if (!contentType || !contentType.startsWith('text')) return - const { value: html } = await pReflect( - got(playerUrl, { resolveBodyOnly: true, ...gotOpts }) - ) - return html + return response.body } - return memoize(getPlayer, keyvOpts) } diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json index ed237fe89..9ebd0d0ca 100644 --- a/packages/metascraper-video/package.json +++ b/packages/metascraper-video/package.json @@ -27,8 +27,7 @@ "cheerio": "~1.0.0-rc.10", "got": "~11.8.3", "lodash": "~4.17.21", - "p-reflect": "~2.1.0", - "reachable-url": "~1.6.7" + "p-reflect": "~2.1.0" }, "devDependencies": { "mocha": "latest", diff --git a/packages/metascraper-video/test/index.js b/packages/metascraper-video/test/index.js index 357b892f0..e52416cd6 100644 --- a/packages/metascraper-video/test/index.js +++ b/packages/metascraper-video/test/index.js @@ -10,7 +10,7 @@ const createMetascraper = (...args) => require('metascraper')([require('..')(...args)]) describe('metascraper-video', () => { - it('allow to customize keyOpts', async () => { + it('allow to customize keyv options', async () => { const cache = new Map() const html = '' @@ -32,6 +32,7 @@ describe('metascraper-video', () => { snapshot(metadata) }) }) + describe('video', () => { describe('