feat(video): improve iframe detection

packages/metascraper-audio/package.json

@@ -28,6 +28,7 @@
     "p-reflect": "~2.1.0"
   },
   "devDependencies": {
+    "async-listen": "latest",
     "ava": "5"
   },
   "engines": {

packages/metascraper-audio/src/index.js

@@ -29,6 +29,8 @@ const toAudioFromDom = toRule((domNodes, opts) => {
         .get(1)
         // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
         .replace('mpeg', 'mp3')
+        /* mp4 is commonly used for video */
+        .replace('mp4', 'mp3')
         .value()
     }))
     .uniqWith(isEqual)
@@ -79,7 +81,7 @@ module.exports = ({ getIframe = _getIframe } = {}) => {
     audio: audioRules.concat(
       async ({ htmlDom: $, url }) => {
         const srcs = [
-          ...new $('iframe')
+          ...new $('iframe[src^="http"], iframe[src^="/"]')
             .map((_, element) => $(element).attr('src'))
             .get()
             .map(src => normalizeUrl(url, src))

packages/metascraper-audio/test/iframe.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { default: listen } = require('async-listen')
+const { createServer } = require('http')
+const { promisify } = require('util')
+const test = require('ava')
+
+const closeServer = server => promisify(server.close)
+
+const createMetascraper = (...args) =>
+  require('metascraper')([require('../src')(...args)])
+
+test('absolute http', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end(
+      '<meta property="og:audio" content="https://cdn.microlink.io/file-examples/sample.mp3">'
+    )
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = `<iframe src="${url}">`
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.audio, 'https://cdn.microlink.io/file-examples/sample.mp3')
+})
+
+test('relative http', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end('<meta property="og:audio" content="/file-examples/sample.mp3">')
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = '<iframe src="/">'
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.audio, url + 'file-examples/sample.mp3')
+})
+
+test('ignore non http urls', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end(
+      '<meta property="og:audio" content="tg://join?invite=n3gS0R7pjFJhMWM0">'
+    )
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = `<iframe src="${url}">`
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.audio, null)
+})

packages/metascraper-audio/test/index.js

@@ -148,3 +148,18 @@ test('`audio > source:src` with content type and relative src', async t => {
     'https://www.theverge.com/2018/1/22/16921092/audio-small'
   )
 })
+
+test('`audio > source:src` with mp4 mime type', async t => {
+  const html = `
+    <audio controls>
+      <source type="audio/mp4" src="https://cdn.microlink.io/file-examples/sample-audio.mp4">
+    </audio>
+    `
+  const url = 'https://metascraper.js.org'
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(
+    metadata.audio,
+    'https://cdn.microlink.io/file-examples/sample-audio.mp4'
+  )
+})

packages/metascraper-helpers/test/index.js

@@ -354,7 +354,7 @@ test('.date', t => {
   t.is(date(undefined), undefined)
   t.is(date(null), undefined)
   t.is(date('null'), undefined)
-  t.is(date('Jun 20'), '2023-06-20T12:00:00.000Z')
+  t.is(date('Jun 20'), '2024-06-20T12:00:00.000Z')
   t.is(date('Jun 20 2018'), '2018-06-20T12:00:00.000Z')
   t.is(date('Jun 2018'), '2018-06-01T12:00:00.000Z')
   t.is(date(2010), '2010-01-01T00:00:00.000Z')

packages/metascraper-media-provider/package.json

@@ -29,7 +29,6 @@
     "debug-logfmt": "~1.0.4",
     "got": "~11.8.6",
     "lodash": "~4.17.21",
-    "p-do-whilst": "~1.1.0",
     "p-reflect": "~2.1.0",
     "p-retry": "~4.6.1",
     "p-timeout": "~4.1.0",

packages/metascraper-media-provider/src/get-media/provider/generic.js

@@ -6,7 +6,6 @@ const debug = require('debug-logfmt')(
 const { serializeError } = require('serialize-error')
 const youtubedl = require('youtube-dl-exec')
 const { get, constant } = require('lodash')
-const pDoWhilst = require('p-do-whilst')
 const pTimeout = require('p-timeout')
 
 const RE_UNSUPORTED_URL = /Unsupported URL/
@@ -53,16 +52,19 @@ module.exports = ({
     const userAgent = get(gotOpts, 'headers.user-agent')
 
     const task = async () => {
-      await pDoWhilst(async () => {
+      do {
         try {
           const proxy = getProxy({ url, retryCount: retryCount++ })
           const flags = getFlags({ url, proxy, userAgent, cacheDir })
           data = await youtubedl(url, flags, { timeout, ...props })
         } catch (error) {
-          if (condition()) { debug('getInfo:error', { retryCount }, serializeError(error)) }
+          if (condition()) {
+            debug('getInfo:error', { retryCount }, serializeError(error))
+          }
           isSupportedURL = !RE_UNSUPORTED_URL.test(error.stderr)
+          console.log('catch', { isSupportedURL })
         }
-      }, condition)
+      } while (condition())
 
       return data
     }

packages/metascraper-video/package.json

@@ -3,8 +3,8 @@
   "description": "Get video property from HTML markup",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-video",
   "version": "5.42.5",
-  "main": "src/index.js",
   "types": "src/index.d.ts",
+  "main": "src/index.js",
   "author": {
     "email": "hello@microlink.io",
     "name": "microlink.io",
@@ -27,6 +27,7 @@
     "lodash": "~4.17.21"
   },
   "devDependencies": {
+    "async-listen": "latest",
     "ava": "5"
   },
   "engines": {

packages/metascraper-video/src/index.js

@@ -3,13 +3,17 @@
 const {
   $jsonld,
   $twitter,
-  loadIframe,
   findRule,
+  has,
+  loadIframe,
+  normalizeUrl,
   toRule,
   url: urlFn,
   video
 } = require('@metascraper/helpers')
 
+const pReflect = require('p-reflect')
+
 const { chain, find, isEqual } = require('lodash')
 
 const toUrl = toRule(urlFn)
@@ -74,26 +78,25 @@ const _getIframe = (url, $, { src }) =>
 
 const withIframe = (rules, getIframe) =>
   rules.concat(
-    //   async ({ htmlDom: $, url }) => {
-    //   // TODO: write a test embedding a youtube video as iframe
-    //   const srcs = [
-    //     ...new $('iframe[src^="http"], iframe[src^="/"]')
-    //       .map((_, element) => $(element).attr('src'))
-    //       .get()
-    //       .map(src => normalizeUrl(url, src))
-    //   ]
-    //   if (srcs.length === 0) return
-    //   return pReflect(
-    //     Promise.any(
-    //       srcs.map(async src => {
-    //         const htmlDom = await getIframe(url, $, { src })
-    //         const result = await findRule(audioRules, { htmlDom, url })
-    //         if (!has(result)) throw TypeError('no result')
-    //         return result
-    //       })
-    //     )
-    //   ).then(({ value }) => value)
-    // },
+    async ({ htmlDom: $, url }) => {
+      const srcs = [
+        ...new $('iframe[src^="http"], iframe[src^="/"]')
+          .map((_, element) => $(element).attr('src'))
+          .get()
+          .map(src => normalizeUrl(url, src))
+      ]
+      if (srcs.length === 0) return
+      return pReflect(
+        Promise.any(
+          srcs.map(async src => {
+            const htmlDom = await getIframe(url, $, { src })
+            const result = await findRule(rules, { htmlDom, url })
+            if (!has(result)) throw TypeError('no result')
+            return result
+          })
+        )
+      ).then(({ value }) => value)
+    },
     async ({ htmlDom: $, url }) => {
       const src = $twitter($, 'twitter:player')
       return src

packages/metascraper-video/test/iframe.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { default: listen } = require('async-listen')
+const { createServer } = require('http')
+const { promisify } = require('util')
+const test = require('ava')
+
+const closeServer = server => promisify(server.close)
+
+const createMetascraper = (...args) =>
+  require('metascraper')([require('../src')(...args)])
+
+test('absolute http', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end(
+      '<meta property="og:video" content="https://cdn.microlink.io/file-examples/sample.mp4">'
+    )
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = `<iframe src="${url}">`
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.video, 'https://cdn.microlink.io/file-examples/sample.mp4')
+})
+
+test('relative http', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end('<meta property="og:video" content="/file-examples/sample.mp4">')
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = '<iframe src="/">'
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.video, url + 'file-examples/sample.mp4')
+})
+
+test('ignore non http urls', async t => {
+  const server = createServer((_, res) => {
+    res.setHeader('Content-Type', 'text/html')
+    res.end(
+      '<meta property="og:video" content="tg://join?invite=n3gS0R7pjFJhMWM0">'
+    )
+  })
+
+  t.teardown(() => closeServer(server))
+  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
+  const html = `<iframe src="${url}">`
+  const metascraper = createMetascraper()
+  const metadata = await metascraper({ html, url })
+  t.is(metadata.video, null)
+})