feat(media-provider): stream support (#671)

microlinkhq · Dec 6, 2023 · fec73f8 · fec73f8
1 parent 4f60f1a
commit fec73f8
Show file tree

Hide file tree

Showing 32 changed files with 145,103 additions and 60,276 deletions.
diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json
@@ -25,7 +25,7 @@
     "@metascraper/helpers": "^5.39.0"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
@@ -27,7 +27,7 @@
     "p-reflect": "~2.1.0"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-clearbit/package.json b/packages/metascraper-clearbit/package.json
@@ -31,7 +31,7 @@
     "lodash": "~4.17.21"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json
@@ -25,7 +25,7 @@
     "@metascraper/helpers": "^5.39.0"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-feed/package.json b/packages/metascraper-feed/package.json
@@ -27,7 +27,7 @@
     "@metascraper/helpers": "^5.39.0"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json
@@ -46,7 +46,7 @@
     "video-extensions": "~1.2.0"
   },
   "devDependencies": {
-    "ava": "latest",
+    "ava": "5",
     "cheerio": "latest"
   },
   "engines": {

diff --git a/packages/metascraper-iframe/package.json b/packages/metascraper-iframe/package.json
@@ -31,7 +31,8 @@
     "p-reflect": "~2.1.0"
   },
   "devDependencies": {
-    "ava": "latest",
+    "ava": "5",
+    "cacheable-lookup": "6",
     "cheerio": "latest"
   },
   "engines": {

diff --git a/packages/metascraper-iframe/test/index.js b/packages/metascraper-iframe/test/index.js
@@ -1,5 +1,6 @@
 'use strict'
 
+const CacheableLookup = require('cacheable-lookup')
 const { readFile } = require('fs/promises')
 const { resolve } = require('path')
 const test = require('ava')
@@ -12,19 +13,20 @@ const createMetascraper = (...args) =>
   require('metascraper')([createMetascraperIframe(...args)])
 
 test('provide `gotOpts`', async t => {
-  const cache = new Map()
+  console.log('running')
+  const dnsCache = new CacheableLookup()
   const html = await readFile(resolve(__dirname, 'fixtures/genially.html'))
   const url = 'https://view.genial.ly/5dc53cfa759d2a0f4c7db5f4'
-  const metascraper = createMetascraper({ gotOpts: { cache } })
+  const metascraper = createMetascraper({ gotOpts: { dnsCache } })
 
   const metadataOne = await metascraper({
     url,
     html,
     iframe: { maxWidth: 350 }
   })
-
   t.truthy(metadataOne.iframe)
-  t.is(cache.size, 2)
+
+  t.is(dnsCache._cache.size, 2)
 
   const metadataTwo = await metascraper({
     url,
@@ -33,7 +35,7 @@ test('provide `gotOpts`', async t => {
   })
 
   t.truthy(metadataTwo.iframe)
-  t.is(cache.size, 4)
+  t.is(dnsCache._cache.size, 2)
 })
 
 test('provide `iframe`', async t => {

diff --git a/packages/metascraper-instagram/package.json b/packages/metascraper-instagram/package.json
@@ -27,7 +27,7 @@
     "lodash": "~4.17.21"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json
@@ -25,7 +25,7 @@
     "@metascraper/helpers": "^5.39.0"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json
@@ -29,7 +29,7 @@
     "reachable-url": "~1.7.1"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-manifest/package.json b/packages/metascraper-manifest/package.json
@@ -31,7 +31,7 @@
     "lodash": "~4.17.21"
   },
   "devDependencies": {
-    "ava": "latest"
+    "ava": "5"
   },
   "engines": {
     "node": ">= 16"

diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json
@@ -36,7 +36,7 @@
   },
   "devDependencies": {
     "@kikobeats/time-span": "latest",
-    "ava": "latest",
+    "ava": "5",
     "parse-proxy-uri": "latest",
     "pretty-ms": "latest",
     "unique-random-array": "2"

diff --git a/packages/metascraper-media-provider/src/index.js b/packages/metascraper-media-provider/src/index.js
@@ -68,42 +68,51 @@ const isDownloadable = ({ url }) =>
   new URL(url).searchParams.get('download') === '1'
 
 const getFormatUrls =
-  ({ orderBy }) =>
-    (input, filters) => {
+  (basicFilters, { orderBy }) =>
+    (input, extraFilters, { isStream }) => {
+      const filters = extraFilters.concat(basicFilters({ isStream }))
+
       const formats = get(input, 'formats') ||
       get(input, 'entries[0].formats') || [input]
 
       const url = chain(formats)
         .filter(overEvery(filters))
         .orderBy(orderBy, 'asc')
-        .map('url')
+        .map(isStream ? 'manifest_url' : 'url')
         .last()
         .value()
 
       return !isEmpty(url) ? url : undefined
     }
 
-const getVideoUrls = getFormatUrls({ orderBy: 'tbr' })
-
-const getAudioUrls = getFormatUrls({ orderBy: 'abr' })
-
-const VIDEO_FILTERS = [
-  hasVideo,
-  isMp4,
+const VIDEO_FILTERS = ({ isStream }) =>
+  [
+    isStream ? undefined : hasVideo,
+    isMp4,
+    isHttps,
+    negate(isDownloadable),
+    isStream ? undefined : negate(isM3u8),
+    negate(isMpd),
+    isStream ? undefined : hasVideoCodec
+  ].filter(Boolean)
+
+const AUDIO_FILTERS = () => [
+  hasAudio,
   isHttps,
   negate(isDownloadable),
-  negate(isM3u8),
-  negate(isMpd),
-  hasVideoCodec
+  hasAudioCodec
 ]
 
-const AUDIO_FILTERS = [hasAudio, isHttps, negate(isDownloadable), hasAudioCodec]
+const getVideoUrls = getFormatUrls(VIDEO_FILTERS, { orderBy: 'tbr' })
+
+const getAudioUrls = getFormatUrls(AUDIO_FILTERS, { orderBy: 'abr' })
 
 const getVideo = data =>
-  getVideoUrls(data, [hasAudioCodec, ...VIDEO_FILTERS]) ||
-  getVideoUrls(data, VIDEO_FILTERS)
+  getVideoUrls(data, [hasAudioCodec], { isStream: false }) ||
+  getVideoUrls(data, [], { isStream: false }) ||
+  getVideoUrls(data, [], { isStream: true })
 
-const getAudio = data => getAudioUrls(data, AUDIO_FILTERS)
+const getAudio = data => getAudioUrls(data, [], { isStream: false })
 
 const getAuthor = ({ uploader, creator, uploader_id: uploaderId }) =>
   find([creator, uploader, uploaderId], str => authorFn(str))

diff --git a/packages/metascraper-media-provider/test/fixtures/generate.js b/packages/metascraper-media-provider/test/fixtures/generate.js
@@ -6,14 +6,15 @@ const fs = require('fs/promises')
 
 const { getFlags } = require('../../src/get-media/provider/generic')
 
-const [, , url, filename] = process.argv
+let [, , url, filename] = process.argv
 
 if (!url) throw new TypeError('process.argv[2] should be an url')
 if (!filename) throw new TypeError('process.argv[3] should be a filename')
+if (!filename.endsWith('.json')) filename += '.json'
 ;(async () => {
   const flags = getFlags({ url })
   const payload = await youtubedl(url, flags)
-  const filepath = path.resolve(__dirname, 'provider', `${filename}.json`)
+  const filepath = path.resolve(__dirname, 'provider', filename)
   await fs.writeFile(filepath, JSON.stringify(payload, null, 2))
   console.log('Done at', filepath, '✨')
 })()