diff --git a/package.json b/package.json index c8247d0..1b02ced 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "time-span": "~3.1.0", "tldts": "~5.6.7", "top-sites": "~1.1.17", + "whoops": "~4.1.0", "write-json-file": "~4.3.0" }, "devDependencies": { diff --git a/src/index.js b/src/index.js index 51d41fc..db0fad8 100644 --- a/src/index.js +++ b/src/index.js @@ -3,10 +3,12 @@ const { isMediaUrl } = require('@metascraper/helpers') const { getDomainWithoutSuffix } = require('tldts') const debug = require('debug-logfmt')('html-get') + const requireOneOf = require('require-one-of') const PCancelable = require('p-cancelable') const htmlEncode = require('html-encode') const timeSpan = require('time-span') +const whoops = require('whoops') const got = require('got') const he = require('he') @@ -15,6 +17,8 @@ const addHtml = require('./html') const REQ_TIMEOUT = 8000 +const abortError = whoops('AbortError') + const getHtml = html => he.decode(html) const fetch = (url, { reflect = false, toEncode, ...opts }) => @@ -56,12 +60,16 @@ const prerender = async ( fetchRes = fetch(url, { reflect: true, toEncode, headers, ...gotOpts }) const browserless = await getBrowserless() - const getPayload = browserless.evaluate(async (page, response) => ({ - headers: response.headers(), - html: getHtml(await page.content()), - mode: 'prerender', - url: response.url() - })) + const getPayload = browserless.evaluate(async (page, response) => { + if (!response) throw abortError() + + return { + headers: response.headers(), + html: getHtml(await page.content()), + mode: 'prerender', + url: response.url() + } + }) const payload = await getPayload(url, { timeout: REQ_TIMEOUT, @@ -82,11 +90,11 @@ const prerender = async ( return isFetchResRejected ? { - headers: fetchDataProps.headers || {}, - html: '', - url, - mode: 'prerender' - } + headers: fetchDataProps.headers || {}, + html: '', + url, + mode: 'prerender' + } : fetchDataProps }