Skip to content

Commit

Permalink
feat: make it cancelable
Browse files Browse the repository at this point in the history
  • Loading branch information
Kikobeats committed Jun 3, 2023
1 parent 9eb6273 commit 1416d90
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 98 deletions.
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,22 +29,22 @@
"request"
],
"dependencies": {
"@metascraper/helpers": "~5.34.1",
"@metascraper/helpers": "~5.34.7",
"cheerio": "~1.0.0-rc.12",
"css-url-regex": "~4.0.0",
"debug-logfmt": "~1.0.4",
"execall": "~2.0.0",
"got": "~11.8.6",
"html-encode": "~2.1.6",
"html-urls": "~2.4.39",
"html-urls": "~2.4.45",
"is-html-content": "~1.0.0",
"lodash": "~4.17.21",
"mri": "~1.2.0",
"p-cancelable": "~2.1.0",
"p-retry": "~4.6.0",
"replace-string": "~3.1.0",
"time-span": "~4.0.0",
"top-sites": "~1.1.132",
"top-sites": "~1.1.169",
"write-json-file": "~4.3.0"
},
"devDependencies": {
Expand Down
206 changes: 111 additions & 95 deletions src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,78 +59,87 @@ const fetch = PCancelable.fn(
}
)

const prerender = async (
url,
{
getBrowserless,
toEncode,
headers,
gotOpts,
timeout = REQ_TIMEOUT,
abortTypes = ['image', 'stylesheet', 'font'],
...opts
}
) => {
let fetchRes
let data = {}
let isFetchResRejected = false

try {
fetchRes = fetch(url, {
reflect: true,
const prerender = PCancelable.fn(
async (
url,
{
getBrowserless,
toEncode,
...gotOpts,
headers,
timeout
})
const browserless = await getBrowserless()

const getPayload = browserless.evaluate(
async (page, response) => {
if (!response) throw new AbortError('empty response')

return {
headers: response.headers(),
html: await page.content(),
mode: 'prerender',
url: response.url(),
statusCode: response.status()
}
},
{
timeout,
headers,
abortTypes
}
)
gotOpts,
timeout = REQ_TIMEOUT,
abortTypes = ['image', 'stylesheet', 'font'],
...opts
},
onCancel
) => {
let fetchRes
let data = {}
let isFetchResRejected = false

const payload = await getPayload(url, opts)
onCancel(() => fetchRes.cancel())

await fetchRes.cancel()
debug('prerender', { url, state: 'success' })
return payload
} catch (err) {
const { isRejected, ...dataProps } = await fetchRes
try {
fetchRes = fetch(url, {
reflect: true,
toEncode,
...gotOpts,
headers,
timeout
})
const browserless = await getBrowserless()

const getPayload = browserless.evaluate(
async (page, response) => {
if (!response) throw new AbortError('empty response')

return {
headers: response.headers(),
html: await page.content(),
mode: 'prerender',
url: response.url(),
statusCode: response.status()
}
},
{
timeout,
headers,
abortTypes
}
)

debug('prerender:error', {
url,
isRejected,
error: err.message
})
onCancel(() => {
debug('prerender:cancel', { url })
getPayload.cancel()
})

isFetchResRejected = isRejected
data = dataProps
}
const payload = await getPayload(url, opts)
await fetchRes.cancel()
debug('prerender', { url, state: 'success' })
return payload
} catch (err) {
const { isRejected, ...dataProps } = await fetchRes

return isFetchResRejected
? {
headers: data.headers || {},
html: '',
debug('prerender:error', {
url,
mode: 'prerender'
}
: data
}
isRejected,
error: err.message
})

isFetchResRejected = isRejected
data = dataProps
}

return isFetchResRejected
? {
headers: data.headers || {},
html: '',
url,
mode: 'prerender'
}
: data
}
)

const modes = { fetch, prerender }

Expand Down Expand Up @@ -162,40 +171,47 @@ const getContent = async (
return { ...content, html }
}

module.exports = async (
targetUrl,
{
encoding = 'utf-8',
getBrowserless,
getMode = determinateMode,
gotOpts,
headers,
prerender = 'auto',
puppeteerOpts,
rewriteUrls = false
} = {}
) => {
if (!getBrowserless && prerender !== false) {
throw TypeError(
"Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
)
}
module.exports = PCancelable.fn(
async (
targetUrl,
{
encoding = 'utf-8',
getBrowserless,
getMode = determinateMode,
gotOpts,
headers,
prerender = 'auto',
puppeteerOpts,
rewriteUrls = false
} = {},
onCancel
) => {
if (!getBrowserless && prerender !== false) {
throw TypeError(
"Need to provide a `getBrowserless` function. Try to pass `getBrowserless: require('browserless')`"
)
}

const toEncode = htmlEncode(encoding)
const reqMode = getMode(targetUrl, { prerender })
const toEncode = htmlEncode(encoding)
const reqMode = getMode(targetUrl, { prerender })

const time = timeSpan()
const time = timeSpan()

const { mode, ...payload } = await getContent(targetUrl, reqMode, {
getBrowserless,
gotOpts,
headers,
puppeteerOpts,
rewriteUrls,
toEncode
})
const promise = getContent(targetUrl, reqMode, {
getBrowserless,
gotOpts,
headers,
puppeteerOpts,
rewriteUrls,
toEncode
})

return { ...payload, stats: { mode, timing: time.rounded() } }
}
onCancel(() => promise.onCancel())

const { mode, ...payload } = await promise

return { ...payload, stats: { mode, timing: time.rounded() } }
}
)

module.exports.REQ_TIMEOUT = REQ_TIMEOUT
8 changes: 8 additions & 0 deletions test/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@ const wait = async (promise, prop) => {
return prop ? res[prop] : res
}

test('promise is cancelable', async t => {
const targetUrl = 'https://example.com'
const promise = getHTML(targetUrl, { getBrowserless })
promise.catch(() => {})
t.is(!!promise.cancel, true)
promise.cancel()
})

test('reachable URL', async t => {
const url = 'https://example.com'
const [prerenderDisabled, prerenderEnabled] = await Promise.all([
Expand Down

0 comments on commit 1416d90

Please sign in to comment.