Skip to content

Commit

Permalink
feat(gatsby-core-utils): Add file download functions (gatsbyjs#29531)
Browse files Browse the repository at this point in the history 
* feat(gatsby-core-utils): Add file download functions

* skiplibcheck

* Export interface

* Port tests

* Fix type

Co-authored-by: gatsbybot <mathews.kyle+gatsbybot@gmail.com>
  • Loading branch information
@ascorbic
ascorbic and gatsbybot authored Feb 20, 2021
1 parent 475eae3 commit cc1c0cd
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 196 deletions.
198 changes: 3 additions & 195 deletions src/create-remote-file-node.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,11 @@
const fs = require(`fs-extra`)
const got = require(`got`)
const { createContentDigest } = require(`gatsby-core-utils`)
const path = require(`path`)
const { isWebUri } = require(`valid-url`)
const Queue = require(`better-queue`)
const fileType = require(`file-type`)

const { fetchRemoteFile } = require(`gatsby-core-utils`)
const { createFileNode } = require(`./create-file-node`)
const {
getRemoteFileExtension,
getRemoteFileName,
createFilePath,
} = require(`./utils`)
const cacheIdForHeaders = url => `create-remote-file-node-headers-${url}`
const cacheIdForExtensions = url => `create-remote-file-node-extension-${url}`
const { getRemoteFileExtension, createFilePath } = require(`./utils`)

let showFlagWarning = !!process.env.GATSBY_EXPERIMENTAL_REMOTE_FILE_PLACEHOLDER

Expand Down Expand Up @@ -51,13 +43,6 @@ let showFlagWarning = !!process.env.GATSBY_EXPERIMENTAL_REMOTE_FILE_PLACEHOLDER
* @param {Reporter} [options.reporter]
*/

const STALL_RETRY_LIMIT = process.env.GATSBY_STALL_RETRY_LIMIT || 3
const STALL_TIMEOUT = process.env.GATSBY_STALL_TIMEOUT || 30000

const CONNECTION_TIMEOUT = process.env.GATSBY_CONNECTION_TIMEOUT || 30000

const INCOMPLETE_RETRY_LIMIT = process.env.GATSBY_INCOMPLETE_RETRY_LIMIT || 3

/********************
* Queue Management *
********************/
Expand Down Expand Up @@ -103,111 +88,6 @@ async function pushToQueue(task, cb) {
* Core Functions *
******************/

/**
* requestRemoteNode
* --
* Download the requested file
*
* @param {String} url
* @param {Headers} headers
* @param {String} tmpFilename
* @param {Object} httpOpts
* @param {number} attempt
* @return {Promise<Object>} Resolves with the [http Result Object]{@link https://nodejs.org/api/http.html#http_class_http_serverresponse}
*/
const requestRemoteNode = (url, headers, tmpFilename, httpOpts, attempt = 1) =>
new Promise((resolve, reject) => {
let timeout

// Called if we stall for 30s without receiving any data
const handleTimeout = async () => {
fsWriteStream.close()
fs.removeSync(tmpFilename)
if (attempt < STALL_RETRY_LIMIT) {
// Retry by calling ourself recursively
resolve(
requestRemoteNode(url, headers, tmpFilename, httpOpts, attempt + 1)
)
} else {
reject(`Failed to download ${url} after ${STALL_RETRY_LIMIT} attempts`)
}
}

const resetTimeout = () => {
if (timeout) {
clearTimeout(timeout)
}
timeout = setTimeout(handleTimeout, STALL_TIMEOUT)
}
const responseStream = got.stream(url, {
headers,
timeout: {
send: CONNECTION_TIMEOUT, // https://github.com/sindresorhus/got#timeout
},
...httpOpts,
})

let haveAllBytesBeenWritten = false
responseStream.on(`downloadProgress`, progress => {
if (progress.transferred === progress.total || progress.total === null) {
haveAllBytesBeenWritten = true
}
})

const fsWriteStream = fs.createWriteStream(tmpFilename)
responseStream.pipe(fsWriteStream)

// If there's a 400/500 response or other error.
responseStream.on(`error`, error => {
if (timeout) {
clearTimeout(timeout)
}
fs.removeSync(tmpFilename)
reject(error)
})

fsWriteStream.on(`error`, error => {
if (timeout) {
clearTimeout(timeout)
}
reject(error)
})

responseStream.on(`response`, response => {
resetTimeout()

fsWriteStream.on(`finish`, () => {
fsWriteStream.close()

// We have an incomplete download
if (!haveAllBytesBeenWritten) {
fs.removeSync(tmpFilename)

if (attempt < INCOMPLETE_RETRY_LIMIT) {
resolve(
requestRemoteNode(
url,
headers,
tmpFilename,
httpOpts,
attempt + 1
)
)
} else {
reject(
`Failed to download ${url} after ${INCOMPLETE_RETRY_LIMIT} attempts`
)
}
}

if (timeout) {
clearTimeout(timeout)
}
resolve(response)
})
})
})

/**
* processRemoteNode
* --
Expand Down Expand Up @@ -237,7 +117,7 @@ async function processRemoteNode({
name,
})
} else {
filename = await fetchRemoteNode({
filename = await fetchRemoteFile({
url,
cache,
auth,
Expand All @@ -261,77 +141,6 @@ async function processRemoteNode({
return fileNode
}

async function fetchRemoteNode({
url,
cache,
auth = {},
httpHeaders = {},
ext,
name,
}) {
const pluginCacheDir = cache.directory
// See if there's response headers for this url
// from a previous request.
const cachedHeaders = await cache.get(cacheIdForHeaders(url))

const headers = { ...httpHeaders }
if (cachedHeaders && cachedHeaders.etag) {
headers[`If-None-Match`] = cachedHeaders.etag
}

// Add htaccess authentication if passed in. This isn't particularly
// extensible. We should define a proper API that we validate.
const httpOpts = {}
if (auth && (auth.htaccess_pass || auth.htaccess_user)) {
httpOpts.auth = `${auth.htaccess_user}:${auth.htaccess_pass}`
}

// Create the temp and permanent file names for the url.
const digest = createContentDigest(url)
if (!name) {
name = getRemoteFileName(url)
}
if (!ext) {
ext = getRemoteFileExtension(url)
}

const tmpFilename = createFilePath(pluginCacheDir, `tmp-${digest}`, ext)

// Fetch the file.
const response = await requestRemoteNode(url, headers, tmpFilename, httpOpts)

if (response.statusCode === 200) {
// Save the response headers for future requests.
await cache.set(cacheIdForHeaders(url), response.headers)
}

// If the user did not provide an extension and we couldn't get one from remote file, try and guess one
if (ext === ``) {
if (response.statusCode === 200) {
// if this is fresh response - try to guess extension and cache result for future
const filetype = await fileType.fromFile(tmpFilename)
if (filetype) {
ext = `.${filetype.ext}`
await cache.set(cacheIdForExtensions(url), ext)
}
} else if (response.statusCode === 304) {
// if file on server didn't change - grab cached extension
ext = await cache.get(cacheIdForExtensions(url))
}
}

const filename = createFilePath(path.join(pluginCacheDir, digest), name, ext)
// If the status code is 200, move the piped temp file to the real name.
if (response.statusCode === 200) {
await fs.move(tmpFilename, filename, { overwrite: true })
// Else if 304, remove the empty response.
} else {
await fs.remove(tmpFilename)
}

return filename
}

async function fetchPlaceholder({ fromPath, url, cache, ext, name }) {
const pluginCacheDir = cache.directory
const digest = createContentDigest(url)
Expand Down Expand Up @@ -396,7 +205,6 @@ module.exports = function createRemoteFileNode({
createNodeId,
ext = null,
name = null,
reporter,
}) {
if (showFlagWarning) {
showFlagWarning = false
Expand Down
1 change: 0 additions & 1 deletion src/utils.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
const path = require(`path`)
const Url = require(`url`)
const ProgressBar = require(`progress`)

/**
* getParsedPath
Expand Down

0 comments on commit cc1c0cd

Please sign in to comment.