Skip to content

Commit

Permalink
fix: extract tarball to temp directory on Windows
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsanders11
dsanders11 committed May 8, 2023
1 parent 70bcef0 commit b21e845
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 60 deletions.
179 changes: 119 additions & 60 deletions lib/install.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

const fs = require('graceful-fs')
const os = require('os')
const { backOff } = require('exponential-backoff')
const rm = require('rimraf')
const tar = require('tar')
const path = require('path')
const util = require('util')
Expand Down Expand Up @@ -144,72 +146,129 @@ async function install (fs, gyp, argv) {

// download the tarball and extract!

if (tarPath) {
await tar.extract({
file: tarPath,
strip: 1,
filter: isValid,
onwarn,
cwd: devDir
})
} else {
try {
const res = await download(gyp, release.tarballUrl)
let tarExtractDir = devDir

if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}
// on Windows there can be file errors from tar if parallel installs
// are happening (not uncommon with multiple native modules) so
// extract the tarball to a temp directory first and then swap
if (win) {
// place the temp directory next to devDir rather than in os.tmp()
// since fs.rename() doesn't work if the source and destination
// are on different drives, which might be the case here
tarExtractDir = await fs.promises.mkdtemp(`${devDir}-tmp-`)
}

await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: devDir,
filter: isValid,
onwarn
})
)
} catch (err) {
// something went wrong downloading the tarball?
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
'network settings.')
try {
if (tarPath) {
await tar.extract({
file: tarPath,
strip: 1,
filter: isValid,
onwarn,
cwd: tarExtractDir
})
} else {
try {
const res = await download(gyp, release.tarballUrl)

if (res.status !== 200) {
throw new Error(`${res.status} response downloading ${release.tarballUrl}`)
}

await streamPipeline(
res.body,
// content checksum
new ShaSum((_, checksum) => {
const filename = path.basename(release.tarballUrl).trim()
contentShasums[filename] = checksum
log.verbose('content checksum', filename, checksum)
}),
tar.extract({
strip: 1,
cwd: tarExtractDir,
filter: isValid,
onwarn
})
)
} catch (err) {
// something went wrong downloading the tarball?
if (err.code === 'ENOTFOUND') {
throw new Error('This is most likely not a problem with node-gyp or the package itself and\n' +
'is related to network connectivity. In most cases you are behind a proxy or have bad \n' +
'network settings.')
}
throw err
}
throw err
}
}

// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}
// invoked after the tarball has finished being extracted
if (extractErrors || extractCount === 0) {
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}

log.verbose('tarball', 'done parsing tarball')

const installVersionPath = path.resolve(devDir, 'installVersion')
await Promise.all([
// need to download node.lib
...(win ? downloadNodeLib() : []),
// write the "installVersion" file
fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
// Only download SHASUMS.txt if we downloaded something in need of SHA verification
...(!tarPath || win ? [downloadShasums()] : [])
])

log.verbose('download contents checksum', JSON.stringify(contentShasums))
// check content shasums
for (const k in contentShasums) {
log.verbose('validating download checksum for ' + k, '(%s == %s)', contentShasums[k], expectShasums[k])
if (contentShasums[k] !== expectShasums[k]) {
throw new Error(k + ' local checksum ' + contentShasums[k] + ' not match remote ' + expectShasums[k])
log.verbose('tarball', 'done parsing tarball')

const installVersionPath = path.resolve(tarExtractDir, 'installVersion')
await Promise.all([
// need to download node.lib
...(win ? downloadNodeLib() : []),
// write the "installVersion" file
fs.promises.writeFile(installVersionPath, gyp.package.installVersion + '\n'),
// Only download SHASUMS.txt if we downloaded something in need of SHA verification
...(!tarPath || win ? [downloadShasums()] : [])
])

log.verbose('download contents checksum', JSON.stringify(contentShasums))
// check content shasums
for (const k in contentShasums) {
log.verbose('validating download checksum for ' + k, '(%s == %s)', contentShasums[k], expectShasums[k])
if (contentShasums[k] !== expectShasums[k]) {
throw new Error(k + ' local checksum ' + contentShasums[k] + ' not match remote ' + expectShasums[k])
}
}

// swap in the temp tarball extract directory for devDir;
// with parallel installs this may cause file errors on Windows
// so use an exponential backoff to resolve collisions.
// while graceful-fs has built-in retry logic for rename on
// Windows, it's not exponential backoff and it doesn't handle
// the case where the directory being renamed is missing, which
// needs to be handled here. uses fs.renameSync rather than
// fs.promises.rename as the latter seems to have a bug and would
// let multiple parallel invocations think they "won" the race
if (tarExtractDir !== devDir) {
// per comments in graceful-fs, retry rename for up to 60 seconds
// on Windows due to A/V software locking files for that long
const options = win ? {
numOfAttempts: Infinity,
retry: () => Date.now() - start < 60000
} : undefined
let start
try {
start = Date.now()
await backOff(() => fs.renameSync(devDir, `${tarExtractDir}-old`), options)
start = Date.now()
await backOff(() => fs.renameSync(tarExtractDir, devDir), options)
} catch (err) {
log.error('error while swapping temp tarball extract directory', err)
throw new Error('There was a fatal problem while downloading/extracting the tarball')
}
try {
await util.promisify(rm)(`${tarExtractDir}-old`)
} catch {
log.warn('failed to clean up old version directory')
}
}
} catch (err) {
if (tarExtractDir !== devDir) {
// try to cleanup temp dir
try {
await util.promisify(rm)(tarExtractDir)
} catch {
log.warn('failed to clean up temp tarball extract directory')
}
}
throw err
}

async function downloadShasums () {
Expand Down Expand Up @@ -240,7 +299,7 @@ async function install (fs, gyp, argv) {
log.verbose('on Windows; need to download `' + release.name + '.lib`...')
const archs = ['ia32', 'x64', 'arm64']
return archs.map(async (arch) => {
const dir = path.resolve(devDir, arch)
const dir = path.resolve(tarExtractDir, arch)
const targetLibPath = path.resolve(dir, release.name + '.lib')
const { libUrl, libPath } = release[arch]
const name = `${arch} ${release.name}.lib`
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"main": "./lib/node-gyp.js",
"dependencies": {
"env-paths": "^2.2.0",
"exponential-backoff": "^3.1.1",
"glob": "^7.1.4",
"graceful-fs": "^4.2.6",
"make-fetch-happen": "^10.0.3",
Expand Down

0 comments on commit b21e845

Please sign in to comment.