Skip to content

Commit

Permalink
Limit concurrent open files during 'npm cache verify'
Browse files Browse the repository at this point in the history
  • Loading branch information
oikumene committed Jul 11, 2024
1 parent f9ebcea commit 2ffdc51
Showing 1 changed file with 13 additions and 6 deletions.
19 changes: 13 additions & 6 deletions lib/entry-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ const hashToSegments = require('./util/hash-to-segments')
const indexV = require('../package.json')['cache-version'].index
const { moveFile } = require('@npmcli/fs')

const pMap = require('p-map')
const lsStreamConcurrency = 5


Check failure on line 25 in lib/entry-index.js

View workflow job for this annotation

GitHub Actions / Lint

More than 1 blank line not allowed
module.exports.NotFoundError = class NotFoundError extends Error {
constructor (cache, key) {
super(`No cache entry for ${key} found in ${cache}`)
Expand Down Expand Up @@ -182,15 +186,15 @@ function lsStream (cache) {
// Set all this up to run on the stream and then just return the stream
Promise.resolve().then(async () => {
const buckets = await readdirOrEmpty(indexDir)
await Promise.all(buckets.map(async (bucket) => {
await pMap(buckets, async (bucket) => {
const bucketPath = path.join(indexDir, bucket)
const subbuckets = await readdirOrEmpty(bucketPath)
await Promise.all(subbuckets.map(async (subbucket) => {
await pMap(subbuckets, async (subbucket) => {
const subbucketPath = path.join(bucketPath, subbucket)

// "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
const subbucketEntries = await readdirOrEmpty(subbucketPath)
await Promise.all(subbucketEntries.map(async (entry) => {
await pMap(subbucketEntries, async (entry) => {
const entryPath = path.join(subbucketPath, entry)
try {
const entries = await bucketEntries(entryPath)
Expand All @@ -213,9 +217,12 @@ function lsStream (cache) {
}
throw err
}
}))
}))
}))
},
{ concurrency: lsStreamConcurrency })
},
{ concurrency: lsStreamConcurrency })
},
{ concurrency: lsStreamConcurrency })
stream.end()
return stream
}).catch(err => stream.emit('error', err))
Expand Down

0 comments on commit 2ffdc51

Please sign in to comment.