Skip to content

Commit

Permalink
reduce the number of write syscalls by buffering (#23863)
Browse files Browse the repository at this point in the history
buffer the output of bfj to avoid 1000s of small write calls
  • Loading branch information
sokra authored Apr 9, 2021
1 parent de567d9 commit 2c2ac2e
Showing 1 changed file with 44 additions and 2 deletions.
46 changes: 44 additions & 2 deletions packages/next/build/webpack/plugins/build-stats-plugin.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import fs from 'fs'
import path from 'path'
import { Transform, TransformCallback } from 'stream'
// @ts-ignore no types package
import bfj from 'next/dist/compiled/bfj'
import { spans } from './profiling-plugin'
Expand Down Expand Up @@ -63,6 +64,46 @@ function reduceSize(stats: any) {
return stats
}

const THRESHOLD = 16 * 1024
class BufferingStream extends Transform {
private items: Buffer[] = []
private itemsSize = 0

_transform(
chunk: Buffer | string,
encoding: BufferEncoding,
callback: TransformCallback
): void {
const buffer = Buffer.isBuffer(chunk)
? chunk
: Buffer.from(chunk as string, encoding)
const size = buffer.length
if (this.itemsSize > 0 && this.itemsSize + size > THRESHOLD) {
this.push(Buffer.concat(this.items))
this.itemsSize = 0
this.items.length = 0
}
if (size > THRESHOLD) {
this.push(buffer)
} else {
this.items.push(buffer)
this.itemsSize += size
}

callback()
}

_flush(callback: TransformCallback): void {
if (this.itemsSize > 0) {
this.push(Buffer.concat(this.items))
this.itemsSize = 0
this.items.length = 0
}

callback()
}
}

// This plugin creates a stats.json for a build when enabled
export default class BuildStatsPlugin {
private distDir: string
Expand Down Expand Up @@ -97,13 +138,14 @@ export default class BuildStatsPlugin {
})
)
const fileStream = fs.createWriteStream(
path.join(this.distDir, 'next-stats.json')
path.join(this.distDir, 'next-stats.json'),
{ highWaterMark: THRESHOLD }
)
const jsonStream = bfj.streamify({
version: STATS_VERSION,
stats: statsJson,
})
jsonStream.pipe(fileStream)
jsonStream.pipe(new BufferingStream()).pipe(fileStream)
jsonStream.on('error', reject)
fileStream.on('error', reject)
jsonStream.on('dataError', reject)
Expand Down

0 comments on commit 2c2ac2e

Please sign in to comment.