Unverified Commit 2c2ac2e8 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

reduce the number of write syscalls by buffering (#23863)

buffer the output of bfj to avoid 1000s of small write calls
parent de567d91
import fs from 'fs' import fs from 'fs'
import path from 'path' import path from 'path'
import { Transform, TransformCallback } from 'stream'
// @ts-ignore no types package // @ts-ignore no types package
import bfj from 'next/dist/compiled/bfj' import bfj from 'next/dist/compiled/bfj'
import { spans } from './profiling-plugin' import { spans } from './profiling-plugin'
...@@ -63,6 +64,46 @@ function reduceSize(stats: any) { ...@@ -63,6 +64,46 @@ function reduceSize(stats: any) {
return stats return stats
} }
const THRESHOLD = 16 * 1024
class BufferingStream extends Transform {
private items: Buffer[] = []
private itemsSize = 0
_transform(
chunk: Buffer | string,
encoding: BufferEncoding,
callback: TransformCallback
): void {
const buffer = Buffer.isBuffer(chunk)
? chunk
: Buffer.from(chunk as string, encoding)
const size = buffer.length
if (this.itemsSize > 0 && this.itemsSize + size > THRESHOLD) {
this.push(Buffer.concat(this.items))
this.itemsSize = 0
this.items.length = 0
}
if (size > THRESHOLD) {
this.push(buffer)
} else {
this.items.push(buffer)
this.itemsSize += size
}
callback()
}
_flush(callback: TransformCallback): void {
if (this.itemsSize > 0) {
this.push(Buffer.concat(this.items))
this.itemsSize = 0
this.items.length = 0
}
callback()
}
}
// This plugin creates a stats.json for a build when enabled // This plugin creates a stats.json for a build when enabled
export default class BuildStatsPlugin { export default class BuildStatsPlugin {
private distDir: string private distDir: string
...@@ -97,13 +138,14 @@ export default class BuildStatsPlugin { ...@@ -97,13 +138,14 @@ export default class BuildStatsPlugin {
}) })
) )
const fileStream = fs.createWriteStream( const fileStream = fs.createWriteStream(
path.join(this.distDir, 'next-stats.json') path.join(this.distDir, 'next-stats.json'),
{ highWaterMark: THRESHOLD }
) )
const jsonStream = bfj.streamify({ const jsonStream = bfj.streamify({
version: STATS_VERSION, version: STATS_VERSION,
stats: statsJson, stats: statsJson,
}) })
jsonStream.pipe(fileStream) jsonStream.pipe(new BufferingStream()).pipe(fileStream)
jsonStream.on('error', reject) jsonStream.on('error', reject)
fileStream.on('error', reject) fileStream.on('error', reject)
jsonStream.on('dataError', reject) jsonStream.on('dataError', reject)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment