Unverified Commit 5b224854 authored by Andrew Imm's avatar Andrew Imm Committed by GitHub

Limit build workers to pages+1 (#90475)

During build time, we spawn a pool of workers for static page
generation. The size is `max(4, NUM_CORES - 1)`.

With smaller sites, this ends up with far more workers than pages. This
tries to cap the number of workers we actually spin up to something more
appropriate for the number of pages. This change saved 90ms on an app
with fewer static pages, on a 10-core Macbook.

After this change, the behavior will be:
 - If you have 16 cores and 4 pages, we start 5 workers
 - If you have 8 cores and 100 pages, we start 7 workers
parent 96618b87
......@@ -782,28 +782,35 @@ async function writeStandaloneDirectory(
})
}
function getNumberOfWorkers(config: NextConfigComplete) {
function getNumberOfWorkers(config: NextConfigComplete, maxTasks?: number) {
let workers: number
if (
config.experimental.cpus &&
config.experimental.cpus !== defaultConfig.experimental!.cpus
) {
return config.experimental.cpus
}
if (config.experimental.memoryBasedWorkersCount) {
return Math.max(
// If it's not set to the default, it's a user override
workers = config.experimental.cpus
} else if (config.experimental.memoryBasedWorkersCount) {
workers = Math.max(
Math.min(config.experimental.cpus || 1, Math.floor(os.freemem() / 1e9)),
// enforce a minimum of 4 workers
4
)
} else if (config.experimental.cpus) {
workers = config.experimental.cpus
} else {
// Fall back to 4 workers if a count is not specified
workers = 4
}
if (config.experimental.cpus) {
return config.experimental.cpus
// Cap workers to one more than the number of tasks.
// Tasks can be passed to avoid over-allocating workers.
if (maxTasks !== undefined && maxTasks > 0) {
workers = Math.min(workers, maxTasks + 1)
}
// Fall back to 4 workers if a count is not specified
return 4
return Math.max(workers, 1)
}
const staticWorkerPath = require.resolve('./worker')
......@@ -1946,7 +1953,8 @@ export default async function build(
// #endregion
// #region Collect data
const numberOfWorkers = getNumberOfWorkers(config)
const totalPageCount = pageKeys.pages.length + (pageKeys.app?.length || 0)
const numberOfWorkers = getNumberOfWorkers(config, totalPageCount)
const collectingPageDataStart = process.hrtime()
const postCompileSpinner = createSpinner(
`Collecting page data using ${numberOfWorkers} worker${numberOfWorkers > 1 ? 's' : ''}`
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment