Unverified Commit 5b224854 authored by Andrew Imm's avatar Andrew Imm Committed by GitHub

Limit build workers to pages+1 (#90475)

During build time, we spawn a pool of workers for static page
generation. The size is `max(4, NUM_CORES - 1)`.

With smaller sites, this ends up with far more workers than pages. This
tries to cap the number of workers we actually spin up to something more
appropriate for the number of pages. This change saved 90ms on an app
with fewer static pages, on a 10-core Macbook.

After this change, the behavior will be:
 - If you have 16 cores and 4 pages, we start 5 workers
 - If you have 8 cores and 100 pages, we start 7 workers
parent 96618b87
...@@ -782,28 +782,35 @@ async function writeStandaloneDirectory( ...@@ -782,28 +782,35 @@ async function writeStandaloneDirectory(
}) })
} }
function getNumberOfWorkers(config: NextConfigComplete) { function getNumberOfWorkers(config: NextConfigComplete, maxTasks?: number) {
let workers: number
if ( if (
config.experimental.cpus && config.experimental.cpus &&
config.experimental.cpus !== defaultConfig.experimental!.cpus config.experimental.cpus !== defaultConfig.experimental!.cpus
) { ) {
return config.experimental.cpus // If it's not set to the default, it's a user override
} workers = config.experimental.cpus
} else if (config.experimental.memoryBasedWorkersCount) {
if (config.experimental.memoryBasedWorkersCount) { workers = Math.max(
return Math.max(
Math.min(config.experimental.cpus || 1, Math.floor(os.freemem() / 1e9)), Math.min(config.experimental.cpus || 1, Math.floor(os.freemem() / 1e9)),
// enforce a minimum of 4 workers // enforce a minimum of 4 workers
4 4
) )
} else if (config.experimental.cpus) {
workers = config.experimental.cpus
} else {
// Fall back to 4 workers if a count is not specified
workers = 4
} }
if (config.experimental.cpus) { // Cap workers to one more than the number of tasks.
return config.experimental.cpus // Tasks can be passed to avoid over-allocating workers.
if (maxTasks !== undefined && maxTasks > 0) {
workers = Math.min(workers, maxTasks + 1)
} }
// Fall back to 4 workers if a count is not specified return Math.max(workers, 1)
return 4
} }
const staticWorkerPath = require.resolve('./worker') const staticWorkerPath = require.resolve('./worker')
...@@ -1946,7 +1953,8 @@ export default async function build( ...@@ -1946,7 +1953,8 @@ export default async function build(
// #endregion // #endregion
// #region Collect data // #region Collect data
const numberOfWorkers = getNumberOfWorkers(config) const totalPageCount = pageKeys.pages.length + (pageKeys.app?.length || 0)
const numberOfWorkers = getNumberOfWorkers(config, totalPageCount)
const collectingPageDataStart = process.hrtime() const collectingPageDataStart = process.hrtime()
const postCompileSpinner = createSpinner( const postCompileSpinner = createSpinner(
`Collecting page data using ${numberOfWorkers} worker${numberOfWorkers > 1 ? 's' : ''}` `Collecting page data using ${numberOfWorkers} worker${numberOfWorkers > 1 ? 's' : ''}`
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment