Unverified Commit 46521db5 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: limit build concurrency, show progress bar (#62319)

### What?

Run 10 concurrent Turbopack compilation jobs instead of all of them
show a progress spinner

### Why?

* In future this allows to reclaim memory for finished jobs
* It looks nicer in tracing


Closes PACK-2561
parent 8421a971
...@@ -19,6 +19,7 @@ import { defaultConfig } from '../server/config-shared' ...@@ -19,6 +19,7 @@ import { defaultConfig } from '../server/config-shared'
import devalue from 'next/dist/compiled/devalue' import devalue from 'next/dist/compiled/devalue'
import findUp from 'next/dist/compiled/find-up' import findUp from 'next/dist/compiled/find-up'
import { nanoid } from 'next/dist/compiled/nanoid/index.cjs' import { nanoid } from 'next/dist/compiled/nanoid/index.cjs'
import { Sema } from 'next/dist/compiled/async-sema'
import path from 'path' import path from 'path'
import { import {
STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR, STATIC_STATUS_PAGE_GET_INITIAL_PROPS_ERROR,
...@@ -177,6 +178,7 @@ import { ...@@ -177,6 +178,7 @@ import {
import { TurbopackManifestLoader } from '../server/dev/turbopack/manifest-loader' import { TurbopackManifestLoader } from '../server/dev/turbopack/manifest-loader'
import type { Entrypoints } from '../server/dev/turbopack/types' import type { Entrypoints } from '../server/dev/turbopack/types'
import { buildCustomRoute } from '../lib/build-custom-route' import { buildCustomRoute } from '../lib/build-custom-route'
import { createProgress } from './progress'
interface ExperimentalBypassForInfo { interface ExperimentalBypassForInfo {
experimentalBypassFor?: RouteHas[] experimentalBypassFor?: RouteHas[]
...@@ -1416,9 +1418,28 @@ export default async function build( ...@@ -1416,9 +1418,28 @@ export default async function build(
rewrites: emptyRewritesObjToBeImplemented, rewrites: emptyRewritesObjToBeImplemented,
}) })
const promises = [] const progress = createProgress(
for (const [page, route] of currentEntrypoints.page) { currentEntrypoints.page.size + currentEntrypoints.app.size + 1,
'Building'
)
const promises: Promise<any>[] = []
const sema = new Sema(10)
const enqueue = (fn: () => Promise<void>) => {
promises.push( promises.push(
(async () => {
await sema.acquire()
try {
await fn()
} finally {
sema.release()
progress()
}
})()
)
}
for (const [page, route] of currentEntrypoints.page) {
enqueue(() =>
handleRouteType({ handleRouteType({
dev, dev,
page, page,
...@@ -1434,7 +1455,7 @@ export default async function build( ...@@ -1434,7 +1455,7 @@ export default async function build(
} }
for (const [page, route] of currentEntrypoints.app) { for (const [page, route] of currentEntrypoints.app) {
promises.push( enqueue(() =>
handleRouteType({ handleRouteType({
page, page,
dev: false, dev: false,
...@@ -1448,7 +1469,7 @@ export default async function build( ...@@ -1448,7 +1469,7 @@ export default async function build(
) )
} }
promises.push( enqueue(() =>
handlePagesErrorRoute({ handlePagesErrorRoute({
currentIssues, currentIssues,
entrypoints: currentEntrypoints, entrypoints: currentEntrypoints,
......
import * as Log from '../build/output/log'
import createSpinner from './spinner'
function divideSegments(number: number, segments: number): number[] {
const result = []
while (number > 0 && segments > 0) {
const dividedNumber =
number < segments ? number : Math.floor(number / segments)
number -= dividedNumber
segments--
result.push(dividedNumber)
}
return result
}
export const createProgress = (total: number, label: string) => {
const segments = divideSegments(total, 4)
if (total === 0) {
throw new Error('invariant: progress total can not be zero')
}
let currentSegmentTotal = segments.shift()
let currentSegmentCount = 0
let lastProgressOutput = Date.now()
let curProgress = 0
let progressSpinner = createSpinner(`${label} (${curProgress}/${total})`, {
spinner: {
frames: [
'[ ]',
'[= ]',
'[== ]',
'[=== ]',
'[ ===]',
'[ ==]',
'[ =]',
'[ ]',
'[ =]',
'[ ==]',
'[ ===]',
'[====]',
'[=== ]',
'[== ]',
'[= ]',
],
interval: 200,
},
})
return () => {
curProgress++
// Make sure we only log once
// - per fully generated segment, or
// - per minute
// when not showing the spinner
if (!progressSpinner) {
currentSegmentCount++
if (currentSegmentCount === currentSegmentTotal) {
currentSegmentTotal = segments.shift()
currentSegmentCount = 0
} else if (lastProgressOutput + 60000 > Date.now()) {
return
}
lastProgressOutput = Date.now()
}
const isFinished = curProgress === total
if (progressSpinner && !isFinished) {
progressSpinner.setText(`${label} (${curProgress}/${total})`)
} else {
if (progressSpinner) {
progressSpinner.stop()
}
console.log(
` ${
isFinished ? Log.prefixes.event : Log.prefixes.info
} ${label} (${curProgress}/${total}) ${
isFinished ? '' : process.stdout.isTTY ? '\n' : '\r'
}`
)
}
}
}
...@@ -11,11 +11,9 @@ export default function createSpinner( ...@@ -11,11 +11,9 @@ export default function createSpinner(
options: ora.Options = {}, options: ora.Options = {},
logFn: (...data: any[]) => void = console.log logFn: (...data: any[]) => void = console.log
) { ) {
let spinner: undefined | ora.Ora let spinner: undefined | (ora.Ora & { setText: (text: string) => void })
const prefixText = ` ${Log.prefixes.info} ${text} ` let prefixText = ` ${Log.prefixes.info} ${text} `
// Add \r at beginning to reset the current line of loading status text
const suffixText = `\r ${Log.prefixes.event} ${text} `
if (process.stdout.isTTY) { if (process.stdout.isTTY) {
spinner = ora({ spinner = ora({
...@@ -24,7 +22,7 @@ export default function createSpinner( ...@@ -24,7 +22,7 @@ export default function createSpinner(
spinner: dotsSpinner, spinner: dotsSpinner,
stream: process.stdout, stream: process.stdout,
...options, ...options,
}).start() }).start() as ora.Ora & { setText: (text: string) => void }
// Add capturing of console.log/warn/error to allow pausing // Add capturing of console.log/warn/error to allow pausing
// the spinner before logging and then restarting spinner after // the spinner before logging and then restarting spinner after
...@@ -49,18 +47,24 @@ export default function createSpinner( ...@@ -49,18 +47,24 @@ export default function createSpinner(
console.warn = origWarn console.warn = origWarn
console.error = origError console.error = origError
} }
spinner.setText = (newText: string) => {
text = newText
prefixText = ` ${Log.prefixes.info} ${newText} `
spinner!.prefixText = prefixText
return spinner!
}
spinner.stop = (): ora.Ora => { spinner.stop = (): ora.Ora => {
origStop() origStop()
resetLog() resetLog()
return spinner! return spinner!
} }
spinner.stopAndPersist = (): ora.Ora => { spinner.stopAndPersist = (): ora.Ora => {
if (suffixText) { // Add \r at beginning to reset the current line of loading status text
if (spinner) { const suffixText = `\r ${Log.prefixes.event} ${text} `
spinner.text = suffixText if (spinner) {
} else { spinner.text = suffixText
logFn(suffixText) } else {
} logFn(suffixText)
} }
origStopAndPersist() origStopAndPersist()
resetLog() resetLog()
......
...@@ -18,7 +18,6 @@ import { dirname, join, resolve, sep } from 'path' ...@@ -18,7 +18,6 @@ import { dirname, join, resolve, sep } from 'path'
import { formatAmpMessages } from '../build/output/index' import { formatAmpMessages } from '../build/output/index'
import type { AmpPageStatus } from '../build/output/index' import type { AmpPageStatus } from '../build/output/index'
import * as Log from '../build/output/log' import * as Log from '../build/output/log'
import createSpinner from '../build/spinner'
import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants' import { RSC_SUFFIX, SSG_FALLBACK_EXPORT_ERROR } from '../lib/constants'
import { recursiveCopy } from '../lib/recursive-copy' import { recursiveCopy } from '../lib/recursive-copy'
import { import {
...@@ -56,93 +55,7 @@ import { needsExperimentalReact } from '../lib/needs-experimental-react' ...@@ -56,93 +55,7 @@ import { needsExperimentalReact } from '../lib/needs-experimental-react'
import { formatManifest } from '../build/manifests/formatter/format-manifest' import { formatManifest } from '../build/manifests/formatter/format-manifest'
import { validateRevalidate } from '../server/lib/patch-fetch' import { validateRevalidate } from '../server/lib/patch-fetch'
import { TurborepoAccessTraceResult } from '../build/turborepo-access-trace' import { TurborepoAccessTraceResult } from '../build/turborepo-access-trace'
import { createProgress } from '../build/progress'
function divideSegments(number: number, segments: number): number[] {
const result = []
while (number > 0 && segments > 0) {
const dividedNumber =
number < segments ? number : Math.floor(number / segments)
number -= dividedNumber
segments--
result.push(dividedNumber)
}
return result
}
const createProgress = (total: number, label: string) => {
const segments = divideSegments(total, 4)
if (total === 0) {
throw new Error('invariant: progress total can not be zero')
}
let currentSegmentTotal = segments.shift()
let currentSegmentCount = 0
let lastProgressOutput = Date.now()
let curProgress = 0
let progressSpinner = createSpinner(`${label} (${curProgress}/${total})`, {
spinner: {
frames: [
'[ ]',
'[= ]',
'[== ]',
'[=== ]',
'[ ===]',
'[ ==]',
'[ =]',
'[ ]',
'[ =]',
'[ ==]',
'[ ===]',
'[====]',
'[=== ]',
'[== ]',
'[= ]',
],
interval: 200,
},
})
return () => {
curProgress++
// Make sure we only log once
// - per fully generated segment, or
// - per minute
// when not showing the spinner
if (!progressSpinner) {
currentSegmentCount++
if (currentSegmentCount === currentSegmentTotal) {
currentSegmentTotal = segments.shift()
currentSegmentCount = 0
} else if (lastProgressOutput + 60000 > Date.now()) {
return
}
lastProgressOutput = Date.now()
}
const isFinished = curProgress === total
// Use \r to reset current line with spinner.
// If it's 100% progressed, then we don't need to break a new line to avoid logging from routes while building.
const newText = `\r ${
isFinished ? Log.prefixes.event : Log.prefixes.info
} ${label} (${curProgress}/${total}) ${
isFinished ? '' : process.stdout.isTTY ? '\n' : '\r'
}`
if (progressSpinner) {
progressSpinner.text = newText
} else {
console.log(newText)
}
if (isFinished && progressSpinner) {
progressSpinner.stop()
console.log(newText)
}
}
}
export class ExportError extends Error { export class ExportError extends Error {
code = 'NEXT_EXPORT_ERROR' code = 'NEXT_EXPORT_ERROR'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment