Unverified Commit 493036df authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

turbopack: improve turbopack/test stability (#56024)

### What?

a few cherry picked fixes to test execution, issue reporting and hmr
behavior

### Why?

### How?


Closes WEB-1657
parent 7102e297
...@@ -51,6 +51,7 @@ interface Dispatcher { ...@@ -51,6 +51,7 @@ interface Dispatcher {
let mostRecentCompilationHash: any = null let mostRecentCompilationHash: any = null
let __nextDevClientId = Math.round(Math.random() * 100 + Date.now()) let __nextDevClientId = Math.round(Math.random() * 100 + Date.now())
let reloading = false
function onBeforeFastRefresh(dispatcher: Dispatcher, hasUpdates: boolean) { function onBeforeFastRefresh(dispatcher: Dispatcher, hasUpdates: boolean) {
if (hasUpdates) { if (hasUpdates) {
...@@ -115,6 +116,8 @@ function performFullReload(err: any, sendMessage: any) { ...@@ -115,6 +116,8 @@ function performFullReload(err: any, sendMessage: any) {
}) })
) )
if (reloading) return
reloading = true
window.location.reload() window.location.reload()
} }
...@@ -357,6 +360,8 @@ function processMessage( ...@@ -357,6 +360,8 @@ function processMessage(
}) })
) )
if (RuntimeErrorHandler.hadRuntimeError) { if (RuntimeErrorHandler.hadRuntimeError) {
if (reloading) return
reloading = true
return window.location.reload() return window.location.reload()
} }
startTransition(() => { startTransition(() => {
...@@ -381,6 +386,8 @@ function processMessage( ...@@ -381,6 +386,8 @@ function processMessage(
clientId: __nextDevClientId, clientId: __nextDevClientId,
}) })
) )
if (reloading) return
reloading = true
return window.location.reload() return window.location.reload()
} }
case HMR_ACTIONS_SENT_TO_BROWSER.REMOVED_PAGE: { case HMR_ACTIONS_SENT_TO_BROWSER.REMOVED_PAGE: {
......
...@@ -3,21 +3,20 @@ import type { NextServer, RequestHandler } from '../next' ...@@ -3,21 +3,20 @@ import type { NextServer, RequestHandler } from '../next'
import next from '../next' import next from '../next'
import { PropagateToWorkersField } from './router-utils/types' import { PropagateToWorkersField } from './router-utils/types'
const result: Record< let initializations: Record<
string, string,
| undefined | Promise<{
| {
requestHandler: ReturnType< requestHandler: ReturnType<
InstanceType<typeof NextServer>['getRequestHandler'] InstanceType<typeof NextServer>['getRequestHandler']
> >
upgradeHandler: ReturnType< upgradeHandler: ReturnType<
InstanceType<typeof NextServer>['getUpgradeHandler'] InstanceType<typeof NextServer>['getUpgradeHandler']
> >
} app: ReturnType<typeof next>
}>
| undefined
> = {} > = {}
let apps: Record<string, ReturnType<typeof next> | undefined> = {}
let sandboxContext: undefined | typeof import('../web/sandbox/context') let sandboxContext: undefined | typeof import('../web/sandbox/context')
let requireCacheHotReloader: let requireCacheHotReloader:
| undefined | undefined
...@@ -47,10 +46,11 @@ export async function propagateServerField( ...@@ -47,10 +46,11 @@ export async function propagateServerField(
field: PropagateToWorkersField, field: PropagateToWorkersField,
value: any value: any
) { ) {
const app = apps[dir] const initialization = await initializations[dir]
if (!app) { if (!initialization) {
throw new Error('Invariant cant propagate server field, no app initialized') throw new Error('Invariant cant propagate server field, no app initialized')
} }
const { app } = initialization
let appField = (app as any).server let appField = (app as any).server
if (appField) { if (appField) {
...@@ -65,7 +65,7 @@ export async function propagateServerField( ...@@ -65,7 +65,7 @@ export async function propagateServerField(
} }
} }
export async function initialize(opts: { async function initializeImpl(opts: {
dir: string dir: string
port: number port: number
dev: boolean dev: boolean
...@@ -80,12 +80,6 @@ export async function initialize(opts: { ...@@ -80,12 +80,6 @@ export async function initialize(opts: {
_ipcPort?: string _ipcPort?: string
_ipcKey?: string _ipcKey?: string
}) { }) {
// if we already setup the server return as we only need to do
// this on first worker boot
if (result[opts.dir]) {
return result[opts.dir]
}
const type = process.env.__NEXT_PRIVATE_RENDER_WORKER const type = process.env.__NEXT_PRIVATE_RENDER_WORKER
if (type) { if (type) {
process.title = 'next-render-worker-' + type process.title = 'next-render-worker-' + type
...@@ -102,15 +96,32 @@ export async function initialize(opts: { ...@@ -102,15 +96,32 @@ export async function initialize(opts: {
port: opts.port, port: opts.port,
isNodeDebugging: opts.isNodeDebugging, isNodeDebugging: opts.isNodeDebugging,
}) })
apps[opts.dir] = app
requestHandler = app.getRequestHandler() requestHandler = app.getRequestHandler()
upgradeHandler = app.getUpgradeHandler() upgradeHandler = app.getUpgradeHandler()
await app.prepare(opts.serverFields) await app.prepare(opts.serverFields)
result[opts.dir] = { return {
requestHandler, requestHandler,
upgradeHandler, upgradeHandler,
app,
}
}
export async function initialize(
opts: Parameters<typeof initializeImpl>[0]
): Promise<{
requestHandler: ReturnType<
InstanceType<typeof NextServer>['getRequestHandler']
>
upgradeHandler: ReturnType<
InstanceType<typeof NextServer>['getUpgradeHandler']
>
}> {
// if we already setup the server return as we only need to do
// this on first worker boot
if (initializations[opts.dir]) {
return initializations[opts.dir]!
} }
return result[opts.dir] return (initializations[opts.dir] = initializeImpl(opts))
} }
...@@ -219,7 +219,10 @@ async function startWatcher(opts: SetupOpts) { ...@@ -219,7 +219,10 @@ async function startWatcher(opts: SetupOpts) {
}) })
const iter = project.entrypointsSubscribe() const iter = project.entrypointsSubscribe()
const curEntries: Map<string, Route> = new Map() const curEntries: Map<string, Route> = new Map()
const changeSubscriptions: Map<string, AsyncIterator<any>> = new Map() const changeSubscriptions: Map<
string,
Promise<AsyncIterator<any>>
> = new Map()
let prevMiddleware: boolean | undefined = undefined let prevMiddleware: boolean | undefined = undefined
const globalEntries: { const globalEntries: {
app: Endpoint | undefined app: Endpoint | undefined
...@@ -290,6 +293,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -290,6 +293,7 @@ async function startWatcher(opts: SetupOpts) {
class ModuleBuildError extends Error {} class ModuleBuildError extends Error {}
function processIssues( function processIssues(
displayName: string,
name: string, name: string,
result: TurbopackResult, result: TurbopackResult,
throwIssue = false throwIssue = false
...@@ -298,25 +302,29 @@ async function startWatcher(opts: SetupOpts) { ...@@ -298,25 +302,29 @@ async function startWatcher(opts: SetupOpts) {
const newSet = new Map<string, Issue>() const newSet = new Map<string, Issue>()
issues.set(name, newSet) issues.set(name, newSet)
const relevantIssues = new Set()
for (const issue of result.issues) { for (const issue of result.issues) {
// TODO better formatting // TODO better formatting
if (issue.severity !== 'error' && issue.severity !== 'fatal') continue if (issue.severity !== 'error' && issue.severity !== 'fatal') continue
const key = issueKey(issue) const key = issueKey(issue)
const formatted = formatIssue(issue) const formatted = formatIssue(issue)
if (!throwIssue && !oldSet.has(key)) { if (!oldSet.has(key) && !newSet.has(key)) {
console.error(` ⚠ ${key} ${formatted}\n\n`) console.error(` ⚠ ${displayName} ${key} ${formatted}\n\n`)
} }
newSet.set(key, issue) newSet.set(key, issue)
if (throwIssue) { relevantIssues.add(formatted)
throw new ModuleBuildError(formatted)
}
} }
for (const issue of oldSet.keys()) { for (const issue of oldSet.keys()) {
if (!newSet.has(issue)) { if (!newSet.has(issue)) {
console.error(`✅ ${name} fixed ${issue}`) console.error(`✅ ${displayName} fixed ${issue}`)
} }
} }
if (relevantIssues.size && throwIssue) {
throw new ModuleBuildError([...relevantIssues].join('\n\n'))
}
} }
async function processResult( async function processResult(
...@@ -503,8 +511,9 @@ async function startWatcher(opts: SetupOpts) { ...@@ -503,8 +511,9 @@ async function startWatcher(opts: SetupOpts) {
) { ) {
if (!endpoint || changeSubscriptions.has(page)) return if (!endpoint || changeSubscriptions.has(page)) return
const changed = await endpoint.changed() const changedPromise = endpoint.changed()
changeSubscriptions.set(page, changed) changeSubscriptions.set(page, changedPromise)
const changed = await changedPromise
for await (const change of changed) { for await (const change of changed) {
consoleStore.setState( consoleStore.setState(
...@@ -515,14 +524,14 @@ async function startWatcher(opts: SetupOpts) { ...@@ -515,14 +524,14 @@ async function startWatcher(opts: SetupOpts) {
true true
) )
processIssues(page, change) processIssues(page, page, change)
const payload = makePayload(page, change) const payload = makePayload(page, change)
if (payload) sendHmr('endpoint-change', page, payload) if (payload) sendHmr('endpoint-change', page, payload)
} }
} }
function clearChangeSubscription(page: string) { async function clearChangeSubscription(page: string) {
const subscription = changeSubscriptions.get(page) const subscription = await changeSubscriptions.get(page)
if (subscription) { if (subscription) {
subscription.return?.() subscription.return?.()
changeSubscriptions.delete(page) changeSubscriptions.delete(page)
...@@ -560,13 +569,14 @@ async function startWatcher(opts: SetupOpts) { ...@@ -560,13 +569,14 @@ async function startWatcher(opts: SetupOpts) {
} }
} }
for (const [pathname, subscription] of changeSubscriptions) { for (const [pathname, subscriptionPromise] of changeSubscriptions) {
if (pathname === '') { if (pathname === '') {
// middleware is handled below // middleware is handled below
continue continue
} }
if (!curEntries.has(pathname)) { if (!curEntries.has(pathname)) {
const subscription = await subscriptionPromise
subscription.return?.() subscription.return?.()
changeSubscriptions.delete(pathname) changeSubscriptions.delete(pathname)
} }
...@@ -578,7 +588,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -578,7 +588,7 @@ async function startWatcher(opts: SetupOpts) {
// unnecessary during the first serve) // unnecessary during the first serve)
if (prevMiddleware === true && !middleware) { if (prevMiddleware === true && !middleware) {
// Went from middleware to no middleware // Went from middleware to no middleware
clearChangeSubscription('middleware') await clearChangeSubscription('middleware')
sendHmr('entrypoint-change', 'middleware', { sendHmr('entrypoint-change', 'middleware', {
event: HMR_ACTIONS_SENT_TO_BROWSER.MIDDLEWARE_CHANGES, event: HMR_ACTIONS_SENT_TO_BROWSER.MIDDLEWARE_CHANGES,
}) })
...@@ -592,7 +602,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -592,7 +602,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult( const writtenEndpoint = await processResult(
await middleware.endpoint.writeToDisk() await middleware.endpoint.writeToDisk()
) )
processIssues('middleware', writtenEndpoint) processIssues('middleware', 'middleware', writtenEndpoint)
await loadMiddlewareManifest('middleware', 'middleware') await loadMiddlewareManifest('middleware', 'middleware')
serverFields.actualMiddlewareFile = 'middleware' serverFields.actualMiddlewareFile = 'middleware'
serverFields.middleware = { serverFields.middleware = {
...@@ -881,7 +891,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -881,7 +891,7 @@ async function startWatcher(opts: SetupOpts) {
} }
for await (const data of subscription) { for await (const data of subscription) {
processIssues(id, data) processIssues('hmr', id, data)
sendTurbopackMessage(data) sendTurbopackMessage(data)
} }
} }
...@@ -1044,7 +1054,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1044,7 +1054,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult( const writtenEndpoint = await processResult(
await globalEntries.app.writeToDisk() await globalEntries.app.writeToDisk()
) )
processIssues('_app', writtenEndpoint) processIssues('_app', '_app', writtenEndpoint)
} }
await loadBuildManifest('_app') await loadBuildManifest('_app')
await loadPagesManifest('_app') await loadPagesManifest('_app')
...@@ -1056,7 +1066,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1056,7 +1066,7 @@ async function startWatcher(opts: SetupOpts) {
changeSubscription('_document', globalEntries.document, () => { changeSubscription('_document', globalEntries.document, () => {
return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE } return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE }
}) })
processIssues('_document', writtenEndpoint) processIssues('_document', '_document', writtenEndpoint)
} }
await loadPagesManifest('_document') await loadPagesManifest('_document')
...@@ -1064,7 +1074,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1064,7 +1074,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult( const writtenEndpoint = await processResult(
await globalEntries.error.writeToDisk() await globalEntries.error.writeToDisk()
) )
processIssues(page, writtenEndpoint) processIssues(page, page, writtenEndpoint)
} }
await loadBuildManifest('_error') await loadBuildManifest('_error')
await loadPagesManifest('_error') await loadPagesManifest('_error')
...@@ -1137,7 +1147,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1137,7 +1147,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult( const writtenEndpoint = await processResult(
await globalEntries.app.writeToDisk() await globalEntries.app.writeToDisk()
) )
processIssues('_app', writtenEndpoint) processIssues('_app', '_app', writtenEndpoint)
} }
await loadBuildManifest('_app') await loadBuildManifest('_app')
await loadPagesManifest('_app') await loadPagesManifest('_app')
...@@ -1150,7 +1160,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1150,7 +1160,7 @@ async function startWatcher(opts: SetupOpts) {
changeSubscription('_document', globalEntries.document, () => { changeSubscription('_document', globalEntries.document, () => {
return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE } return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE }
}) })
processIssues('_document', writtenEndpoint) processIssues('_document', '_document', writtenEndpoint)
} }
await loadPagesManifest('_document') await loadPagesManifest('_document')
...@@ -1186,7 +1196,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1186,7 +1196,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest() await writeMiddlewareManifest()
await writeOtherManifests() await writeOtherManifests()
processIssues(page, writtenEndpoint, true) processIssues(page, page, writtenEndpoint, true)
break break
} }
...@@ -1212,7 +1222,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1212,7 +1222,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest() await writeMiddlewareManifest()
await writeOtherManifests() await writeOtherManifests()
processIssues(page, writtenEndpoint, true) processIssues(page, page, writtenEndpoint, true)
break break
} }
...@@ -1243,7 +1253,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1243,7 +1253,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest() await writeMiddlewareManifest()
await writeOtherManifests() await writeOtherManifests()
processIssues(page, writtenEndpoint, true) processIssues(page, page, writtenEndpoint, true)
break break
} }
...@@ -1267,7 +1277,7 @@ async function startWatcher(opts: SetupOpts) { ...@@ -1267,7 +1277,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest() await writeMiddlewareManifest()
await writeOtherManifests() await writeOtherManifests()
processIssues(page, writtenEndpoint, true) processIssues(page, page, writtenEndpoint, true)
break break
} }
......
...@@ -128,7 +128,9 @@ export class NextInstance { ...@@ -128,7 +128,9 @@ export class NextInstance {
if (this.isDestroyed) { if (this.isDestroyed) {
throw new Error('next instance already destroyed') throw new Error('next instance already destroyed')
} }
require('console').log(`Creating test directory with isolated next...`) require('console').log(
`Creating test directory with isolated next... (use NEXT_SKIP_ISOLATE=1 to opt-out)`
)
await parentSpan await parentSpan
.traceChild('createTestDir') .traceChild('createTestDir')
...@@ -156,7 +158,7 @@ export class NextInstance { ...@@ -156,7 +158,7 @@ export class NextInstance {
...this.packageJson?.dependencies, ...this.packageJson?.dependencies,
} }
if (skipInstall) { if (skipInstall || skipIsolatedNext) {
const pkgScripts = (this.packageJson['scripts'] as {}) || {} const pkgScripts = (this.packageJson['scripts'] as {}) || {}
await fs.ensureDir(this.testDir) await fs.ensureDir(this.testDir)
await fs.writeFile( await fs.writeFile(
...@@ -195,7 +197,7 @@ export class NextInstance { ...@@ -195,7 +197,7 @@ export class NextInstance {
!(global as any).isNextDeploy !(global as any).isNextDeploy
) { ) {
await fs.copy(process.env.NEXT_TEST_STARTER, this.testDir) await fs.copy(process.env.NEXT_TEST_STARTER, this.testDir)
} else if (!skipIsolatedNext) { } else {
this.testDir = await createNextInstall({ this.testDir = await createNextInstall({
parentSpan: rootSpan, parentSpan: rootSpan,
dependencies: finalDependencies, dependencies: finalDependencies,
...@@ -432,7 +434,16 @@ export class NextInstance { ...@@ -432,7 +434,16 @@ export class NextInstance {
public async readJSON(filename: string) { public async readJSON(filename: string) {
return fs.readJSON(path.join(this.testDir, filename)) return fs.readJSON(path.join(this.testDir, filename))
} }
private async handleDevWatchDelay(filename: string) { private async handleDevWatchDelayBeforeChange(filename: string) {
// This is a temporary workaround for turbopack starting watching too late.
// So we delay file changes by 500ms to give it some time
// to connect the WebSocket and start watching.
if (process.env.TURBOPACK) {
require('console').log('fs dev delay before', filename)
await new Promise((resolve) => setTimeout(resolve, 500))
}
}
private async handleDevWatchDelayAfterChange(filename: string) {
// to help alleviate flakiness with tests that create // to help alleviate flakiness with tests that create
// dynamic routes // and then request it we give a buffer // dynamic routes // and then request it we give a buffer
// of 500ms to allow WatchPack to detect the changed files // of 500ms to allow WatchPack to detect the changed files
...@@ -447,32 +458,40 @@ export class NextInstance { ...@@ -447,32 +458,40 @@ export class NextInstance {
} }
} }
public async patchFile(filename: string, content: string) { public async patchFile(filename: string, content: string) {
await this.handleDevWatchDelayBeforeChange(filename)
const outputPath = path.join(this.testDir, filename) const outputPath = path.join(this.testDir, filename)
const newFile = !(await fs.pathExists(outputPath)) const newFile = !(await fs.pathExists(outputPath))
await fs.ensureDir(path.dirname(outputPath)) await fs.ensureDir(path.dirname(outputPath))
await fs.writeFile(outputPath, content) await fs.writeFile(outputPath, content)
if (newFile) { if (newFile) {
await this.handleDevWatchDelay(filename) await this.handleDevWatchDelayAfterChange(filename)
} }
} }
public async renameFile(filename: string, newFilename: string) { public async renameFile(filename: string, newFilename: string) {
await this.handleDevWatchDelayBeforeChange(filename)
await fs.rename( await fs.rename(
path.join(this.testDir, filename), path.join(this.testDir, filename),
path.join(this.testDir, newFilename) path.join(this.testDir, newFilename)
) )
await this.handleDevWatchDelay(filename) await this.handleDevWatchDelayAfterChange(filename)
} }
public async renameFolder(foldername: string, newFoldername: string) { public async renameFolder(foldername: string, newFoldername: string) {
await this.handleDevWatchDelayBeforeChange(foldername)
await fs.move( await fs.move(
path.join(this.testDir, foldername), path.join(this.testDir, foldername),
path.join(this.testDir, newFoldername) path.join(this.testDir, newFoldername)
) )
await this.handleDevWatchDelay(foldername) await this.handleDevWatchDelayAfterChange(foldername)
} }
public async deleteFile(filename: string) { public async deleteFile(filename: string) {
await this.handleDevWatchDelayBeforeChange(filename)
await fs.remove(path.join(this.testDir, filename)) await fs.remove(path.join(this.testDir, filename))
await this.handleDevWatchDelay(filename) await this.handleDevWatchDelayAfterChange(filename)
} }
/** /**
......
...@@ -39,6 +39,13 @@ export class NextDevInstance extends NextInstance { ...@@ -39,6 +39,13 @@ export class NextDevInstance extends NextInstance {
startArgs = this.startCommand.split(' ') startArgs = this.startCommand.split(' ')
} }
if (process.env.NEXT_SKIP_ISOLATE) {
// without isolation yarn can't be used and pnpm must be used instead
if (startArgs[0] === 'yarn') {
startArgs[0] = 'pnpm'
}
}
console.log('running', startArgs.join(' ')) console.log('running', startArgs.join(' '))
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
try { try {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment