Unverified Commit 493036df authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

turbopack: improve turbopack/test stability (#56024)

### What?

a few cherry picked fixes to test execution, issue reporting and hmr
behavior

### Why?

### How?


Closes WEB-1657
parent 7102e297
......@@ -51,6 +51,7 @@ interface Dispatcher {
let mostRecentCompilationHash: any = null
let __nextDevClientId = Math.round(Math.random() * 100 + Date.now())
let reloading = false
function onBeforeFastRefresh(dispatcher: Dispatcher, hasUpdates: boolean) {
if (hasUpdates) {
......@@ -115,6 +116,8 @@ function performFullReload(err: any, sendMessage: any) {
})
)
if (reloading) return
reloading = true
window.location.reload()
}
......@@ -357,6 +360,8 @@ function processMessage(
})
)
if (RuntimeErrorHandler.hadRuntimeError) {
if (reloading) return
reloading = true
return window.location.reload()
}
startTransition(() => {
......@@ -381,6 +386,8 @@ function processMessage(
clientId: __nextDevClientId,
})
)
if (reloading) return
reloading = true
return window.location.reload()
}
case HMR_ACTIONS_SENT_TO_BROWSER.REMOVED_PAGE: {
......
......@@ -3,21 +3,20 @@ import type { NextServer, RequestHandler } from '../next'
import next from '../next'
import { PropagateToWorkersField } from './router-utils/types'
const result: Record<
let initializations: Record<
string,
| undefined
| {
| Promise<{
requestHandler: ReturnType<
InstanceType<typeof NextServer>['getRequestHandler']
>
upgradeHandler: ReturnType<
InstanceType<typeof NextServer>['getUpgradeHandler']
>
}
app: ReturnType<typeof next>
}>
| undefined
> = {}
let apps: Record<string, ReturnType<typeof next> | undefined> = {}
let sandboxContext: undefined | typeof import('../web/sandbox/context')
let requireCacheHotReloader:
| undefined
......@@ -47,10 +46,11 @@ export async function propagateServerField(
field: PropagateToWorkersField,
value: any
) {
const app = apps[dir]
if (!app) {
const initialization = await initializations[dir]
if (!initialization) {
throw new Error('Invariant cant propagate server field, no app initialized')
}
const { app } = initialization
let appField = (app as any).server
if (appField) {
......@@ -65,7 +65,7 @@ export async function propagateServerField(
}
}
export async function initialize(opts: {
async function initializeImpl(opts: {
dir: string
port: number
dev: boolean
......@@ -80,12 +80,6 @@ export async function initialize(opts: {
_ipcPort?: string
_ipcKey?: string
}) {
// if we already setup the server return as we only need to do
// this on first worker boot
if (result[opts.dir]) {
return result[opts.dir]
}
const type = process.env.__NEXT_PRIVATE_RENDER_WORKER
if (type) {
process.title = 'next-render-worker-' + type
......@@ -102,15 +96,32 @@ export async function initialize(opts: {
port: opts.port,
isNodeDebugging: opts.isNodeDebugging,
})
apps[opts.dir] = app
requestHandler = app.getRequestHandler()
upgradeHandler = app.getUpgradeHandler()
await app.prepare(opts.serverFields)
result[opts.dir] = {
return {
requestHandler,
upgradeHandler,
app,
}
}
export async function initialize(
opts: Parameters<typeof initializeImpl>[0]
): Promise<{
requestHandler: ReturnType<
InstanceType<typeof NextServer>['getRequestHandler']
>
upgradeHandler: ReturnType<
InstanceType<typeof NextServer>['getUpgradeHandler']
>
}> {
// if we already setup the server return as we only need to do
// this on first worker boot
if (initializations[opts.dir]) {
return initializations[opts.dir]!
}
return result[opts.dir]
return (initializations[opts.dir] = initializeImpl(opts))
}
......@@ -219,7 +219,10 @@ async function startWatcher(opts: SetupOpts) {
})
const iter = project.entrypointsSubscribe()
const curEntries: Map<string, Route> = new Map()
const changeSubscriptions: Map<string, AsyncIterator<any>> = new Map()
const changeSubscriptions: Map<
string,
Promise<AsyncIterator<any>>
> = new Map()
let prevMiddleware: boolean | undefined = undefined
const globalEntries: {
app: Endpoint | undefined
......@@ -290,6 +293,7 @@ async function startWatcher(opts: SetupOpts) {
class ModuleBuildError extends Error {}
function processIssues(
displayName: string,
name: string,
result: TurbopackResult,
throwIssue = false
......@@ -298,25 +302,29 @@ async function startWatcher(opts: SetupOpts) {
const newSet = new Map<string, Issue>()
issues.set(name, newSet)
const relevantIssues = new Set()
for (const issue of result.issues) {
// TODO better formatting
if (issue.severity !== 'error' && issue.severity !== 'fatal') continue
const key = issueKey(issue)
const formatted = formatIssue(issue)
if (!throwIssue && !oldSet.has(key)) {
console.error(` ⚠ ${key} ${formatted}\n\n`)
if (!oldSet.has(key) && !newSet.has(key)) {
console.error(` ⚠ ${displayName} ${key} ${formatted}\n\n`)
}
newSet.set(key, issue)
if (throwIssue) {
throw new ModuleBuildError(formatted)
}
relevantIssues.add(formatted)
}
for (const issue of oldSet.keys()) {
if (!newSet.has(issue)) {
console.error(`✅ ${name} fixed ${issue}`)
console.error(`✅ ${displayName} fixed ${issue}`)
}
}
if (relevantIssues.size && throwIssue) {
throw new ModuleBuildError([...relevantIssues].join('\n\n'))
}
}
async function processResult(
......@@ -503,8 +511,9 @@ async function startWatcher(opts: SetupOpts) {
) {
if (!endpoint || changeSubscriptions.has(page)) return
const changed = await endpoint.changed()
changeSubscriptions.set(page, changed)
const changedPromise = endpoint.changed()
changeSubscriptions.set(page, changedPromise)
const changed = await changedPromise
for await (const change of changed) {
consoleStore.setState(
......@@ -515,14 +524,14 @@ async function startWatcher(opts: SetupOpts) {
true
)
processIssues(page, change)
processIssues(page, page, change)
const payload = makePayload(page, change)
if (payload) sendHmr('endpoint-change', page, payload)
}
}
function clearChangeSubscription(page: string) {
const subscription = changeSubscriptions.get(page)
async function clearChangeSubscription(page: string) {
const subscription = await changeSubscriptions.get(page)
if (subscription) {
subscription.return?.()
changeSubscriptions.delete(page)
......@@ -560,13 +569,14 @@ async function startWatcher(opts: SetupOpts) {
}
}
for (const [pathname, subscription] of changeSubscriptions) {
for (const [pathname, subscriptionPromise] of changeSubscriptions) {
if (pathname === '') {
// middleware is handled below
continue
}
if (!curEntries.has(pathname)) {
const subscription = await subscriptionPromise
subscription.return?.()
changeSubscriptions.delete(pathname)
}
......@@ -578,7 +588,7 @@ async function startWatcher(opts: SetupOpts) {
// unnecessary during the first serve)
if (prevMiddleware === true && !middleware) {
// Went from middleware to no middleware
clearChangeSubscription('middleware')
await clearChangeSubscription('middleware')
sendHmr('entrypoint-change', 'middleware', {
event: HMR_ACTIONS_SENT_TO_BROWSER.MIDDLEWARE_CHANGES,
})
......@@ -592,7 +602,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult(
await middleware.endpoint.writeToDisk()
)
processIssues('middleware', writtenEndpoint)
processIssues('middleware', 'middleware', writtenEndpoint)
await loadMiddlewareManifest('middleware', 'middleware')
serverFields.actualMiddlewareFile = 'middleware'
serverFields.middleware = {
......@@ -881,7 +891,7 @@ async function startWatcher(opts: SetupOpts) {
}
for await (const data of subscription) {
processIssues(id, data)
processIssues('hmr', id, data)
sendTurbopackMessage(data)
}
}
......@@ -1044,7 +1054,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult(
await globalEntries.app.writeToDisk()
)
processIssues('_app', writtenEndpoint)
processIssues('_app', '_app', writtenEndpoint)
}
await loadBuildManifest('_app')
await loadPagesManifest('_app')
......@@ -1056,7 +1066,7 @@ async function startWatcher(opts: SetupOpts) {
changeSubscription('_document', globalEntries.document, () => {
return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE }
})
processIssues('_document', writtenEndpoint)
processIssues('_document', '_document', writtenEndpoint)
}
await loadPagesManifest('_document')
......@@ -1064,7 +1074,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult(
await globalEntries.error.writeToDisk()
)
processIssues(page, writtenEndpoint)
processIssues(page, page, writtenEndpoint)
}
await loadBuildManifest('_error')
await loadPagesManifest('_error')
......@@ -1137,7 +1147,7 @@ async function startWatcher(opts: SetupOpts) {
const writtenEndpoint = await processResult(
await globalEntries.app.writeToDisk()
)
processIssues('_app', writtenEndpoint)
processIssues('_app', '_app', writtenEndpoint)
}
await loadBuildManifest('_app')
await loadPagesManifest('_app')
......@@ -1150,7 +1160,7 @@ async function startWatcher(opts: SetupOpts) {
changeSubscription('_document', globalEntries.document, () => {
return { action: HMR_ACTIONS_SENT_TO_BROWSER.RELOAD_PAGE }
})
processIssues('_document', writtenEndpoint)
processIssues('_document', '_document', writtenEndpoint)
}
await loadPagesManifest('_document')
......@@ -1186,7 +1196,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest()
await writeOtherManifests()
processIssues(page, writtenEndpoint, true)
processIssues(page, page, writtenEndpoint, true)
break
}
......@@ -1212,7 +1222,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest()
await writeOtherManifests()
processIssues(page, writtenEndpoint, true)
processIssues(page, page, writtenEndpoint, true)
break
}
......@@ -1243,7 +1253,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest()
await writeOtherManifests()
processIssues(page, writtenEndpoint, true)
processIssues(page, page, writtenEndpoint, true)
break
}
......@@ -1267,7 +1277,7 @@ async function startWatcher(opts: SetupOpts) {
await writeMiddlewareManifest()
await writeOtherManifests()
processIssues(page, writtenEndpoint, true)
processIssues(page, page, writtenEndpoint, true)
break
}
......
......@@ -128,7 +128,9 @@ export class NextInstance {
if (this.isDestroyed) {
throw new Error('next instance already destroyed')
}
require('console').log(`Creating test directory with isolated next...`)
require('console').log(
`Creating test directory with isolated next... (use NEXT_SKIP_ISOLATE=1 to opt-out)`
)
await parentSpan
.traceChild('createTestDir')
......@@ -156,7 +158,7 @@ export class NextInstance {
...this.packageJson?.dependencies,
}
if (skipInstall) {
if (skipInstall || skipIsolatedNext) {
const pkgScripts = (this.packageJson['scripts'] as {}) || {}
await fs.ensureDir(this.testDir)
await fs.writeFile(
......@@ -195,7 +197,7 @@ export class NextInstance {
!(global as any).isNextDeploy
) {
await fs.copy(process.env.NEXT_TEST_STARTER, this.testDir)
} else if (!skipIsolatedNext) {
} else {
this.testDir = await createNextInstall({
parentSpan: rootSpan,
dependencies: finalDependencies,
......@@ -432,7 +434,16 @@ export class NextInstance {
public async readJSON(filename: string) {
return fs.readJSON(path.join(this.testDir, filename))
}
private async handleDevWatchDelay(filename: string) {
private async handleDevWatchDelayBeforeChange(filename: string) {
// This is a temporary workaround for turbopack starting watching too late.
// So we delay file changes by 500ms to give it some time
// to connect the WebSocket and start watching.
if (process.env.TURBOPACK) {
require('console').log('fs dev delay before', filename)
await new Promise((resolve) => setTimeout(resolve, 500))
}
}
private async handleDevWatchDelayAfterChange(filename: string) {
// to help alleviate flakiness with tests that create
// dynamic routes // and then request it we give a buffer
// of 500ms to allow WatchPack to detect the changed files
......@@ -447,32 +458,40 @@ export class NextInstance {
}
}
public async patchFile(filename: string, content: string) {
await this.handleDevWatchDelayBeforeChange(filename)
const outputPath = path.join(this.testDir, filename)
const newFile = !(await fs.pathExists(outputPath))
await fs.ensureDir(path.dirname(outputPath))
await fs.writeFile(outputPath, content)
if (newFile) {
await this.handleDevWatchDelay(filename)
await this.handleDevWatchDelayAfterChange(filename)
}
}
public async renameFile(filename: string, newFilename: string) {
await this.handleDevWatchDelayBeforeChange(filename)
await fs.rename(
path.join(this.testDir, filename),
path.join(this.testDir, newFilename)
)
await this.handleDevWatchDelay(filename)
await this.handleDevWatchDelayAfterChange(filename)
}
public async renameFolder(foldername: string, newFoldername: string) {
await this.handleDevWatchDelayBeforeChange(foldername)
await fs.move(
path.join(this.testDir, foldername),
path.join(this.testDir, newFoldername)
)
await this.handleDevWatchDelay(foldername)
await this.handleDevWatchDelayAfterChange(foldername)
}
public async deleteFile(filename: string) {
await this.handleDevWatchDelayBeforeChange(filename)
await fs.remove(path.join(this.testDir, filename))
await this.handleDevWatchDelay(filename)
await this.handleDevWatchDelayAfterChange(filename)
}
/**
......
......@@ -39,6 +39,13 @@ export class NextDevInstance extends NextInstance {
startArgs = this.startCommand.split(' ')
}
if (process.env.NEXT_SKIP_ISOLATE) {
// without isolation yarn can't be used and pnpm must be used instead
if (startArgs[0] === 'yarn') {
startArgs[0] = 'pnpm'
}
}
console.log('running', startArgs.join(' '))
await new Promise<void>((resolve, reject) => {
try {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment