Unverified Commit a01ada4e authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Turbopack: Use readFileSync / writeFileSync for manifest writing (#83694)

## What?

Use sync fs methods for reading/writing manifests after a Turbopack
writeToDisk. Next step will be not calling it when nothing changed.

Before:

```
 GET / 200 in 41ms
 GET / 200 in 54ms
 GET / 200 in 57ms
 GET / 200 in 165ms
 GET / 200 in 44ms
 GET / 200 in 52ms
 GET / 200 in 42ms
 GET / 200 in 50ms
 GET / 200 in 39ms
 GET / 200 in 53ms
 GET / 200 in 39ms
 GET / 200 in 49ms
 GET / 200 in 40ms
 GET / 200 in 49ms
 GET / 200 in 40ms
 GET / 200 in 54ms
 GET / 200 in 39ms
 GET / 200 in 52ms
 GET / 200 in 47ms
 GET / 200 in 50ms
 GET / 200 in 45ms
 GET / 200 in 59ms
 GET / 200 in 44ms
 GET / 200 in 55ms
 GET / 200 in 46ms
 GET / 200 in 51ms
```


After:

```
 GET / 200 in 31ms
 GET / 200 in 29ms
 GET / 200 in 32ms
 GET / 200 in 33ms
 GET / 200 in 33ms
 GET / 200 in 34ms
 GET / 200 in 32ms
 GET / 200 in 30ms
 GET / 200 in 33ms
 GET / 200 in 30ms
 GET / 200 in 31ms
 GET / 200 in 32ms
 GET / 200 in 31ms
 GET / 200 in 31ms
 GET / 200 in 30ms
 GET / 200 in 33ms
 GET / 200 in 33ms
 GET / 200 in 32ms
 GET / 200 in 29ms
 GET / 200 in 30ms
 GET / 200 in 36ms
 GET / 200 in 35ms
 GET / 200 in 34ms
 GET / 200 in 32ms
 GET / 200 in 30ms
```
parent 5b4cdc3d
......@@ -109,14 +109,14 @@ export async function handleRouteType({
manifestLoader.deleteMiddlewareManifest(key)
}
await manifestLoader.loadAppBuildManifest(page)
await manifestLoader.loadBuildManifest(page, 'app')
await manifestLoader.loadAppPathsManifest(page)
await manifestLoader.loadActionManifest(page)
await manifestLoader.loadFontManifest(page, 'app')
manifestLoader.loadAppBuildManifest(page)
manifestLoader.loadBuildManifest(page, 'app')
manifestLoader.loadAppPathsManifest(page)
manifestLoader.loadActionManifest(page)
manifestLoader.loadFontManifest(page, 'app')
if (shouldCreateWebpackStats) {
await manifestLoader.loadWebpackStats(page, 'app')
manifestLoader.loadWebpackStats(page, 'app')
}
break
......@@ -124,10 +124,12 @@ export async function handleRouteType({
case 'app-route': {
const key = getEntryKey('app', 'server', page)
await manifestLoader.loadAppPathsManifest(page)
manifestLoader.loadAppPathsManifest(page)
const middlewareManifestWritten =
await manifestLoader.loadMiddlewareManifest(page, 'app')
const middlewareManifestWritten = manifestLoader.loadMiddlewareManifest(
page,
'app'
)
if (!middlewareManifestWritten) {
manifestLoader.deleteMiddlewareManifest(key)
......
......@@ -202,7 +202,7 @@ export async function turbopackBuild(): Promise<{
)),
])
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites: undefined,
productionRewrites: rewrites,
entrypoints: currentEntrypoints,
......
......@@ -25,18 +25,18 @@ SOFTWARE.
// This file is based on https://github.com/microsoft/vscode/blob/f860fcf11022f10a992440fd54c6e45674e39617/src/vs/base/node/pfs.ts
// See the LICENSE at the top of the file
import { rename as fsRename, stat } from 'node:fs/promises'
import { renameSync as fsRenameSync, statSync } from 'node:fs'
/**
* A drop-in replacement for `fs.rename` that:
* - allows to move across multiple disks
* - attempts to retry the operation for certain error codes on Windows
*/
export async function rename(
export function renameSync(
source: string,
target: string,
windowsRetryTimeout: number | false = 60000 /* matches graceful-fs */
): Promise<void> {
): void {
if (source === target) {
return // simulate node.js behaviour here and do a no-op if paths match
}
......@@ -46,21 +46,21 @@ export async function rename(
// is locked by AV software. We do leverage graceful-fs to iron
// out these issues, however in case the target file exists,
// graceful-fs will immediately return without retry for fs.rename().
await renameWithRetry(source, target, Date.now(), windowsRetryTimeout)
renameSyncWithRetry(source, target, Date.now(), windowsRetryTimeout)
} else {
await fsRename(source, target)
fsRenameSync(source, target)
}
}
async function renameWithRetry(
function renameSyncWithRetry(
source: string,
target: string,
startTime: number,
retryTimeout: number,
attempt = 0
): Promise<void> {
): void {
try {
return await fsRename(source, target)
return fsRenameSync(source, target)
} catch (error: any) {
if (
error.code !== 'EACCES' &&
......@@ -72,16 +72,23 @@ async function renameWithRetry(
if (Date.now() - startTime >= retryTimeout) {
console.error(
`[node.js fs] rename failed after ${attempt} retries with error: ${error}`
`Node.js fs rename failed after ${attempt} retries with error: ${error}`
)
throw error // give up after configurable timeout
}
if (attempt > 100) {
console.error(
`Node.js fs rename failed after ${attempt} retries with error ${error}`
)
throw error
}
if (attempt === 0) {
let abortRetry = false
try {
const statTarget = await stat(target)
const statTarget = statSync(target)
if (!statTarget.isFile()) {
abortRetry = true // if target is not a file, EPERM error may be raised and we should not attempt to retry
}
......@@ -94,13 +101,13 @@ async function renameWithRetry(
}
}
// Delay with incremental backoff up to 100ms
await timeout(Math.min(100, attempt * 10))
// Attempt again
return renameWithRetry(source, target, startTime, retryTimeout, attempt + 1)
return renameSyncWithRetry(
source,
target,
startTime,
retryTimeout,
attempt + 1
)
}
}
const timeout = (millis: number) =>
new Promise((resolve) => setTimeout(resolve, millis))
import { unlink, writeFile } from 'fs/promises'
import { rename } from './rename'
import { unlinkSync, writeFileSync } from 'fs'
import { renameSync } from './rename'
export async function writeFileAtomic(
filePath: string,
content: string
): Promise<void> {
export function writeFileAtomic(filePath: string, content: string): void {
const tempPath = filePath + '.tmp.' + Math.random().toString(36).slice(2)
try {
await writeFile(tempPath, content, 'utf-8')
await rename(tempPath, filePath)
writeFileSync(tempPath, content, 'utf-8')
renameSync(tempPath, filePath)
} catch (e) {
try {
await unlink(tempPath)
unlinkSync(tempPath)
} catch {
// ignore
}
......
......@@ -243,7 +243,7 @@ export async function handleRouteType({
await manifestLoader.loadWebpackStats(page, 'pages')
}
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites,
entrypoints,
......@@ -334,7 +334,7 @@ export async function handleRouteType({
manifestLoader.deleteMiddlewareManifest(key)
}
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites,
entrypoints,
......@@ -382,22 +382,22 @@ export async function handleRouteType({
const type = writtenEndpoint.type
if (type === 'edge') {
await manifestLoader.loadMiddlewareManifest(page, 'app')
manifestLoader.loadMiddlewareManifest(page, 'app')
} else {
manifestLoader.deleteMiddlewareManifest(key)
}
await manifestLoader.loadAppBuildManifest(page)
await manifestLoader.loadBuildManifest(page, 'app')
await manifestLoader.loadAppPathsManifest(page)
await manifestLoader.loadActionManifest(page)
await manifestLoader.loadFontManifest(page, 'app')
manifestLoader.loadAppBuildManifest(page)
manifestLoader.loadBuildManifest(page, 'app')
manifestLoader.loadAppPathsManifest(page)
manifestLoader.loadActionManifest(page)
manifestLoader.loadFontManifest(page, 'app')
if (shouldCreateWebpackStats) {
await manifestLoader.loadWebpackStats(page, 'app')
manifestLoader.loadWebpackStats(page, 'app')
}
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites,
entrypoints,
......@@ -415,15 +415,15 @@ export async function handleRouteType({
const type = writtenEndpoint.type
await manifestLoader.loadAppPathsManifest(page)
manifestLoader.loadAppPathsManifest(page)
if (type === 'edge') {
await manifestLoader.loadMiddlewareManifest(page, 'app')
manifestLoader.loadMiddlewareManifest(page, 'app')
} else {
manifestLoader.deleteMiddlewareManifest(key)
}
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites,
entrypoints,
......@@ -690,7 +690,7 @@ export async function handleEntrypoints({
'instrumentation',
'instrumentation'
)
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites: undefined,
entrypoints: currentEntrypoints,
......@@ -758,7 +758,7 @@ export async function handleEntrypoints({
'middleware',
dev.serverFields.middleware
)
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites: undefined,
entrypoints: currentEntrypoints,
......@@ -956,7 +956,7 @@ export async function handlePagesErrorRoute({
await manifestLoader.loadPagesManifest('_error')
await manifestLoader.loadFontManifest('_error')
await manifestLoader.writeManifests({
manifestLoader.writeManifests({
devRewrites,
productionRewrites,
entrypoints,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment