Unverified Commit a01ada4e authored by Tim Neutkens's avatar Tim Neutkens Committed by GitHub

Turbopack: Use readFileSync / writeFileSync for manifest writing (#83694)

## What?

Use sync fs methods for reading/writing manifests after a Turbopack
writeToDisk. Next step will be not calling it when nothing changed.

Before:

```
 GET / 200 in 41ms
 GET / 200 in 54ms
 GET / 200 in 57ms
 GET / 200 in 165ms
 GET / 200 in 44ms
 GET / 200 in 52ms
 GET / 200 in 42ms
 GET / 200 in 50ms
 GET / 200 in 39ms
 GET / 200 in 53ms
 GET / 200 in 39ms
 GET / 200 in 49ms
 GET / 200 in 40ms
 GET / 200 in 49ms
 GET / 200 in 40ms
 GET / 200 in 54ms
 GET / 200 in 39ms
 GET / 200 in 52ms
 GET / 200 in 47ms
 GET / 200 in 50ms
 GET / 200 in 45ms
 GET / 200 in 59ms
 GET / 200 in 44ms
 GET / 200 in 55ms
 GET / 200 in 46ms
 GET / 200 in 51ms
```


After:

```
 GET / 200 in 31ms
 GET / 200 in 29ms
 GET / 200 in 32ms
 GET / 200 in 33ms
 GET / 200 in 33ms
 GET / 200 in 34ms
 GET / 200 in 32ms
 GET / 200 in 30ms
 GET / 200 in 33ms
 GET / 200 in 30ms
 GET / 200 in 31ms
 GET / 200 in 32ms
 GET / 200 in 31ms
 GET / 200 in 31ms
 GET / 200 in 30ms
 GET / 200 in 33ms
 GET / 200 in 33ms
 GET / 200 in 32ms
 GET / 200 in 29ms
 GET / 200 in 30ms
 GET / 200 in 36ms
 GET / 200 in 35ms
 GET / 200 in 34ms
 GET / 200 in 32ms
 GET / 200 in 30ms
```
parent 5b4cdc3d
...@@ -109,14 +109,14 @@ export async function handleRouteType({ ...@@ -109,14 +109,14 @@ export async function handleRouteType({
manifestLoader.deleteMiddlewareManifest(key) manifestLoader.deleteMiddlewareManifest(key)
} }
await manifestLoader.loadAppBuildManifest(page) manifestLoader.loadAppBuildManifest(page)
await manifestLoader.loadBuildManifest(page, 'app') manifestLoader.loadBuildManifest(page, 'app')
await manifestLoader.loadAppPathsManifest(page) manifestLoader.loadAppPathsManifest(page)
await manifestLoader.loadActionManifest(page) manifestLoader.loadActionManifest(page)
await manifestLoader.loadFontManifest(page, 'app') manifestLoader.loadFontManifest(page, 'app')
if (shouldCreateWebpackStats) { if (shouldCreateWebpackStats) {
await manifestLoader.loadWebpackStats(page, 'app') manifestLoader.loadWebpackStats(page, 'app')
} }
break break
...@@ -124,10 +124,12 @@ export async function handleRouteType({ ...@@ -124,10 +124,12 @@ export async function handleRouteType({
case 'app-route': { case 'app-route': {
const key = getEntryKey('app', 'server', page) const key = getEntryKey('app', 'server', page)
await manifestLoader.loadAppPathsManifest(page) manifestLoader.loadAppPathsManifest(page)
const middlewareManifestWritten = const middlewareManifestWritten = manifestLoader.loadMiddlewareManifest(
await manifestLoader.loadMiddlewareManifest(page, 'app') page,
'app'
)
if (!middlewareManifestWritten) { if (!middlewareManifestWritten) {
manifestLoader.deleteMiddlewareManifest(key) manifestLoader.deleteMiddlewareManifest(key)
......
...@@ -202,7 +202,7 @@ export async function turbopackBuild(): Promise<{ ...@@ -202,7 +202,7 @@ export async function turbopackBuild(): Promise<{
)), )),
]) ])
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites: undefined, devRewrites: undefined,
productionRewrites: rewrites, productionRewrites: rewrites,
entrypoints: currentEntrypoints, entrypoints: currentEntrypoints,
......
...@@ -25,18 +25,18 @@ SOFTWARE. ...@@ -25,18 +25,18 @@ SOFTWARE.
// This file is based on https://github.com/microsoft/vscode/blob/f860fcf11022f10a992440fd54c6e45674e39617/src/vs/base/node/pfs.ts // This file is based on https://github.com/microsoft/vscode/blob/f860fcf11022f10a992440fd54c6e45674e39617/src/vs/base/node/pfs.ts
// See the LICENSE at the top of the file // See the LICENSE at the top of the file
import { rename as fsRename, stat } from 'node:fs/promises' import { renameSync as fsRenameSync, statSync } from 'node:fs'
/** /**
* A drop-in replacement for `fs.rename` that: * A drop-in replacement for `fs.rename` that:
* - allows to move across multiple disks * - allows to move across multiple disks
* - attempts to retry the operation for certain error codes on Windows * - attempts to retry the operation for certain error codes on Windows
*/ */
export async function rename( export function renameSync(
source: string, source: string,
target: string, target: string,
windowsRetryTimeout: number | false = 60000 /* matches graceful-fs */ windowsRetryTimeout: number | false = 60000 /* matches graceful-fs */
): Promise<void> { ): void {
if (source === target) { if (source === target) {
return // simulate node.js behaviour here and do a no-op if paths match return // simulate node.js behaviour here and do a no-op if paths match
} }
...@@ -46,21 +46,21 @@ export async function rename( ...@@ -46,21 +46,21 @@ export async function rename(
// is locked by AV software. We do leverage graceful-fs to iron // is locked by AV software. We do leverage graceful-fs to iron
// out these issues, however in case the target file exists, // out these issues, however in case the target file exists,
// graceful-fs will immediately return without retry for fs.rename(). // graceful-fs will immediately return without retry for fs.rename().
await renameWithRetry(source, target, Date.now(), windowsRetryTimeout) renameSyncWithRetry(source, target, Date.now(), windowsRetryTimeout)
} else { } else {
await fsRename(source, target) fsRenameSync(source, target)
} }
} }
async function renameWithRetry( function renameSyncWithRetry(
source: string, source: string,
target: string, target: string,
startTime: number, startTime: number,
retryTimeout: number, retryTimeout: number,
attempt = 0 attempt = 0
): Promise<void> { ): void {
try { try {
return await fsRename(source, target) return fsRenameSync(source, target)
} catch (error: any) { } catch (error: any) {
if ( if (
error.code !== 'EACCES' && error.code !== 'EACCES' &&
...@@ -72,16 +72,23 @@ async function renameWithRetry( ...@@ -72,16 +72,23 @@ async function renameWithRetry(
if (Date.now() - startTime >= retryTimeout) { if (Date.now() - startTime >= retryTimeout) {
console.error( console.error(
`[node.js fs] rename failed after ${attempt} retries with error: ${error}` `Node.js fs rename failed after ${attempt} retries with error: ${error}`
) )
throw error // give up after configurable timeout throw error // give up after configurable timeout
} }
if (attempt > 100) {
console.error(
`Node.js fs rename failed after ${attempt} retries with error ${error}`
)
throw error
}
if (attempt === 0) { if (attempt === 0) {
let abortRetry = false let abortRetry = false
try { try {
const statTarget = await stat(target) const statTarget = statSync(target)
if (!statTarget.isFile()) { if (!statTarget.isFile()) {
abortRetry = true // if target is not a file, EPERM error may be raised and we should not attempt to retry abortRetry = true // if target is not a file, EPERM error may be raised and we should not attempt to retry
} }
...@@ -94,13 +101,13 @@ async function renameWithRetry( ...@@ -94,13 +101,13 @@ async function renameWithRetry(
} }
} }
// Delay with incremental backoff up to 100ms
await timeout(Math.min(100, attempt * 10))
// Attempt again // Attempt again
return renameWithRetry(source, target, startTime, retryTimeout, attempt + 1) return renameSyncWithRetry(
source,
target,
startTime,
retryTimeout,
attempt + 1
)
} }
} }
const timeout = (millis: number) =>
new Promise((resolve) => setTimeout(resolve, millis))
import { unlink, writeFile } from 'fs/promises' import { unlinkSync, writeFileSync } from 'fs'
import { rename } from './rename' import { renameSync } from './rename'
export async function writeFileAtomic( export function writeFileAtomic(filePath: string, content: string): void {
filePath: string,
content: string
): Promise<void> {
const tempPath = filePath + '.tmp.' + Math.random().toString(36).slice(2) const tempPath = filePath + '.tmp.' + Math.random().toString(36).slice(2)
try { try {
await writeFile(tempPath, content, 'utf-8') writeFileSync(tempPath, content, 'utf-8')
await rename(tempPath, filePath) renameSync(tempPath, filePath)
} catch (e) { } catch (e) {
try { try {
await unlink(tempPath) unlinkSync(tempPath)
} catch { } catch {
// ignore // ignore
} }
......
...@@ -243,7 +243,7 @@ export async function handleRouteType({ ...@@ -243,7 +243,7 @@ export async function handleRouteType({
await manifestLoader.loadWebpackStats(page, 'pages') await manifestLoader.loadWebpackStats(page, 'pages')
} }
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites, productionRewrites,
entrypoints, entrypoints,
...@@ -334,7 +334,7 @@ export async function handleRouteType({ ...@@ -334,7 +334,7 @@ export async function handleRouteType({
manifestLoader.deleteMiddlewareManifest(key) manifestLoader.deleteMiddlewareManifest(key)
} }
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites, productionRewrites,
entrypoints, entrypoints,
...@@ -382,22 +382,22 @@ export async function handleRouteType({ ...@@ -382,22 +382,22 @@ export async function handleRouteType({
const type = writtenEndpoint.type const type = writtenEndpoint.type
if (type === 'edge') { if (type === 'edge') {
await manifestLoader.loadMiddlewareManifest(page, 'app') manifestLoader.loadMiddlewareManifest(page, 'app')
} else { } else {
manifestLoader.deleteMiddlewareManifest(key) manifestLoader.deleteMiddlewareManifest(key)
} }
await manifestLoader.loadAppBuildManifest(page) manifestLoader.loadAppBuildManifest(page)
await manifestLoader.loadBuildManifest(page, 'app') manifestLoader.loadBuildManifest(page, 'app')
await manifestLoader.loadAppPathsManifest(page) manifestLoader.loadAppPathsManifest(page)
await manifestLoader.loadActionManifest(page) manifestLoader.loadActionManifest(page)
await manifestLoader.loadFontManifest(page, 'app') manifestLoader.loadFontManifest(page, 'app')
if (shouldCreateWebpackStats) { if (shouldCreateWebpackStats) {
await manifestLoader.loadWebpackStats(page, 'app') manifestLoader.loadWebpackStats(page, 'app')
} }
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites, productionRewrites,
entrypoints, entrypoints,
...@@ -415,15 +415,15 @@ export async function handleRouteType({ ...@@ -415,15 +415,15 @@ export async function handleRouteType({
const type = writtenEndpoint.type const type = writtenEndpoint.type
await manifestLoader.loadAppPathsManifest(page) manifestLoader.loadAppPathsManifest(page)
if (type === 'edge') { if (type === 'edge') {
await manifestLoader.loadMiddlewareManifest(page, 'app') manifestLoader.loadMiddlewareManifest(page, 'app')
} else { } else {
manifestLoader.deleteMiddlewareManifest(key) manifestLoader.deleteMiddlewareManifest(key)
} }
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites, productionRewrites,
entrypoints, entrypoints,
...@@ -690,7 +690,7 @@ export async function handleEntrypoints({ ...@@ -690,7 +690,7 @@ export async function handleEntrypoints({
'instrumentation', 'instrumentation',
'instrumentation' 'instrumentation'
) )
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites: undefined, productionRewrites: undefined,
entrypoints: currentEntrypoints, entrypoints: currentEntrypoints,
...@@ -758,7 +758,7 @@ export async function handleEntrypoints({ ...@@ -758,7 +758,7 @@ export async function handleEntrypoints({
'middleware', 'middleware',
dev.serverFields.middleware dev.serverFields.middleware
) )
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites: undefined, productionRewrites: undefined,
entrypoints: currentEntrypoints, entrypoints: currentEntrypoints,
...@@ -956,7 +956,7 @@ export async function handlePagesErrorRoute({ ...@@ -956,7 +956,7 @@ export async function handlePagesErrorRoute({
await manifestLoader.loadPagesManifest('_error') await manifestLoader.loadPagesManifest('_error')
await manifestLoader.loadFontManifest('_error') await manifestLoader.loadFontManifest('_error')
await manifestLoader.writeManifests({ manifestLoader.writeManifests({
devRewrites, devRewrites,
productionRewrites, productionRewrites,
entrypoints, entrypoints,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment