Unverified Commit 488fc1c0 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

clear require cache only when needed (#56198)

### What?

clear require cache only when there has been changes

Before we cleared the require.cache after every ensurePage call. This is
too much. The new approach compares the hashes of all emitted files with
the previous hashes and only clears require.cache when they differ.

### Why?

reloading a page and client navigation is slow due the re-requiring
server files


Closes WEB-1686
parent 2cf8e6c8
use std::ops::Deref;
use napi::{bindgen_prelude::External, JsFunction};
use next_api::route::{Endpoint, WrittenEndpoint};
use next_api::{
route::{Endpoint, WrittenEndpoint},
server_paths::ServerPath,
};
use turbo_tasks::Vc;
use turbopack_binding::turbopack::core::error::PrettyPrintError;
......@@ -14,12 +17,28 @@ use super::utils::{
#[derive(Default)]
pub struct NapiEndpointConfig {}
#[napi(object)]
#[derive(Default)]
pub struct NapiServerPath {
pub path: String,
pub content_hash: String,
}
impl From<&ServerPath> for NapiServerPath {
fn from(server_path: &ServerPath) -> Self {
Self {
path: server_path.path.clone(),
content_hash: format!("{:x}", server_path.content_hash),
}
}
}
#[napi(object)]
#[derive(Default)]
pub struct NapiWrittenEndpoint {
pub r#type: String,
pub entry_path: Option<String>,
pub server_paths: Option<Vec<String>>,
pub server_paths: Option<Vec<NapiServerPath>>,
pub files: Option<Vec<String>>,
pub global_var_name: Option<String>,
pub config: NapiEndpointConfig,
......@@ -34,7 +53,7 @@ impl From<&WrittenEndpoint> for NapiWrittenEndpoint {
} => Self {
r#type: "nodejs".to_string(),
entry_path: Some(server_entry_path.clone()),
server_paths: Some(server_paths.clone()),
server_paths: Some(server_paths.iter().map(From::from).collect()),
..Default::default()
},
WrittenEndpoint::Edge {
......@@ -44,7 +63,7 @@ impl From<&WrittenEndpoint> for NapiWrittenEndpoint {
} => Self {
r#type: "edge".to_string(),
files: Some(files.clone()),
server_paths: Some(server_paths.clone()),
server_paths: Some(server_paths.iter().map(From::from).collect()),
global_var_name: Some(global_var_name.clone()),
..Default::default()
},
......
use anyhow::{bail, Context, Result};
use next_core::{
all_server_paths,
app_structure::{
get_entrypoints, Entrypoint as AppEntrypoint, Entrypoints as AppEntrypoints, LoaderTree,
MetadataItem,
......@@ -55,6 +54,7 @@ use turbopack_binding::{
use crate::{
project::Project,
route::{Endpoint, Route, Routes, WrittenEndpoint},
server_paths::all_server_paths,
};
#[turbo_tasks::value]
......
......@@ -8,6 +8,7 @@ mod middleware;
mod pages;
pub mod project;
pub mod route;
pub mod server_paths;
mod versioned_content_map;
// Declare build-time information variables generated in build.rs
......
use anyhow::{bail, Context, Result};
use next_core::{
all_server_paths,
middleware::get_middleware_module,
mode::NextMode,
next_edge::entry::wrap_edge_entry,
......@@ -27,6 +26,7 @@ use turbopack_binding::{
use crate::{
project::Project,
route::{Endpoint, WrittenEndpoint},
server_paths::all_server_paths,
};
#[turbo_tasks::value]
......
use anyhow::{bail, Context, Result};
use indexmap::IndexMap;
use next_core::{
all_server_paths, create_page_loader_entry_module, get_asset_path_from_pathname,
create_page_loader_entry_module, get_asset_path_from_pathname,
get_edge_resolve_options_context,
mode::NextMode,
next_client::{
......@@ -64,6 +64,7 @@ use turbopack_binding::{
use crate::{
project::Project,
route::{Endpoint, Route, Routes, WrittenEndpoint},
server_paths::all_server_paths,
};
#[turbo_tasks::value]
......
use indexmap::IndexMap;
use turbo_tasks::{Completion, Vc};
use crate::server_paths::ServerPath;
#[turbo_tasks::value(shared)]
#[derive(Copy, Clone, Debug)]
pub enum Route {
......@@ -34,15 +36,13 @@ pub enum WrittenEndpoint {
NodeJs {
/// Relative to the root_path
server_entry_path: String,
/// Relative to the root_path
server_paths: Vec<String>,
server_paths: Vec<ServerPath>,
},
Edge {
/// Relative to the root_path
files: Vec<String>,
global_var_name: String,
/// Relative to the root_path
server_paths: Vec<String>,
server_paths: Vec<ServerPath>,
},
}
......
use anyhow::Result;
use next_core::all_assets_from_entries;
use serde::{Deserialize, Serialize};
use turbo_tasks::{trace::TraceRawVcs, TryFlatJoinIterExt, Vc};
use turbopack_binding::{
turbo::tasks_fs::FileSystemPath,
turbopack::core::{
asset::{Asset, AssetContent},
output::{OutputAsset, OutputAssets},
},
};
/// A reference to a server file with content hash for change detection
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs)]
pub struct ServerPath {
/// Relative to the root_path
pub path: String,
pub content_hash: u64,
}
/// A list of server paths
#[turbo_tasks::value(transparent)]
pub struct ServerPaths(Vec<ServerPath>);
/// Return a list of all server paths with filename and hash for all output
/// assets references from the `assets` list. Server paths are identified by
/// being inside of `node_root`.
#[turbo_tasks::function]
pub async fn all_server_paths(
assets: Vc<OutputAssets>,
node_root: Vc<FileSystemPath>,
) -> Result<Vc<ServerPaths>> {
let all_assets = all_assets_from_entries(assets).await?;
let node_root = &node_root.await?;
Ok(Vc::cell(
all_assets
.iter()
.map(|&asset| async move {
Ok(
if let Some(path) = node_root.get_path_to(&*asset.ident().path().await?) {
let content_hash = match *asset.content().await? {
AssetContent::File(file) => *file.hash().await?,
AssetContent::Redirect { .. } => 0,
};
Some(ServerPath {
path: path.to_string(),
content_hash,
})
} else {
None
},
)
})
.try_flat_join()
.await?,
))
}
use anyhow::Result;
use turbo_tasks::{
graph::{AdjacencyMap, GraphTraversal},
Completion, Completions, TryFlatJoinIterExt, TryJoinIterExt, Vc,
Completion, Completions, TryFlatJoinIterExt, Vc,
};
use turbo_tasks_fs::{rebase, FileSystemPath};
use turbopack_binding::turbopack::core::{
......@@ -9,29 +9,6 @@ use turbopack_binding::turbopack::core::{
output::{OutputAsset, OutputAssets},
};
#[turbo_tasks::function]
pub async fn all_server_paths(
assets: Vc<OutputAssets>,
node_root: Vc<FileSystemPath>,
) -> Result<Vc<Vec<String>>> {
let all_assets = all_assets_from_entries(assets).await?;
let node_root = &node_root.await?;
Ok(Vc::cell(
all_assets
.iter()
.map(|&asset| async move {
Ok(node_root
.get_path_to(&*asset.ident().path().await?)
.map(|s| s.to_string()))
})
.try_join()
.await?
.into_iter()
.flatten()
.collect(),
))
}
/// Emits all assets transitively reachable from the given chunks, that are
/// inside the node root or the client root.
///
......
......@@ -46,9 +46,7 @@ pub mod util;
pub use app_segment_config::{
parse_segment_config_from_loader_tree, parse_segment_config_from_source,
};
pub use emit::{
all_assets_from_entries, all_server_paths, emit_all_assets, emit_assets, emit_client_assets,
};
pub use emit::{all_assets_from_entries, emit_all_assets, emit_assets, emit_client_assets};
pub use next_edge::context::{
get_edge_chunking_context, get_edge_compile_time_info, get_edge_resolve_options_context,
};
......
......@@ -540,20 +540,25 @@ interface EndpointConfig {
preferredRegion?: string
}
export type ServerPath = {
path: string
contentHash: string
}
export type WrittenEndpoint =
| {
type: 'nodejs'
/** The entry path for the endpoint. */
entryPath: string
/** All server paths that has been written for the endpoint. */
serverPaths: string[]
serverPaths: ServerPath[]
config: EndpointConfig
}
| {
type: 'edge'
files: string[]
/** All server paths that has been written for the endpoint. */
serverPaths: string[]
serverPaths: ServerPath[]
globalVarName: string
config: EndpointConfig
}
......
......@@ -328,10 +328,30 @@ async function startWatcher(opts: SetupOpts) {
}
}
const serverPathState = new Map<string, string>()
async function processResult(
id: string,
result: TurbopackResult<WrittenEndpoint>
): Promise<TurbopackResult<WrittenEndpoint>> {
const hasAppPaths = result.serverPaths.some((p) =>
// Figure out if the server files have changed
let hasChange = false
for (const { path: p, contentHash } of result.serverPaths) {
// We ignore source maps
if (p.endsWith('.map')) continue
let key = `${id}:${p}`
const previousHash = serverPathState.get(key)
if (previousHash !== contentHash) {
hasChange = true
serverPathState.set(key, contentHash)
}
}
if (!hasChange) {
return result
}
const hasAppPaths = result.serverPaths.some(({ path: p }) =>
p.startsWith('server/app')
)
......@@ -339,7 +359,11 @@ async function startWatcher(opts: SetupOpts) {
deleteAppClientCache()
}
for (const file of result.serverPaths.map((p) => path.join(distDir, p))) {
const serverPaths = result.serverPaths.map(({ path: p }) =>
path.join(distDir, p)
)
for (const file of serverPaths) {
clearModuleContext(file)
deleteCache(file)
}
......@@ -864,6 +888,7 @@ async function startWatcher(opts: SetupOpts) {
if (middleware) {
const processMiddleware = async () => {
const writtenEndpoint = await processResult(
'middleware',
await middleware.endpoint.writeToDisk()
)
processIssues('middleware', 'middleware', writtenEndpoint)
......@@ -1067,6 +1092,7 @@ async function startWatcher(opts: SetupOpts) {
if (page === '/_error') {
if (globalEntries.app) {
const writtenEndpoint = await processResult(
'_app',
await globalEntries.app.writeToDisk()
)
processIssues('_app', '_app', writtenEndpoint)
......@@ -1076,6 +1102,7 @@ async function startWatcher(opts: SetupOpts) {
if (globalEntries.document) {
const writtenEndpoint = await processResult(
'_document',
await globalEntries.document.writeToDisk()
)
changeSubscription('_document', globalEntries.document, () => {
......@@ -1087,6 +1114,7 @@ async function startWatcher(opts: SetupOpts) {
if (globalEntries.error) {
const writtenEndpoint = await processResult(
'_error',
await globalEntries.error.writeToDisk()
)
processIssues(page, page, writtenEndpoint)
......@@ -1160,6 +1188,7 @@ async function startWatcher(opts: SetupOpts) {
if (globalEntries.app) {
const writtenEndpoint = await processResult(
'_app',
await globalEntries.app.writeToDisk()
)
processIssues('_app', '_app', writtenEndpoint)
......@@ -1169,6 +1198,7 @@ async function startWatcher(opts: SetupOpts) {
if (globalEntries.document) {
const writtenEndpoint = await processResult(
'_document',
await globalEntries.document.writeToDisk()
)
......@@ -1180,6 +1210,7 @@ async function startWatcher(opts: SetupOpts) {
await loadPagesManifest('_document')
const writtenEndpoint = await processResult(
page,
await route.htmlEndpoint.writeToDisk()
)
......@@ -1221,6 +1252,7 @@ async function startWatcher(opts: SetupOpts) {
// api requests to page API routes.
const writtenEndpoint = await processResult(
page,
await route.endpoint.writeToDisk()
)
......@@ -1243,6 +1275,7 @@ async function startWatcher(opts: SetupOpts) {
}
case 'app-page': {
const writtenEndpoint = await processResult(
page,
await route.htmlEndpoint.writeToDisk()
)
......@@ -1274,6 +1307,7 @@ async function startWatcher(opts: SetupOpts) {
}
case 'app-route': {
const writtenEndpoint = await processResult(
page,
await route.endpoint.writeToDisk()
)
......
......@@ -8221,9 +8221,6 @@
"Env Config dev mode with hot reload should have process environment override .env",
"Env Config dev mode with hot reload should have updated runtime values after change",
"Env Config dev mode with hot reload should load env from .env",
"Env Config dev mode with hot reload should provide env correctly for API routes",
"Env Config dev mode with hot reload should provide env correctly for SSR",
"Env Config dev mode with hot reload should provide env for SSG",
"Env Config dev mode with hot reload should provide global env to next.config.js",
"Env Config test environment should provide env correctly for API routes",
"Env Config test environment should provide global env to next.config.js"
......@@ -8231,6 +8228,9 @@
"failed": [
"Env Config dev mode should inline global values during build",
"Env Config dev mode with hot reload should inline global values during build",
"Env Config dev mode with hot reload should provide env correctly for API routes",
"Env Config dev mode with hot reload should provide env correctly for SSR",
"Env Config dev mode with hot reload should provide env for SSG",
"Env Config dev mode with hot reload should trigger HMR correctly when NEXT_PUBLIC_ env is changed",
"Env Config test environment should have process environment override .env",
"Env Config test environment should inline global values during build",
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment