Unverified Commit ec107c16 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

lazy server hmr (#96566)

## Summary

Make Turbopack server HMR demand-driven. Server updates are now compiled
and applied when the next relevant request writes an endpoint, instead
of eagerly evaluating changed server modules after every file change.

This replaces the aggregate server HMR subscription with an on-demand
update API, while preserving incremental updates and falling back to
full cache eviction when a restart is required.

Test Plan: added an e2e test

<!-- NEXT_JS_LLM -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarWill Binns-Smith <755844+wbinnssmith@users.noreply.github.com>
parent 724641c6
This diff is collapsed.
...@@ -2168,14 +2168,26 @@ impl Endpoint for AppEndpoint { ...@@ -2168,14 +2168,26 @@ impl Endpoint for AppEndpoint {
}; };
let written_endpoint = match *output.await? { let written_endpoint = match *output.await? {
AppEndpointOutput::NodeJs { rsc_chunk, .. } => EndpointOutputPaths::NodeJs { AppEndpointOutput::NodeJs { rsc_chunk, .. } => {
server_entry_path: node_root let server_entry_path: RcStr = node_root
.get_path_to(&*rsc_chunk.path().await?) .get_path_to(&*rsc_chunk.path().await?)
.context("Node.js chunk entry path must be inside the node root")? .context("Node.js chunk entry path must be inside the node root")?
.into(), .into();
let hmr_entry_path = server_entry_path
.strip_prefix("server/app/")
.unwrap_or(&server_entry_path)
.strip_suffix(".js")
.unwrap_or(&server_entry_path);
EndpointOutputPaths::NodeJs {
server_hmr_entry_paths: vec![
format!("{hmr_entry_path}.js").into(),
format!("{hmr_entry_path}/client-components-ssr.js").into(),
],
server_entry_path,
server_paths, server_paths,
client_paths, client_paths,
}, }
}
AppEndpointOutput::Edge { .. } => EndpointOutputPaths::Edge { AppEndpointOutput::Edge { .. } => EndpointOutputPaths::Edge {
server_paths, server_paths,
client_paths, client_paths,
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
#![feature(impl_trait_in_assoc_type)] #![feature(impl_trait_in_assoc_type)]
mod aggregate_hmr; pub mod aggregate_hmr;
pub mod analyze; pub mod analyze;
mod app; mod app;
mod asset_hashes_manifest; mod asset_hashes_manifest;
......
...@@ -1736,6 +1736,7 @@ impl Endpoint for PageEndpoint { ...@@ -1736,6 +1736,7 @@ impl Endpoint for PageEndpoint {
EndpointOutputPaths::NodeJs { EndpointOutputPaths::NodeJs {
server_entry_path, server_entry_path,
server_hmr_entry_paths: vec![],
server_paths, server_paths,
client_paths, client_paths,
} }
......
...@@ -84,8 +84,7 @@ use turbopack_core::{ ...@@ -84,8 +84,7 @@ use turbopack_core::{
reference_type::{CommonJsReferenceSubType, ReferenceType}, reference_type::{CommonJsReferenceSubType, ReferenceType},
resolve::{FindContextFileResult, find_context_file}, resolve::{FindContextFileResult, find_context_file},
version::{ version::{
NotFoundVersion, OptionVersionedContent, PartialUpdate, TotalUpdate, Update, Version, NotFoundVersion, OptionVersionedContent, Update, Version, VersionState, VersionedContent,
VersionState, VersionedContent,
}, },
}; };
#[cfg(all(feature = "process_pool", not(target_family = "wasm")))] #[cfg(all(feature = "process_pool", not(target_family = "wasm")))]
...@@ -96,7 +95,7 @@ use turbopack_node::worker_threads_backend; ...@@ -96,7 +95,7 @@ use turbopack_node::worker_threads_backend;
use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher}; use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher};
use crate::{ use crate::{
aggregate_hmr::{AggregateHmrVersion, ChunkListUpdateBuilder, DiffResult, diff_chunks_against}, aggregate_hmr::ServerHmrChunkLists,
app::{AppProject, OptionAppProject}, app::{AppProject, OptionAppProject},
empty::EmptyEndpoint, empty::EmptyEndpoint,
entrypoints::Entrypoints, entrypoints::Entrypoints,
...@@ -2538,98 +2537,25 @@ impl Project { ...@@ -2538,98 +2537,25 @@ impl Project {
} }
} }
/// Aggregate counterpart to [`Self::hmr_version_state`]: one [`VersionState`] /// Server entry chunks shared by all pull baselines.
/// covering every server HMR-eligible chunk. See [`Self::server_hmr_update`].
#[turbo_tasks::function(session_dependent)]
pub async fn server_hmr_version_state(self: ResolvedVc<Self>) -> Result<Vc<VersionState>> {
#[tracing::instrument(level = "info", name = "get server HMR version", skip_all)]
#[turbo_tasks::function(operation, root)]
async fn server_hmr_version_operation(
this: ResolvedVc<Project>,
) -> Result<Vc<Box<dyn Version>>> {
let Some(map) = this.await?.versioned_content_map else {
bail!("must be in dev mode to hmr")
};
let root = this.server_hmr_root_path().owned().await?;
AggregateHmrVersion::from_map(*map, root).await
}
let version_op = server_hmr_version_operation(self);
// INVALIDATION: untracked initial read; the subscription drives invalidation.
let state = VersionState::new(
version_op
.read_trait_strongly_consistent()
.untracked()
.await?,
)
.await?;
Ok(state)
}
/// Aggregate counterpart to [`Self::hmr_update`]: a single `Update` whose
/// combined `ChunkListUpdate` is the union of the server entry chunk diffs.
///
/// Each tracked entry chunk's own update is a `ChunkListUpdate` (carrying
/// the module deltas for its shared chunks via the merger) or a bare
/// `EcmascriptMergedUpdate`; both are folded into one `ChunkListUpdate` that
/// the runtime applies exactly as it would a single chunk list.
///
/// All-or-nothing restart: any chunk needing `Total`/`Missing` escalates
/// the whole batch to `Total` (the runtime can't partially restart). New
/// chunks absent from `from` are skipped; the runtime require()s them on
/// demand.
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn server_hmr_update(self: Vc<Self>, from: Vc<VersionState>) -> Result<Vc<Update>> { pub async fn server_hmr_chunks(self: Vc<Self>) -> Result<Vc<ServerHmrChunkLists>> {
let Some(map) = self.await?.versioned_content_map else { let Some(map) = self.await?.versioned_content_map else {
bail!("must be in dev mode to hmr") bail!("must be in dev mode to hmr")
}; };
let root = self.server_hmr_root_path().owned().await?; let root = self.server_hmr_root_path().owned().await?;
let chunks_versioned_content = map.hmr_chunks_in_path(root).await?; Ok(map.server_hmr_chunks_in_path(root))
// No chunks to diff yet (e.g. before any endpoints have been written).
if chunks_versioned_content.is_empty() {
return Ok(Update::None.cell());
} }
// Build `to` up front so we can return it on every escape hatch below. #[turbo_tasks::function]
let to_aggregate = AggregateHmrVersion::from_chunks(&chunks_versioned_content).await?; pub async fn server_hmr_chunks_for_entries(
let to_ref = Vc::upcast::<Box<dyn Version>>(to_aggregate) self: Vc<Self>,
.into_trait_ref() entry_paths: Vec<RcStr>,
.await?; ) -> Result<Vc<ServerHmrChunkLists>> {
let mut chunk_lists =
let DiffResult { ServerHmrChunkLists::new(self.server_hmr_chunks().await?.as_slice().to_vec());
chunk_updates, chunk_lists.retain_entry_paths(&entry_paths.into_iter().collect());
has_new_chunks, Ok(chunk_lists.cell())
} = diff_chunks_against(&chunks_versioned_content, from).await?;
// Nothing to apply, but `from` still needs to advance to `to`. Reaching
// here means `from` held a version we couldn't diff against (it wasn't an
// `AggregateHmrVersion`), so `diff_chunks_against` gave up and returned
// nothing. An empty `Partial` moves the subscription's state forward so
// the *next* change produces a real diff; returning `Total` instead would
// force a needless full re-evaluation.
if chunk_updates.is_empty() && !has_new_chunks {
return Ok(ChunkListUpdateBuilder::default().build(to_ref).cell());
}
let mut builder = ChunkListUpdateBuilder::default();
for (_path, update) in chunk_updates {
match &*update {
Update::None => {}
Update::Missing | Update::Total(_) => {
return Ok(Update::Total(TotalUpdate { to: to_ref }).cell());
}
Update::Partial(PartialUpdate { instruction, .. }) => {
builder.add_instruction(instruction);
}
}
}
if builder.is_empty() && !has_new_chunks {
return Ok(Update::None.cell());
}
Ok(builder.build(to_ref).cell())
} }
/// Gets a list of all client HMR chunk names that can be subscribed to. /// Gets a list of all client HMR chunk names that can be subscribed to.
......
...@@ -294,6 +294,7 @@ pub enum EndpointOutputPaths { ...@@ -294,6 +294,7 @@ pub enum EndpointOutputPaths {
NodeJs { NodeJs {
/// Relative to the root_path /// Relative to the root_path
server_entry_path: RcStr, server_entry_path: RcStr,
server_hmr_entry_paths: Vec<RcStr>,
server_paths: Vec<AssetPath>, server_paths: Vec<AssetPath>,
client_paths: Vec<RcStr>, client_paths: Vec<RcStr>,
}, },
......
...@@ -14,10 +14,9 @@ use turbopack_core::{ ...@@ -14,10 +14,9 @@ use turbopack_core::{
source_map::GenerateSourceMap, source_map::GenerateSourceMap,
version::OptionVersionedContent, version::OptionVersionedContent,
}; };
use turbopack_nodejs::ecmascript::node::entry::chunk_list_content::EcmascriptBuildNodeChunkListContent;
use crate::aggregate_hmr::{ use crate::aggregate_hmr::{ServerHmrChunkList, ServerHmrChunkLists};
HmrChunkWithContent, HmrChunksWithContent, is_entry_chunk_list_content,
};
#[derive( #[derive(
Clone, TraceRawVcs, PartialEq, Eq, ValueDebugFormat, Debug, NonLocalValue, Encode, Decode, Clone, TraceRawVcs, PartialEq, Eq, ValueDebugFormat, Debug, NonLocalValue, Encode, Decode,
...@@ -94,8 +93,8 @@ impl VersionedContentMap { ...@@ -94,8 +93,8 @@ impl VersionedContentMap {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl VersionedContentMap { impl VersionedContentMap {
/// Lists the aggregate-HMR *entry* chunks under `root` with their /// Lists the aggregate-HMR *entry* chunks under `root` with their
/// [`VersionedContent`], sorted by path. Only entry-chunk-list content is /// [`VersionedContent`], sorted by path. Only Node.js chunk-list content is
/// returned (see [`is_entry_chunk_list_content`]). Callers scope which /// returned. Callers scope which
/// entries are included by narrowing `root` (e.g. the aggregate server-HMR /// entries are included by narrowing `root` (e.g. the aggregate server-HMR
/// subscription passes `server/app` to include App Router entries only). /// subscription passes `server/app` to include App Router entries only).
/// ///
...@@ -106,10 +105,10 @@ impl VersionedContentMap { ...@@ -106,10 +105,10 @@ impl VersionedContentMap {
/// can shift the internals of the map, making iteration order different /// can shift the internals of the map, making iteration order different
/// for the same set of paths. /// for the same set of paths.
#[turbo_tasks::function(session_dependent)] #[turbo_tasks::function(session_dependent)]
pub async fn hmr_chunks_in_path( pub async fn server_hmr_chunks_in_path(
self: Vc<Self>, self: Vc<Self>,
root: FileSystemPath, root: FileSystemPath,
) -> Result<Vc<HmrChunksWithContent>> { ) -> Result<Vc<ServerHmrChunkLists>> {
let this = self.await?; let this = self.await?;
// `State::get` returns a lock guard, which can't be held across the // `State::get` returns a lock guard, which can't be held across the
// awaits below, so snapshot the keys and release it. // awaits below, so snapshot the keys and release it.
...@@ -138,18 +137,20 @@ impl VersionedContentMap { ...@@ -138,18 +137,20 @@ impl VersionedContentMap {
// *Important*: only chunk lists are subscribed to. Individual chunks are already // *Important*: only chunk lists are subscribed to. Individual chunks are already
// covered by the chunk list that owns them, so including them here // covered by the chunk list that owns them, so including them here
// would produce duplicate updates for the same change. // would produce duplicate updates for the same change.
if !is_entry_chunk_list_content(content) { let Some(versioned_content) =
ResolvedVc::try_downcast_type::<EcmascriptBuildNodeChunkListContent>(content)
else {
return Ok(None); return Ok(None);
} };
Ok(Some(HmrChunkWithContent { Ok(Some(ServerHmrChunkList {
path: name, relative_path: name,
content, versioned_content,
})) }))
}) })
.try_flat_join() .try_flat_join()
.await?; .await?;
chunks.sort_by(|a, b| a.path.cmp(&b.path)); chunks.sort_by(|a, b| a.relative_path.cmp(&b.relative_path));
Ok(Vc::cell(chunks)) Ok(Vc::cell(chunks))
} }
......
...@@ -52,6 +52,7 @@ impl From<AssetPath> for NapiAssetPath { ...@@ -52,6 +52,7 @@ impl From<AssetPath> for NapiAssetPath {
pub struct NapiWrittenEndpoint { pub struct NapiWrittenEndpoint {
pub r#type: String, pub r#type: String,
pub entry_path: Option<String>, pub entry_path: Option<String>,
pub server_hmr_entry_paths: Vec<String>,
pub client_paths: Vec<String>, pub client_paths: Vec<String>,
pub server_paths: Vec<NapiAssetPath>, pub server_paths: Vec<NapiAssetPath>,
pub config: NapiEndpointConfig, pub config: NapiEndpointConfig,
...@@ -62,11 +63,16 @@ impl From<Option<EndpointOutputPaths>> for NapiWrittenEndpoint { ...@@ -62,11 +63,16 @@ impl From<Option<EndpointOutputPaths>> for NapiWrittenEndpoint {
match written_endpoint { match written_endpoint {
Some(EndpointOutputPaths::NodeJs { Some(EndpointOutputPaths::NodeJs {
server_entry_path, server_entry_path,
server_hmr_entry_paths,
server_paths, server_paths,
client_paths, client_paths,
}) => Self { }) => Self {
r#type: "nodejs".to_string(), r#type: "nodejs".to_string(),
entry_path: Some(server_entry_path.into_owned()), entry_path: Some(server_entry_path.into_owned()),
server_hmr_entry_paths: server_hmr_entry_paths
.into_iter()
.map(String::from)
.collect(),
client_paths: client_paths.into_iter().map(From::from).collect(), client_paths: client_paths.into_iter().map(From::from).collect(),
server_paths: server_paths.into_iter().map(From::from).collect(), server_paths: server_paths.into_iter().map(From::from).collect(),
..Default::default() ..Default::default()
......
...@@ -19,6 +19,9 @@ use napi::{ ...@@ -19,6 +19,9 @@ use napi::{
}; };
use napi_derive::napi; use napi_derive::napi;
use next_api::{ use next_api::{
aggregate_hmr::{
ServerHmrChunkListVersion, ServerHmrChunkLists, ServerHmrUpdate, compute_server_hmr_update,
},
entrypoints::Entrypoints, entrypoints::Entrypoints,
next_server_nft::next_server_nft_assets, next_server_nft::next_server_nft_assets,
operation::{ operation::{
...@@ -1854,124 +1857,147 @@ async fn hmr_update_with_issues_operation( ...@@ -1854,124 +1857,147 @@ async fn hmr_update_with_issues_operation(
.cell()) .cell())
} }
/// Aggregate counterpart to [`project_hmr_update_operation`]. #[turbo_tasks::value(serialization = "skip")]
struct ServerHmrSnapshotWithEffects {
chunk_lists: ReadRef<ServerHmrChunkLists>,
version: ReadRef<ServerHmrChunkListVersion>,
issues: Arc<Vec<ReadRef<PlainIssue>>>,
effects: Arc<Effects>,
}
#[turbo_tasks::value(serialization = "skip")]
struct ServerHmrSnapshot {
chunk_lists: ReadRef<ServerHmrChunkLists>,
version: ReadRef<ServerHmrChunkListVersion>,
}
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
fn project_server_hmr_update_operation( async fn project_server_hmr_snapshot_operation(
project: ResolvedVc<Project>, project: ResolvedVc<Project>,
state: ResolvedVc<VersionState>, entry_paths: Vec<RcStr>,
) -> Vc<Update> { ) -> Result<Vc<ServerHmrSnapshot>> {
project.server_hmr_update(*state) let chunk_lists = project.server_hmr_chunks_for_entries(entry_paths).await?;
let version = ServerHmrChunkListVersion::from_chunk_lists(chunk_lists.as_slice())
.await?
.cell()
.await?;
Ok(ServerHmrSnapshot {
chunk_lists,
version,
}
.cell())
} }
/// Aggregate counterpart to [`hmr_update_with_issues_operation`]. /// Snapshot only; diffing here would keep old baselines active.
#[tracing::instrument(level = "info", name = "server hmr subscription", skip_all)] #[tracing::instrument(level = "info", name = "server hmr snapshot", skip_all)]
#[turbo_tasks::function(operation, root)] #[turbo_tasks::function(operation, root)]
async fn server_hmr_update_with_issues_operation( async fn server_hmr_snapshot_with_effects_operation(
project: ResolvedVc<Project>, project: ResolvedVc<Project>,
state: ResolvedVc<VersionState>, entry_paths: Vec<RcStr>,
) -> Result<Vc<HmrUpdateWithIssues>> { ) -> Result<Vc<ServerHmrSnapshotWithEffects>> {
tracing::info!("server hmr subscription"); tracing::info!("server hmr snapshot");
let update_op = project_server_hmr_update_operation(project, state); let snapshot_op = project_server_hmr_snapshot_operation(project, entry_paths);
// See `hmr_update_with_issues_operation`: the JS consumer relies on this // Build-graph failures must reach the JS recovery path.
// read *throwing* on build-graph failures; don't swallow errors. let snapshot = snapshot_op
let update = update_op
.read_strongly_consistent() .read_strongly_consistent()
.final_read_hint() .final_read_hint()
.await?; .await?;
let filter = project.issue_filter().await?; let filter = project.issue_filter().await?;
let issues = get_issues(update_op, &filter).await?; let issues = get_issues(snapshot_op, &filter).await?;
let effects = Arc::new(take_effects(update_op).await?); let effects = Arc::new(take_effects(snapshot_op).await?);
Ok(HmrUpdateWithIssues { Ok(ServerHmrSnapshotWithEffects {
update, chunk_lists: snapshot.chunk_lists.clone(),
version: snapshot.version.clone(),
issues, issues,
effects, effects,
} }
.cell()) .cell())
} }
#[tracing::instrument( pub struct ServerHmrVersion(ReadRef<ServerHmrChunkListVersion>);
level = "info",
name = "get server HMR events", #[napi(object, object_from_js = false)]
skip(env, project, func) pub struct NapiServerHmrUpdate {
)] #[napi(ts_type = "\"none\" | \"partial\" | \"restart\"")]
#[napi(ts_return_type = "{ __napiType: \"RootTask\" }")] pub kind: String,
pub fn project_server_hmr_events( /// `unknown` forces the TypeScript boundary to narrow the payload.
env: Env, #[napi(ts_type = "unknown")]
pub instruction: Option<serde_json::Value>,
pub version: Option<External<ServerHmrVersion>>,
}
impl NapiServerHmrUpdate {
/// Flattens the union for napi; `swc/types.ts` restores it.
fn new(update: &ServerHmrUpdate) -> Result<Self> {
let (kind, to, instruction) = match update {
ServerHmrUpdate::NoRuntimeUpdate { to } => ("none", to.as_ref(), None),
ServerHmrUpdate::FullReevaluation { to } => ("restart", Some(to), None),
ServerHmrUpdate::Partial { to, instruction } => {
("partial", Some(to), Some(instruction))
}
};
Ok(Self {
kind: kind.into(),
instruction: instruction.map(serde_json::to_value).transpose()?,
version: to.map(|to| External::new(ServerHmrVersion(to.clone()))),
})
}
}
#[tracing::instrument(level = "info", name = "get server HMR update", skip_all)]
#[napi]
pub async fn project_get_server_hmr_update(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: &External<ProjectInstance>, #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: &External<ProjectInstance>,
#[napi(ts_arg_type = "(err: Error, value: TurbopackResult<NodeJsHmrUpdate>) => void")] from: Option<&External<ServerHmrVersion>>,
func: FunctionRef<TurbopackResult<Unknown<'static>>, ()>, entry_paths: Vec<RcStr>,
) -> napi::Result<External<RootTask>> { ) -> napi::Result<TurbopackResult<NapiServerHmrUpdate>> {
let container = project.container; let container = project.container;
// Sentinel resource id for the aggregated stream (no real chunk path). let turbo_tasks = project.turbopack_ctx.turbo_tasks();
let identifier_path: RcStr = rcstr!("__next_all_hmr__"); let from = from.map(|from| from.0.clone());
subscribe(
project.turbopack_ctx.clone(), let (project, read) = turbo_tasks
&env, .run(async move {
&func,
move || async move {
// HACK(bgw): Remove this unmark call // HACK(bgw): Remove this unmark call
unmark_top_level_task_may_leak_eventually_consistent_state(); unmark_top_level_task_may_leak_eventually_consistent_state();
let project = container.project().to_resolved().await?; let project = container.project().to_resolved().await?;
let state = project.server_hmr_version_state().to_resolved().await?;
let update_op = server_hmr_update_with_issues_operation(project, state);
// HACK(bgw): Remove this mark call // HACK(bgw): Remove this mark call
mark_top_level_task(); mark_top_level_task();
let snapshot_op = server_hmr_snapshot_with_effects_operation(project, entry_paths);
let read = let read =
read_strongly_consistent_and_apply_effects(update_op, |v| &v.effects).await?; read_strongly_consistent_and_apply_effects(snapshot_op, |v| &v.effects).await?;
Ok((project, read))
// HACK(bgw): Remove this unmark call })
unmark_top_level_task_may_leak_eventually_consistent_state(); .await
.map_err(|e| napi::Error::from_reason(PrettyPrintError(&e.into()).to_string()))?;
let HmrUpdateWithIssues { update, issues, .. } = &*read; // Diffing must not remain active after the pull completes.
match &**update { let (update, issues) = turbo_tasks
Update::Missing | Update::None => {} .run(async move {
Update::Total(TotalUpdate { to }) => { // The snapshot's chunk-list `Vc`s are only valid while the project is held.
state.set(to.clone()).await?; let _project_keep_alive = project;
} let ServerHmrSnapshotWithEffects {
Update::Partial(PartialUpdate { to, .. }) => { chunk_lists,
state.set(to.clone()).await?; version,
} issues,
} ..
Ok((Some(update.clone()), issues.clone())) } = &*read;
}, let update =
move |ctx| { compute_server_hmr_update(chunk_lists.as_slice(), from.as_deref(), version.clone())
let (update, issues) = ctx.value; .await?;
Ok::<_, anyhow::Error>((update, issues.clone()))
})
.await
.map_err(|e| napi::Error::from_reason(PrettyPrintError(&e.into()).to_string()))?;
let napi_issues = issues Ok(TurbopackResult {
result: NapiServerHmrUpdate::new(&update)
.map_err(|error| napi::Error::from_reason(PrettyPrintError(&error).to_string()))?,
issues: issues
.iter() .iter()
.map(|issue| NapiIssue::from(&**issue)) .map(|issue| NapiIssue::from(&**issue))
.collect(); .collect(),
let update_issues = issues
.iter()
.map(|issue| Issue::from(&**issue))
.collect::<Vec<_>>();
let identifier = ResourceIdentifier {
path: identifier_path.clone(),
headers: None,
};
let update = match update.as_deref() {
None | Some(Update::Missing) | Some(Update::Total(_)) => {
ClientUpdateInstruction::restart(&identifier, &update_issues)
}
Some(Update::Partial(update)) => ClientUpdateInstruction::partial(
&identifier,
&update.instruction,
&update_issues,
),
Some(Update::None) => ClientUpdateInstruction::issues(&identifier, &update_issues),
};
Ok(TurbopackResult {
result: ctx.env.to_js_value(&update)?,
issues: napi_issues,
}) })
},
)
} }
#[tracing::instrument(level = "info", name = "get client HMR events", skip(env, project, func), fields(chunk_name = %chunk_name))] #[tracing::instrument(level = "info", name = "get client HMR events", skip(env, project, func), fields(chunk_name = %chunk_name))]
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
import type { import type {
CompilationEvent, CompilationEvent,
NodeJsHmrUpdate,
TurbopackResult, TurbopackResult,
Update, Update,
UpdateMessage, UpdateMessage,
...@@ -10,6 +9,7 @@ import type { ...@@ -10,6 +9,7 @@ import type {
export type TurboTasks = { readonly __tag: unique symbol } export type TurboTasks = { readonly __tag: unique symbol }
export type ExternalEndpoint = { readonly __tag: unique symbol } export type ExternalEndpoint = { readonly __tag: unique symbol }
export type ServerHmrVersion = { readonly __tag: unique symbol }
export type NextTurboTasks = { readonly __tag: unique symbol } export type NextTurboTasks = { readonly __tag: unique symbol }
export type RefCell<_T = unknown> = { readonly __tag: unique symbol } export type RefCell<_T = unknown> = { readonly __tag: unique symbol }
export type FlushGuard = { readonly __tag: unique symbol } export type FlushGuard = { readonly __tag: unique symbol }
...@@ -494,6 +494,13 @@ export interface NapiRoute { ...@@ -494,6 +494,13 @@ export interface NapiRoute {
dataEndpoint?: ExternalObject<ExternalEndpoint> dataEndpoint?: ExternalObject<ExternalEndpoint>
} }
export interface NapiServerHmrUpdate {
kind: 'none' | 'partial' | 'restart'
/** `unknown` forces the TypeScript boundary to narrow the payload. */
instruction?: unknown
version?: ExternalObject<ServerHmrVersion>
}
export interface NapiSource { export interface NapiSource {
ident: RcStr ident: RcStr
filePath: RcStr filePath: RcStr
...@@ -552,6 +559,7 @@ export interface NapiWatchOptions { ...@@ -552,6 +559,7 @@ export interface NapiWatchOptions {
export interface NapiWrittenEndpoint { export interface NapiWrittenEndpoint {
type: string type: string
entryPath?: string entryPath?: string
serverHmrEntryPaths: Array<string>
clientPaths: Array<string> clientPaths: Array<string>
serverPaths: Array<NapiAssetPath> serverPaths: Array<NapiAssetPath>
config: NapiEndpointConfig config: NapiEndpointConfig
...@@ -625,6 +633,12 @@ export declare function projectGetAllCompilationIssues(project: { ...@@ -625,6 +633,12 @@ export declare function projectGetAllCompilationIssues(project: {
__napiType: 'Project' __napiType: 'Project'
}): Promise<TurbopackResult<undefined>> }): Promise<TurbopackResult<undefined>>
export declare function projectGetServerHmrUpdate(
project: { __napiType: 'Project' },
from: ExternalObject<ServerHmrVersion> | undefined | null,
entryPaths: Array<RcStr>
): Promise<TurbopackResult<NapiServerHmrUpdate>>
export declare function projectGetSourceForAsset( export declare function projectGetSourceForAsset(
project: { __napiType: 'Project' }, project: { __napiType: 'Project' },
filePath: RcStr filePath: RcStr
...@@ -664,11 +678,6 @@ export declare function projectOnExit(project: { ...@@ -664,11 +678,6 @@ export declare function projectOnExit(project: {
__napiType: 'Project' __napiType: 'Project'
}): Promise<void> }): Promise<void>
export declare function projectServerHmrEvents(
project: { __napiType: 'Project' },
func: (err: Error, value: TurbopackResult<NodeJsHmrUpdate>) => void
): { __napiType: 'RootTask' }
/** /**
* Runs `project_on_exit`, and then waits for turbo_tasks to gracefully shut down. * Runs `project_on_exit`, and then waits for turbo_tasks to gracefully shut down.
* *
......
...@@ -28,12 +28,13 @@ import type { ...@@ -28,12 +28,13 @@ import type {
Endpoint, Endpoint,
HmrChunkNames, HmrChunkNames,
Lockfile, Lockfile,
NodeJsHmrUpdate,
PartialProjectOptions, PartialProjectOptions,
Project, Project,
ProjectOptions, ProjectOptions,
RawEntrypoints, RawEntrypoints,
Route, Route,
ServerHmrUpdate,
ServerHmrVersion,
TurboEngineOptions, TurboEngineOptions,
TurbopackResult, TurbopackResult,
TurbopackStackFrame, TurbopackStackFrame,
...@@ -758,10 +759,16 @@ function bindingToApi( ...@@ -758,10 +759,16 @@ function bindingToApi(
})() })()
} }
serverHmrEvents(): AsyncIterableIterator<TurbopackResult<NodeJsHmrUpdate>> { async getServerHmrUpdate(
return subscribe(true, async (callback) => from: ServerHmrVersion | undefined,
binding.projectServerHmrEvents(this._nativeProject, callback) entryPaths: string[]
) ): Promise<TurbopackResult<ServerHmrUpdate>> {
// napi cannot express the field correlation.
return binding.projectGetServerHmrUpdate(
this._nativeProject,
from,
entryPaths
) as Promise<TurbopackResult<ServerHmrUpdate>>
} }
clientHmrEvents( clientHmrEvents(
......
...@@ -13,6 +13,7 @@ import type { ...@@ -13,6 +13,7 @@ import type {
TraceQueryOptions, TraceQueryOptions,
TraceQueryResult, TraceQueryResult,
MemoryEvictionMode, MemoryEvictionMode,
ServerHmrVersion as NativeServerHmrVersion,
} from './generated-native' } from './generated-native'
export type { TraceServerHandle, TraceQueryOptions, TraceQueryResult } export type { TraceServerHandle, TraceQueryOptions, TraceQueryResult }
...@@ -254,19 +255,24 @@ export interface NodeJsChunkListUpdate { ...@@ -254,19 +255,24 @@ export interface NodeJsChunkListUpdate {
chunks?: Record<string, { type: 'added' | 'deleted' | 'total' | 'partial' }> chunks?: Record<string, { type: 'added' | 'deleted' | 'total' | 'partial' }>
} }
export interface NodeJsPartialHmrUpdate extends BaseUpdate { /** In-process update; unlike wire updates, it has no resource or issues. */
export interface NodeJsPartialHmrUpdate {
type: 'partial' type: 'partial'
instruction: NodeJsEcmascriptMergedUpdate | NodeJsChunkListUpdate instruction: NodeJsEcmascriptMergedUpdate | NodeJsChunkListUpdate
} }
export interface NodeJsRestartHmrUpdate { /** Opaque baseline for the next pull. */
type: 'restart' export type ServerHmrVersion = ExternalObject<NativeServerHmrVersion>
}
export type NodeJsHmrUpdate = /** Restores the union flattened by napi. */
| IssuesUpdate export type ServerHmrUpdate =
| NodeJsPartialHmrUpdate | { kind: 'none'; version?: ServerHmrVersion }
| NodeJsRestartHmrUpdate | { kind: 'restart'; version: ServerHmrVersion }
| {
kind: 'partial'
version: ServerHmrVersion
instruction: NodeJsPartialHmrUpdate['instruction']
}
export interface HmrChunkNames { export interface HmrChunkNames {
/** Relative paths to output chunks that can receive HMR updates (e.g., "server/chunks/ssr/..._.js") */ /** Relative paths to output chunks that can receive HMR updates (e.g., "server/chunks/ssr/..._.js") */
...@@ -334,7 +340,10 @@ export interface Project { ...@@ -334,7 +340,10 @@ export interface Project {
TurbopackResult<RawEntrypoints | {}> TurbopackResult<RawEntrypoints | {}>
> >
serverHmrEvents(): AsyncIterableIterator<TurbopackResult<NodeJsHmrUpdate>> getServerHmrUpdate(
from: ServerHmrVersion | undefined,
entryPaths: string[]
): Promise<TurbopackResult<ServerHmrUpdate>>
clientHmrEvents( clientHmrEvents(
identifier: string identifier: string
...@@ -444,6 +453,8 @@ export type WrittenEndpoint = ...@@ -444,6 +453,8 @@ export type WrittenEndpoint =
type: 'nodejs' type: 'nodejs'
/** The entry path for the endpoint. */ /** The entry path for the endpoint. */
entryPath: string entryPath: string
/** Server HMR entry chunk lists owned by this endpoint. */
serverHmrEntryPaths: string[]
/** All client paths that have been written for the endpoint. */ /** All client paths that have been written for the endpoint. */
clientPaths: string[] clientPaths: string[]
/** All server paths that have been written for the endpoint. */ /** All server paths that have been written for the endpoint. */
...@@ -452,6 +463,7 @@ export type WrittenEndpoint = ...@@ -452,6 +463,7 @@ export type WrittenEndpoint =
} }
| { | {
type: 'edge' type: 'edge'
serverHmrEntryPaths: []
/** All client paths that have been written for the endpoint. */ /** All client paths that have been written for the endpoint. */
clientPaths: string[] clientPaths: string[]
/** All server paths that have been written for the endpoint. */ /** All server paths that have been written for the endpoint. */
...@@ -460,6 +472,7 @@ export type WrittenEndpoint = ...@@ -460,6 +472,7 @@ export type WrittenEndpoint =
} }
| { | {
type: 'none' type: 'none'
serverHmrEntryPaths: []
clientPaths: [] clientPaths: []
serverPaths: [] serverPaths: []
config: EndpointConfig config: EndpointConfig
......
...@@ -142,14 +142,6 @@ type HandleRouteTypeHooks = { ...@@ -142,14 +142,6 @@ type HandleRouteTypeHooks = {
handleWrittenEndpoint: HandleWrittenEndpoint handleWrittenEndpoint: HandleWrittenEndpoint
subscribeToChanges: StartChangeSubscription subscribeToChanges: StartChangeSubscription
handleServerComponentChanges?: () => void handleServerComponentChanges?: () => void
// When Turbopack server fast refresh is enabled, the aggregate server-HMR
// subscription (setupServerHmr `onApplied` in hot-reloader-turbopack.ts)
// owns the browser refresh signal for app-page RSC changes and only fires
// after the server module cache is refreshed. In that mode the per-page
// `rscHmrEndpoint` subscription must NOT also send SERVER_COMPONENT_CHANGES,
// or every edit triggers two RSC refetches (the first immediately
// superseded).
serverFastRefresh?: boolean
} }
export async function handleRouteType({ export async function handleRouteType({
...@@ -373,21 +365,6 @@ export async function handleRouteType({ ...@@ -373,21 +365,6 @@ export async function handleRouteType({
} }
// Report the next compilation again // Report the next compilation again
readyIds?.delete(pathname) readyIds?.delete(pathname)
// When server fast refresh is enabled, the aggregate server-HMR
// subscription sends SERVER_COMPONENT_CHANGES after applying the
// update in-process. Sending here too would double the refresh.
//
// But the aggregate subscription only fires when there is a live
// server-HMR handler registered (i.e. the page has rendered at
// least once). When recovering from a build error the page never
// rendered, so no handler exists, the aggregate stays silent, and
// this per-page send is the only thing that clears the redbox.
// Only suppress when a handler is actually live to own the refresh.
const hasLiveServerHmrHandler =
(globalThis.__turbopack_server_hmr_handlers__?.size ?? 0) > 0
if (hooks?.serverFastRefresh && hasLiveServerHmrHandler) {
return
}
hooks?.handleServerComponentChanges?.() hooks?.handleServerComponentChanges?.()
}, },
(e) => { (e) => {
......
import { revision } from '../shared'
console.log('lazy-server-hmr-page-0 evaluated')
export default function Page() {
return <p id="value">0: {revision}</p>
}
import { revision } from '../shared'
console.log('lazy-server-hmr-page-1 evaluated')
export default function Page() {
return <p id="value">1: {revision}</p>
}
import { revision } from '../shared'
console.log('lazy-server-hmr-page-2 evaluated')
export default function Page() {
return <p id="value">2: {revision}</p>
}
import { value } from './probe'
export default function Page() {
return <p id="value">{value}</p>
}
console.log('lazy-rebuild-probe evaluated')
export const value = 'initial'
import type { Response } from 'node-fetch' import type { Response } from 'node-fetch'
import { join } from 'path' import { join } from 'path'
import { nextTestSetup, FileRef } from 'e2e-utils' import { nextTestSetup, FileRef } from 'e2e-utils'
import { retry } from 'next-test-utils' import { retry, waitFor } from 'next-test-utils'
describe('server-hmr', () => { describe('server-hmr', () => {
const { next, isTurbopack, isNextDev } = nextTestSetup({ const { next, isTurbopack, isNextDev } = nextTestSetup({
files: __dirname, files: __dirname,
env: { NEXT_TEST_SERVER_HMR_DIFFING: '1' },
}) })
// Server HMR is a Turbopack-only feature, only available in dev mode // Server HMR is a Turbopack-only feature, only available in dev mode
const itTurbopackDev = isTurbopack && isNextDev ? it : it.skip const itTurbopackDev = isTurbopack && isNextDev ? it : it.skip
describe('module preservation', () => { describe('module preservation', () => {
itTurbopackDev(
'does not evaluate a changed server module until the next request',
async () => {
const browser = await next.browser('/lazy-rebuild')
await retry(async () => {
expect(await browser.elementByCss('#value').text()).toBe('initial')
})
await browser.eval(() => {
const originalFetch = window.fetch
window.fetch = (input, init) => {
const headers = new Headers(init?.headers)
if (headers.get('next-hmr-refresh') === '1') {
;(window as any).__didBlockHmrRequest = true
return new Promise(() => {})
}
return originalFetch(input, init)
}
})
const evaluationMarker = 'lazy-rebuild-probe evaluated'
const outputLengthBeforePatch = next.cliOutput.length
await next.patchFile('app/lazy-rebuild/probe.js', (content) =>
content.replace(
"export const value = 'initial'",
"export const value = 'updated'"
)
)
await retry(async () => {
expect(
await browser.eval(() => (window as any).__didBlockHmrRequest)
).toBe(true)
})
expect(next.cliOutput.slice(outputLengthBeforePatch)).not.toContain(
evaluationMarker
)
const response = await next.fetch('/lazy-rebuild')
expect(await response.text()).toContain('updated')
await retry(async () => {
expect(next.cliOutput.slice(outputLengthBeforePatch)).toContain(
evaluationMarker
)
})
}
)
itTurbopackDev(
'only evaluates the last visited page when a shared module changes',
async () => {
const browser = await next.browser('/lazy-pages/0')
for (let page = 1; page < 3; page++) {
await browser.loadPage(`${next.url}/lazy-pages/${page}`)
}
expect(await browser.elementByCss('#value').text()).toBe('2: rev-0')
const outputLengthBeforePatch = next.cliOutput.length
await next.patchFile('app/lazy-pages/shared.ts', (content) =>
content.replace(
"export const revision = 'rev-0'",
"export const revision = 'rev-1'"
)
)
await retry(async () => {
expect(await browser.elementByCss('#value').text()).toBe('2: rev-1')
})
await waitFor(1000)
const outputAfterPatch = next.cliOutput.slice(outputLengthBeforePatch)
const evaluatedPages = Array.from(
outputAfterPatch.matchAll(/lazy-server-hmr-page-(\d+) evaluated/g),
(match) => Number(match[1])
)
expect(new Set(evaluatedPages)).toEqual(new Set([2]))
const diffedPages = Array.from(
outputAfterPatch.matchAll(
/Diffing server HMR entry lazy-pages\/(\d+)\/page/g
),
(match) => Number(match[1])
)
expect(new Set(diffedPages)).toEqual(new Set([2]))
}
)
itTurbopackDev( itTurbopackDev(
'does not re-evaluate an unmodified module when page module changes', 'does not re-evaluate an unmodified module when page module changes',
async () => { async () => {
......
...@@ -15,8 +15,10 @@ type VersionTraitRef = TraitRef<Box<dyn Version>>; ...@@ -15,8 +15,10 @@ type VersionTraitRef = TraitRef<Box<dyn Version>>;
/// their versions are merged. Other chunks are tracked by path. /// their versions are merged. Other chunks are tracked by path.
/// ///
/// [`MergeableVersionedContent`]: turbopack_core::version::MergeableVersionedContent /// [`MergeableVersionedContent`]: turbopack_core::version::MergeableVersionedContent
#[turbo_tasks::value(serialization = "skip", shared)] #[turbo_tasks::value(serialization = "skip", shared, eq = "manual")]
#[derive(Debug)]
pub struct ChunkListVersion { pub struct ChunkListVersion {
pub id: RcStr,
/// A map from chunk path to its version. /// A map from chunk path to its version.
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
pub by_path: FxIndexMap<String, VersionTraitRef>, pub by_path: FxIndexMap<String, VersionTraitRef>,
...@@ -29,13 +31,30 @@ pub struct ChunkListVersion { ...@@ -29,13 +31,30 @@ pub struct ChunkListVersion {
pub by_merger: FxIndexMap<ResolvedVc<Box<dyn VersionedContentMerger>>, VersionTraitRef>, pub by_merger: FxIndexMap<ResolvedVc<Box<dyn VersionedContentMerger>>, VersionTraitRef>,
} }
// `id` is a hash over every tracked chunk version, so comparing it is equivalent to comparing
// the maps, which hold `VersionTraitRef`s that are not structurally comparable.
impl PartialEq for ChunkListVersion {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for ChunkListVersion {}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Version for ChunkListVersion { impl Version for ChunkListVersion {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn id(&self) -> Result<Vc<RcStr>> { async fn id(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(self.id.clone()))
}
}
async fn chunk_list_version_id(
by_path: &FxIndexMap<String, VersionTraitRef>,
by_merger: &FxIndexMap<ResolvedVc<Box<dyn VersionedContentMerger>>, VersionTraitRef>,
) -> Result<RcStr> {
let by_path = { let by_path = {
let mut by_path = self let mut by_path = by_path
.by_path
.iter() .iter()
.map(|(path, version)| (path, TraitRef::cell(version.clone()))) .map(|(path, version)| (path, TraitRef::cell(version.clone())))
.map(async |(path, version)| { .map(async |(path, version)| {
...@@ -48,8 +67,7 @@ impl Version for ChunkListVersion { ...@@ -48,8 +67,7 @@ impl Version for ChunkListVersion {
by_path by_path
}; };
let by_merger = { let by_merger = {
let mut by_merger = self let mut by_merger = by_merger
.by_merger
.iter() .iter()
.map(|(_merger, version)| TraitRef::cell(version.clone()).id().owned()) .map(|(_merger, version)| TraitRef::cell(version.clone()).id().owned())
.try_join() .try_join()
...@@ -69,8 +87,7 @@ impl Version for ChunkListVersion { ...@@ -69,8 +87,7 @@ impl Version for ChunkListVersion {
} }
let hash = hasher.finish(); let hash = hasher.finish();
let hash = encode_base64(hash); let hash = encode_base64(hash);
Ok(Vc::cell(hash.into())) Ok(hash.into())
}
} }
/// Computes a [`ChunkListVersion`] from a map of chunk paths to their /// Computes a [`ChunkListVersion`] from a map of chunk paths to their
...@@ -115,5 +132,11 @@ pub async fn compute_chunk_list_version( ...@@ -115,5 +132,11 @@ pub async fn compute_chunk_list_version(
.into_iter() .into_iter()
.collect(); .collect();
Ok(ChunkListVersion { by_path, by_merger }.cell()) let id = chunk_list_version_id(&by_path, &by_merger).await?;
Ok(ChunkListVersion {
id,
by_path,
by_merger,
}
.cell())
} }
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use turbo_tasks::{FxIndexMap, ResolvedVc, TryJoinIterExt, Vc}; use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TransientInstance, TryJoinIterExt, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -54,6 +54,20 @@ pub struct EcmascriptBuildNodeChunkListContent { ...@@ -54,6 +54,20 @@ pub struct EcmascriptBuildNodeChunkListContent {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl EcmascriptBuildNodeChunkListContent { impl EcmascriptBuildNodeChunkListContent {
#[turbo_tasks::function]
pub async fn compute_update_from_version(
self: Vc<Self>,
from: TransientInstance<ReadRef<ChunkListVersion>>,
) -> Result<Vc<Update>> {
let to = self.version();
update_chunk_list(
&self.await?.chunks_contents,
to,
ResolvedVc::upcast(ReadRef::resolved_cell((*from).clone())),
)
.await
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn new( pub async fn new(
chunking_context: ResolvedVc<NodeJsChunkingContext>, chunking_context: ResolvedVc<NodeJsChunkingContext>,
...@@ -143,3 +157,11 @@ impl VersionedContent for EcmascriptBuildNodeChunkListContent { ...@@ -143,3 +157,11 @@ impl VersionedContent for EcmascriptBuildNodeChunkListContent {
update_chunk_list(&this.chunks_contents, to_version, from_version).await update_chunk_list(&this.chunks_contents, to_version, from_version).await
} }
} }
#[turbo_tasks::function(operation, root)]
pub fn compute_update_from_version_operation(
content: ResolvedVc<EcmascriptBuildNodeChunkListContent>,
from: TransientInstance<ReadRef<ChunkListVersion>>,
) -> Vc<Update> {
content.compute_update_from_version(from)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment