Unverified Commit 05aeb5bb authored by vercel[bot]'s avatar vercel[bot] Committed by GitHub

Fix route HMR reactivation ordering

Co-Authored-By: 's avatarMarcos Hernanz <96699542+marcoshernanz@users.noreply.github.com>
parent 2dbbe886
......@@ -33,6 +33,8 @@ pub fn is_entry_chunk_list_content(content: ResolvedVc<Box<dyn VersionedContent>
pub struct AggregateHmrVersion {
#[turbo_tasks(trace_ignore)]
pub versions: FxIndexMap<RcStr, TraitRef<Box<dyn Version>>>,
/// Monotonic token identifying the active-route aggregate transition.
pub reactivation_epoch: u32,
}
#[turbo_tasks::value_impl]
......@@ -54,6 +56,7 @@ impl Version for AggregateHmrVersion {
.await?;
let mut hasher = Xxh3Hash64Hasher::new();
hasher.write_value(self.reactivation_epoch);
hasher.write_value(entries.len());
for (path, id) in entries {
hasher.write_value(path.as_str());
......@@ -71,15 +74,24 @@ impl AggregateHmrVersion {
// An empty `versions` map behaves the same as `NotFoundVersion` would in
// `diff_chunks_against`, so no special case is needed here.
let snapshot = map.hmr_chunks_in_path(root).await?;
Ok(Vc::upcast(Self::from_chunks(&snapshot.active).await?))
Ok(Vc::upcast(
Self::from_chunks(&snapshot.active, snapshot.reactivation_epoch).await?,
))
}
pub async fn from_chunks(chunks: &[HmrChunkWithContent]) -> Result<Vc<Self>> {
pub async fn from_chunks(
chunks: &[HmrChunkWithContent],
reactivation_epoch: u32,
) -> Result<Vc<Self>> {
let versions = Self::versions_from_chunks(chunks)
.await?
.into_iter()
.collect();
Ok(Self { versions }.cell())
Ok(Self {
versions,
reactivation_epoch,
}
.cell())
}
/// Builds the next aggregate version while retaining the last version of
......@@ -89,6 +101,7 @@ impl AggregateHmrVersion {
pub async fn from_chunks_with_previous(
chunks: &[HmrChunkWithContent],
inactive_paths: &FxHashSet<RcStr>,
reactivation_epoch: u32,
previous: Vc<VersionState>,
) -> Result<Vc<Self>> {
let mut versions = Self::versions_from_chunks(chunks).await?;
......@@ -110,10 +123,29 @@ impl AggregateHmrVersion {
versions.sort_by(|(a, _), (b, _)| a.cmp(b));
Ok(Self {
versions: versions.into_iter().collect(),
reactivation_epoch,
}
.cell())
}
pub async fn reactivation_epoch_from_state(state: Vc<VersionState>) -> Result<Option<u32>> {
let version = state.get().to_resolved().await?;
let Some(version) = ResolvedVc::try_downcast_type::<AggregateHmrVersion>(version) else {
return Ok(None);
};
Ok(Some(version.await?.reactivation_epoch))
}
pub async fn reactivation_epoch_from_version(
version: TraitRef<Box<dyn Version>>,
) -> Result<Option<u32>> {
let version = TraitRef::cell(version).to_resolved().await?;
let Some(version) = ResolvedVc::try_downcast_type::<AggregateHmrVersion>(version) else {
return Ok(None);
};
Ok(Some(version.await?.reactivation_epoch))
}
async fn versions_from_chunks(
chunks: &[HmrChunkWithContent],
) -> Result<Vec<(RcStr, TraitRef<Box<dyn Version>>)>> {
......
......@@ -40,7 +40,7 @@ use tracing::{Instrument, field::Empty};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, Completions, FxIndexMap, NonLocalValue, OperationValue, OperationVc, ReadRef,
ResolvedVc, State, TransientInstance, TryFlatJoinIterExt, TryJoinIterExt, Vc,
ResolvedVc, State, TraitRef, TransientInstance, TryFlatJoinIterExt, TryJoinIterExt, Vc,
debug::ValueDebugFormat, fxindexmap, trace::TraceRawVcs,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
......@@ -211,6 +211,13 @@ impl std::str::FromStr for HmrTarget {
}
}
/// Extracts the route-reactivation acknowledgement carried by an aggregate HMR version.
pub async fn aggregate_hmr_reactivation_epoch(
version: TraitRef<Box<dyn Version>>,
) -> Result<Option<u32>> {
AggregateHmrVersion::reactivation_epoch_from_version(version).await
}
/// Pre-converted route keys from debug build paths for O(1) lookups.
struct DebugBuildPathsRouteKeys {
app: FxHashSet<RcStr>,
......@@ -820,7 +827,9 @@ impl ProjectContainer {
chunk_names: Vec<RcStr>,
target: HmrTarget,
active: bool,
) -> Result<()> {
reactivation_source_paths: Vec<RcStr>,
advance_reactivation_epoch: bool,
) -> Result<u32> {
let project_op = project_operation(self);
let project = project_op.resolve().strongly_consistent().await?;
let project_ref = project_op.read_strongly_consistent().await?;
......@@ -830,20 +839,36 @@ impl ProjectContainer {
let root = project_hmr_root_path_operation(project, target)
.read_strongly_consistent()
.await?;
let project_fs = project_fs_operation(project)
.read_strongly_consistent()
.await?;
let project_fs_op = project_fs_operation(project);
let project_fs_vc = project_fs_op.resolve().strongly_consistent().await?;
let project_fs = project_fs_op.read_strongly_consistent().await?;
let paths = chunk_names
.into_iter()
.map(|chunk_name| root.join(&chunk_name))
.collect::<Result<Vec<_>>>()?;
let reactivation_source_paths = reactivation_source_paths
.into_iter()
.map(|path| {
project_fs
.try_from_sys_path(project_fs_vc, Path::new(path.as_str()), None)
.map(|path| project_fs.to_sys_path_raw(&path))
.with_context(|| {
format!("source path is outside the project filesystem: {path}")
})
})
.collect::<Result<Vec<_>>>()?;
versioned_content_map_operation(map)
.read_strongly_consistent()
.await?
.set_hmr_chunks_active(paths, active, &project_fs)
.await?;
Ok(())
.set_hmr_chunks_active(
paths,
active,
&project_fs,
reactivation_source_paths,
advance_reactivation_epoch,
)
.await
}
/// Permanently removes output entry chunks for routes deleted during a dev
......@@ -2721,15 +2746,16 @@ impl Project {
let root = self.aggregate_hmr_root_path(target).owned().await?;
let chunk_snapshot = map.hmr_chunks_in_path(&root).await?;
// No chunks to diff yet (e.g. before any endpoints have been written).
if chunk_snapshot.active.is_empty() {
return Ok(Update::None.cell());
}
// Build `to` up front so we can return it on every escape hatch below.
// Build `to` up front so even an empty active set can carry a route
// reactivation acknowledgement to the server runtime.
let from_reactivation_epoch =
AggregateHmrVersion::reactivation_epoch_from_state(from).await?;
let reactivation_epoch_changed =
from_reactivation_epoch != Some(chunk_snapshot.reactivation_epoch);
let to_aggregate = AggregateHmrVersion::from_chunks_with_previous(
&chunk_snapshot.active,
&chunk_snapshot.inactive_paths,
chunk_snapshot.reactivation_epoch,
from,
)
.await?;
......@@ -2765,7 +2791,7 @@ impl Project {
}
}
if builder.is_empty() && !has_new_chunks {
if builder.is_empty() && !has_new_chunks && !reactivation_epoch_changed {
return Ok(Update::None.cell());
}
......
use anyhow::Result;
use std::path::PathBuf;
use anyhow::{Result, bail};
use bincode::{Decode, Encode};
use next_core::emit_assets;
use rustc_hash::{FxHashMap, FxHashSet};
......@@ -116,6 +118,8 @@ unsafe impl OperationValue for InactiveOutputOperations {}
pub struct HmrChunkSnapshot {
pub active: Vec<HmrChunkWithContent>,
pub inactive_paths: FxHashSet<RcStr>,
/// Monotonic acknowledgement token for aggregate-HMR route reactivation.
pub reactivation_epoch: u32,
}
// TODO: Ideally this structure is never persisted, so new sessions start from scratch and don't
......@@ -137,6 +141,9 @@ pub struct VersionedContentMap {
/// changes do not invalidate or rebuild the aggregate subscription until the
/// entry becomes active again.
inactive_hmr_paths: State<InactiveHmrPaths>,
/// Advances after a route rejoins the aggregate HMR working set. The token is
/// carried by the aggregate version so JS can wait for that exact transition.
reactivation_epoch: State<u32>,
}
impl VersionedContentMap {
......@@ -149,6 +156,7 @@ impl VersionedContentMap {
inactive_output_operations: State::new(InactiveOutputOperations::default()),
removed_hmr_paths: State::new(InactiveHmrPaths::default()),
inactive_hmr_paths: State::new(InactiveHmrPaths::default()),
reactivation_epoch: State::new(0),
}
.resolved_cell()
}
......@@ -174,7 +182,11 @@ impl VersionedContentMap {
// awaits below, so snapshot the keys and release it. Keep the inactive
// paths from the same snapshot: only those paths may retain their prior
// aggregate version when absent from the active chunks below.
let (paths, inactive_paths): (Vec<FileSystemPath>, FxHashSet<RcStr>) = {
let (paths, inactive_paths, reactivation_epoch): (
Vec<FileSystemPath>,
FxHashSet<RcStr>,
u32,
) = {
let map = &this.map_path_to_op.get().0;
let inactive_hmr_paths = &this.inactive_hmr_paths.get().0;
let removed_hmr_paths = &this.removed_hmr_paths.get().0;
......@@ -189,6 +201,7 @@ impl VersionedContentMap {
.iter()
.filter_map(|path| root.get_path_to(path).map(RcStr::from))
.collect(),
*this.reactivation_epoch.get(),
)
};
......@@ -227,6 +240,7 @@ impl VersionedContentMap {
Ok(HmrChunkSnapshot {
active: chunks,
inactive_paths,
reactivation_epoch,
})
}
......@@ -244,7 +258,12 @@ impl VersionedContentMap {
paths: impl IntoIterator<Item = FileSystemPath>,
active: bool,
project_fs: &DiskFileSystem,
) -> Result<()> {
reactivation_source_paths: Vec<PathBuf>,
advance_reactivation_epoch: bool,
) -> Result<u32> {
if advance_reactivation_epoch && !active {
bail!("cannot acknowledge an inactive HMR chunk");
}
let paths = paths.into_iter().collect::<Vec<_>>();
let operations = {
let map = &self.map_path_to_op.get().0;
......@@ -263,6 +282,9 @@ impl VersionedContentMap {
let removed = self.removed_hmr_paths.get_untracked();
paths.iter().any(|path| removed.0.contains(path))
};
if was_removed && reactivation_source_paths.is_empty() {
bail!("re-adding an HMR chunk requires its source path");
}
let reactivated = {
let inactive = self.inactive_output_operations.get_untracked();
operations
......@@ -287,10 +309,15 @@ impl VersionedContentMap {
});
// Deleted routes need conservative catch-up because Turbopack may
// reuse the disconnected operation when the same file is added again.
// Invalidate only the re-added source file. Route discovery already
// observed the file before this call, so invalidating ancestor directory
// reads here would needlessly invalidate unrelated route graphs.
// Ordinary inactive routes never disconnect their output graph.
if was_removed {
project_fs.invalidate_with_reason(|path| invalidation::Initialize {
path: RcStr::from(path.to_string_lossy()),
project_fs.invalidate_path_with_reason(reactivation_source_paths, |path| {
invalidation::Initialize {
path: RcStr::from(path.to_string_lossy()),
}
});
}
// Reconnect and strongly read removed output graphs after catch-up.
......@@ -323,6 +350,12 @@ impl VersionedContentMap {
}
changed
});
if advance_reactivation_epoch {
let mut epoch = self.reactivation_epoch.get_untracked();
*epoch = epoch
.checked_add(1)
.expect("HMR reactivation epoch overflow");
}
} else {
// Keep the endpoint output operations connected so source changes
// continue to invalidate them while the route is inactive. Omitting
......@@ -337,7 +370,7 @@ impl VersionedContentMap {
changed
});
}
Ok(())
Ok(*self.reactivation_epoch.get_untracked())
}
/// Removes deleted routes from aggregate HMR versions and drops their
......
use std::{
borrow::Cow,
collections::BTreeMap,
fs::{canonicalize, create_dir_all},
io::Write,
path::{Path, PathBuf},
......@@ -27,7 +28,7 @@ use next_api::{
},
project::{
DebugBuildPaths, DefineEnv, DraftModeOptions, HmrTarget, PartialProjectOptions, Project,
ProjectContainer, ProjectOptions, WatchOptions,
ProjectContainer, ProjectOptions, WatchOptions, aggregate_hmr_reactivation_epoch,
},
project_asset_hashes_manifest::immutable_hashes_manifest_asset_if_enabled,
route::{Endpoint, EndpointGroupKey, Route},
......@@ -766,7 +767,9 @@ pub async fn project_set_hmr_chunks_active(
chunk_names: Vec<RcStr>,
target: String,
active: bool,
) -> napi::Result<()> {
reactivation_source_paths: Option<Vec<RcStr>>,
advance_reactivation_epoch: Option<bool>,
) -> napi::Result<u32> {
let hmr_target = target
.parse::<HmrTarget>()
.map_err(napi::Error::from_reason)?;
......@@ -776,7 +779,13 @@ pub async fn project_set_hmr_chunks_active(
ctx.turbo_tasks()
.run(async move {
container
.set_hmr_chunks_active(chunk_names, hmr_target, active)
.set_hmr_chunks_active(
chunk_names,
hmr_target,
active,
reactivation_source_paths.unwrap_or_default(),
advance_reactivation_epoch.unwrap_or_default(),
)
.await
})
.or_else(|e| ctx.throw_turbopack_internal_result(&e.into()))
......@@ -1979,19 +1988,23 @@ pub fn project_all_hmr_events(
unmark_top_level_task_may_leak_eventually_consistent_state();
let HmrUpdateWithIssues { update, issues, .. } = &*read;
match &**update {
Update::Missing | Update::None => {}
let reactivation_epoch = match &**update {
Update::Missing | Update::None => None,
Update::Total(TotalUpdate { to }) => {
let epoch = aggregate_hmr_reactivation_epoch(to.clone()).await?;
state.set(to.clone()).await?;
epoch
}
Update::Partial(PartialUpdate { to, .. }) => {
let epoch = aggregate_hmr_reactivation_epoch(to.clone()).await?;
state.set(to.clone()).await?;
epoch
}
}
Ok((Some(update.clone()), issues.clone()))
};
Ok((Some(update.clone()), issues.clone(), reactivation_epoch))
},
move |ctx| {
let (update, issues) = ctx.value;
let (update, issues, reactivation_epoch) = ctx.value;
let napi_issues = issues
.iter()
......@@ -2004,7 +2017,12 @@ pub fn project_all_hmr_events(
let identifier = ResourceIdentifier {
path: identifier_path.clone(),
headers: None,
headers: reactivation_epoch.map(|epoch| {
BTreeMap::from([(
rcstr!("x-next-hmr-reactivation-epoch"),
RcStr::from(epoch.to_string()),
)])
}),
};
let update = match update.as_deref() {
None | Some(Update::Missing) | Some(Update::Total(_)) => {
......
......@@ -341,8 +341,10 @@ export declare function projectSetHmrChunksActive(
project: { __napiType: 'Project' },
chunkNames: Array<RcStr>,
target: string,
active: boolean
): Promise<void>
active: boolean,
reactivationSourcePaths?: Array<RcStr> | undefined | null,
advanceReactivationEpoch?: boolean | undefined | null
): Promise<number>
export declare function projectRemoveHmrChunks(
project: { __napiType: 'Project' },
chunkNames: Array<RcStr>,
......
......@@ -775,13 +775,17 @@ function bindingToApi(
setHmrChunksActive(
chunkNames: string[],
target: HmrTarget.Server,
active: boolean
): Promise<void> {
active: boolean,
reactivationSourcePaths?: string[],
advanceReactivationEpoch?: boolean
): Promise<number> {
return binding.projectSetHmrChunksActive(
this._nativeProject,
chunkNames,
target,
active
active,
reactivationSourcePaths,
advanceReactivationEpoch
)
}
......
......@@ -343,8 +343,10 @@ export interface Project {
setHmrChunksActive(
chunkNames: string[],
target: import('./index').HmrTarget.Server,
active: boolean
): Promise<void>
active: boolean,
reactivationSourcePaths?: string[],
advanceReactivationEpoch?: boolean
): Promise<number>
removeHmrChunks(
chunkNames: string[],
......
......@@ -759,15 +759,24 @@ describe('server-hmr route activity', () => {
await browser.loadPage(`${next.url}/route-activity`)
await waitForRedbox(browser)
await next.patchFile(routeFile, (content) =>
content.replace(
'export default function Page(',
'export default function Page()'
)
content
.replace(
'export default function Page(',
'export default function Page()'
)
.replace('inactive local update', 'recovered route')
)
// The request retry owns the catch-up left behind by the failed compile.
// Its first response must wait for the recovery update to reach the runtime.
const firstRecoveryResponse = await next.fetch('/route-activity')
expect(firstRecoveryResponse.status).toBe(200)
const firstRecoveryHtml = await firstRecoveryResponse.text()
expect(firstRecoveryHtml).toContain('recovered route')
expect(firstRecoveryHtml).toContain('inactive route update')
await retry(async () => {
expect(
await browser.elementByCss('#route-activity-greeting').text()
).toBe('inactive local update: inactive route update')
).toBe('recovered route: inactive route update')
}, 10_000)
// Deleting a retired route must prune its emitted output mappings and frozen
......@@ -783,9 +792,16 @@ describe('server-hmr route activity', () => {
// must rebuild from the current files and install a fresh HMR subscription
// rather than reviving the deleted output snapshot.
await next.patchFile(routeFile, originalRouteSource)
let firstReaddedRouteHtml: string | undefined
await retry(async () => {
expect((await next.fetch('/route-activity')).status).toBe(200)
const response = await next.fetch('/route-activity')
expect(response.status).toBe(200)
firstReaddedRouteHtml = await response.text()
}, 10_000)
// Assert outside retry so a stale first successful response cannot be hidden
// by a later request after the aggregate HMR update has caught up.
expect(firstReaddedRouteHtml).toContain('route activity page')
expect(firstReaddedRouteHtml).toContain('inactive route update')
await browser.loadPage(`${next.url}/route-activity`)
expect(
await browser.elementByCss('#route-activity-greeting').text()
......
......@@ -376,11 +376,12 @@ impl DiskFileSystemInner {
}
/// Invalidates tracked files/directories for `paths` and their children.
/// Also invalidates tracked directory reads for all parent directories to
/// account for file creations/deletions under the deferred subtree.
/// Optionally invalidates tracked directory reads for all parent directories
/// to account for file creations/deletions under a subtree.
fn invalidate_path_and_children_with_reason<R: InvalidationReason + Clone>(
&self,
paths: impl IntoIterator<Item = PathBuf>,
invalidate_parent_dirs: bool,
reason: impl Fn(&Path) -> R + Sync,
) {
let _span =
......@@ -396,10 +397,12 @@ impl DiskFileSystemInner {
let mut parent_dirs_to_invalidate = FxHashSet::default();
for path in paths {
let mut current_parent = path.parent();
while let Some(parent) = current_parent {
parent_dirs_to_invalidate.insert(parent.to_path_buf());
current_parent = parent.parent();
if invalidate_parent_dirs {
let mut current_parent = path.parent();
while let Some(parent) = current_parent {
parent_dirs_to_invalidate.insert(parent.to_path_buf());
current_parent = parent.parent();
}
}
for (invalidated_path, path_invalidators) in
......@@ -504,7 +507,18 @@ impl DiskFileSystem {
reason: impl Fn(&Path) -> R + Sync,
) {
self.inner
.invalidate_path_and_children_with_reason(paths, reason);
.invalidate_path_and_children_with_reason(paths, true, reason);
}
/// Invalidates tracked reads at `paths` without invalidating directory reads
/// for their ancestors. Use this when discovery has already observed the path.
pub fn invalidate_path_with_reason<R: InvalidationReason + Clone>(
&self,
paths: impl IntoIterator<Item = PathBuf>,
reason: impl Fn(&Path) -> R + Sync,
) {
self.inner
.invalidate_path_and_children_with_reason(paths, false, reason);
}
pub async fn start_watching(&self, poll_interval: Option<Duration>) -> Result<()> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment