Unverified Commit 8cfa5510 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Turbopack: extract chunk list version/update into turbopack-ecmascript (#95661)

This moves modules out of `turbopack-browser` and into
`turbopack-ecmascript` so it can be used in the server hmr
implementation as well.

Move the chunk-list version, update, and merged-update wire types out of
`turbopack-browser` into a shared `turbopack-ecmascript::chunk_list`
module so both the browser and node chunking contexts can build on them.
Rename `EcmascriptDevChunkListVersion` to `ChunkListVersion` and make
`update_chunk_list` runtime-agnostic (plain path->`VersionedContent`
maps).

Pure refactor: no behavior change.

<!-- NEXT_JS_LLM_PR -->

---------
Co-authored-by: 's avatarClaude <noreply@anthropic.com>
parent 88155cd4
...@@ -10636,6 +10636,7 @@ dependencies = [ ...@@ -10636,6 +10636,7 @@ dependencies = [
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"serde_json", "serde_json",
"serde_qs",
"smallvec", "smallvec",
"strsim 0.11.1", "strsim 0.11.1",
"swc_core", "swc_core",
......
...@@ -148,6 +148,9 @@ impl VersionedContentMap { ...@@ -148,6 +148,9 @@ impl VersionedContentMap {
impl VersionedContentMap { impl VersionedContentMap {
/// Inserts output assets into the map and returns a completion that when /// Inserts output assets into the map and returns a completion that when
/// awaited will emit the assets that were inserted. /// awaited will emit the assets that were inserted.
//
// TODO: If `VersionedContentMap` becomes transient as described above, these methods should be
// `#[turbo_tasks::function(session_dependent)]`
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn insert_output_assets( pub async fn insert_output_assets(
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
......
...@@ -13,17 +13,17 @@ use turbopack_core::{ ...@@ -13,17 +13,17 @@ use turbopack_core::{
chunk::ChunkingContext, chunk::ChunkingContext,
code_builder::{Code, CodeBuilder}, code_builder::{Code, CodeBuilder},
output::OutputAsset, output::OutputAsset,
version::{ version::{Update, Version, VersionedContent},
MergeableVersionedContent, Update, Version, VersionedContent, VersionedContentMerger, };
use turbopack_ecmascript::{
chunk_list::{
update::update_chunk_list,
version::{ChunkListVersion, compute_chunk_list_version},
}, },
utils::StringifyJs,
}; };
use turbopack_ecmascript::utils::StringifyJs;
use super::{ use super::asset::{EcmascriptDevChunkList, EcmascriptDevChunkListSource};
asset::{EcmascriptDevChunkList, EcmascriptDevChunkListSource},
update::update_chunk_list,
version::EcmascriptDevChunkListVersion,
};
use crate::chunking_context::{ use crate::chunking_context::{
CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod, CURRENT_CHUNK_METHOD_DOCUMENT_CURRENT_SCRIPT_EXPR, CurrentChunkMethod,
}; };
...@@ -102,39 +102,8 @@ impl EcmascriptDevChunkListContent { ...@@ -102,39 +102,8 @@ impl EcmascriptDevChunkListContent {
/// Computes the version of this content. /// Computes the version of this content.
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn version(&self) -> Result<Vc<EcmascriptDevChunkListVersion>> { pub async fn version(&self) -> Result<Vc<ChunkListVersion>> {
let mut by_merger = FxIndexMap::<_, Vec<_>>::default(); compute_chunk_list_version(&self.chunks_contents).await
let mut by_path = FxIndexMap::<_, _>::default();
for (chunk_path, chunk_content) in &self.chunks_contents {
if let Some(mergeable) =
ResolvedVc::try_sidecast::<Box<dyn MergeableVersionedContent>>(*chunk_content)
{
let merger = mergeable.get_merger().to_resolved().await?;
by_merger.entry(merger).or_default().push(*chunk_content);
} else {
by_path.insert(
chunk_path.clone(),
chunk_content.version().into_trait_ref().await?,
);
}
}
let by_merger = by_merger
.into_iter()
.map(|(merger, contents)| (merger, Vc::cell(contents)))
.map(async |(merger, contents)| {
Ok((
merger,
merger.merge(contents).version().into_trait_ref().await?,
))
})
.try_join()
.await?
.into_iter()
.collect();
Ok(EcmascriptDevChunkListVersion { by_path, by_merger }.cell())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -200,6 +169,7 @@ impl VersionedContent for EcmascriptDevChunkListContent { ...@@ -200,6 +169,7 @@ impl VersionedContent for EcmascriptDevChunkListContent {
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
from_version: ResolvedVc<Box<dyn Version>>, from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> { ) -> Result<Vc<Update>> {
update_chunk_list(self, from_version).await let this = self.await?;
update_chunk_list(&this.chunks_contents, self.version(), from_version).await
} }
} }
pub(crate) mod asset; pub(crate) mod asset;
pub(crate) mod content; pub(crate) mod content;
pub(crate) mod update;
pub(crate) mod version;
use std::sync::Arc; use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use serde::Serialize; use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks::{FxIndexMap, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbo_tasks_fs::rope::Rope;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkingContext, ModuleId}, chunk::{ChunkingContext, ModuleId},
code_builder::Code,
output::OutputAsset, output::OutputAsset,
source_map::GenerateSourceMap,
version::{PartialUpdate, TotalUpdate, Update, Version}, version::{PartialUpdate, TotalUpdate, Update, Version},
}; };
use turbopack_ecmascript::chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
};
use super::{ use super::{
super::{ super::{
...@@ -21,87 +21,6 @@ use super::{ ...@@ -21,87 +21,6 @@ use super::{
version::EcmascriptBrowserMergedChunkVersion, version::EcmascriptBrowserMergedChunkVersion,
}; };
#[derive(Serialize, Default)]
#[serde(tag = "type", rename_all = "camelCase")]
struct EcmascriptMergedUpdate<'a> {
/// A map from module id to latest module entry.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
entries: FxIndexMap<ModuleId, EcmascriptModuleEntry>,
/// A map from chunk path to the chunk update.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
chunks: FxIndexMap<&'a str, EcmascriptMergedChunkUpdate>,
}
impl EcmascriptMergedUpdate<'_> {
fn is_empty(&self) -> bool {
self.entries.is_empty() && self.chunks.is_empty()
}
}
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "camelCase")]
enum EcmascriptMergedChunkUpdate {
Added(EcmascriptMergedChunkAdded),
Deleted(EcmascriptMergedChunkDeleted),
Partial(EcmascriptMergedChunkPartial),
}
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
struct EcmascriptMergedChunkAdded {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
modules: FxIndexSet<ModuleId>,
}
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
struct EcmascriptMergedChunkDeleted {
// Technically, this is redundant, since the client will already know all
// modules in the chunk from the previous version. However, it's useful for
// merging updates without access to an initial state.
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
modules: FxIndexSet<ModuleId>,
}
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
struct EcmascriptMergedChunkPartial {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
added: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
deleted: FxIndexSet<ModuleId>,
}
#[derive(Serialize)]
struct EcmascriptModuleEntry {
#[serde(with = "turbo_tasks_fs::rope::ser_as_string")]
code: Rope,
url: String,
#[serde(with = "turbo_tasks_fs::rope::ser_option_as_string")]
map: Option<Rope>,
}
impl EcmascriptModuleEntry {
async fn from_code(id: &ModuleId, code: Vc<Code>, chunk_path: &str) -> Result<Self> {
let map = &*code.generate_source_map().await?;
let map = map.as_content().map(|f| f.content().clone());
/// serde_qs can't serialize a lone enum when it's [serde::untagged].
#[derive(Serialize)]
struct Id<'a> {
id: &'a ModuleId,
}
let id = serde_qs::to_string(&Id { id }).unwrap();
Ok(EcmascriptModuleEntry {
// Cloning a rope is cheap.
code: code.await?.source_code().clone(),
url: format!("{}?{}", chunk_path, id),
map,
})
}
}
/// Helper structure to get a module's hash from multiple different chunk /// Helper structure to get a module's hash from multiple different chunk
/// versions, without having to actually merge the versions into a single /// versions, without having to actually merge the versions into a single
/// hashmap, which would be expensive. /// hashmap, which would be expensive.
......
...@@ -43,6 +43,7 @@ regex = { workspace = true } ...@@ -43,6 +43,7 @@ regex = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true, features = ["raw_value"] } serde_json = { workspace = true, features = ["raw_value"] }
serde_qs = { workspace = true }
swc_ecma_react_compiler = { workspace = true } swc_ecma_react_compiler = { workspace = true }
react_compiler = { workspace = true } react_compiler = { workspace = true }
next-custom-transforms = { workspace = true, default-features = false } next-custom-transforms = { workspace = true, default-features = false }
......
//! Shared wire-format types for merged ecmascript chunk updates.
//!
//! These serde structures are the protocol contract sent to the JS HMR client
//! (see `applyEcmascriptMergedUpdateShared` in the ecmascript runtime). Both the
//! browser and node chunking contexts serialize into exactly this shape, so the
//! definitions live here to keep the two runtimes from drifting apart on the
//! wire format.
//!
//! The turbo-tasks value types (`*MergedChunkContent`, `*MergedChunkVersion`,
//! `*ChunkContentMerger`) cannot be generic and therefore remain per-runtime,
//! but they all build and serialize these shared structs.
use anyhow::Result;
use serde::Serialize;
use turbo_tasks::{FxIndexMap, FxIndexSet, Vc};
use turbo_tasks_fs::rope::Rope;
use turbopack_core::{chunk::ModuleId, code_builder::Code, source_map::GenerateSourceMap};
/// A merged update covering one or more ecmascript chunks that share a merger.
#[derive(Serialize, Default)]
#[serde(
tag = "type",
rename = "EcmascriptMergedUpdate",
rename_all = "camelCase"
)]
pub struct EcmascriptMergedUpdate<'a> {
/// A map from module id to its latest module entry (code + source map url).
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
pub entries: FxIndexMap<ModuleId, EcmascriptModuleEntry>,
/// A map from chunk path to the update for that chunk.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
pub chunks: FxIndexMap<&'a str, EcmascriptMergedChunkUpdate>,
}
impl EcmascriptMergedUpdate<'_> {
pub fn is_empty(&self) -> bool {
self.entries.is_empty() && self.chunks.is_empty()
}
}
/// Per-chunk portion of an [`EcmascriptMergedUpdate`].
#[derive(Serialize)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum EcmascriptMergedChunkUpdate {
Added(EcmascriptMergedChunkAdded),
Deleted(EcmascriptMergedChunkDeleted),
Partial(EcmascriptMergedChunkPartial),
}
/// A chunk that was newly added in this version.
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkAdded {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub modules: FxIndexSet<ModuleId>,
}
/// A chunk that was removed in this version.
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkDeleted {
// Technically, this is redundant, since the client will already know all
// modules in the chunk from the previous version. However, it's useful for
// merging updates without access to an initial state.
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub modules: FxIndexSet<ModuleId>,
}
/// A chunk that was present in both versions and whose module membership
/// changed.
#[derive(Serialize, Default)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkPartial {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub added: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub deleted: FxIndexSet<ModuleId>,
}
/// The code (and source map) for a single module in a merged update.
#[derive(Serialize)]
pub struct EcmascriptModuleEntry {
#[serde(with = "turbo_tasks_fs::rope::ser_as_string")]
pub code: Rope,
pub url: String,
#[serde(with = "turbo_tasks_fs::rope::ser_option_as_string")]
pub map: Option<Rope>,
}
impl EcmascriptModuleEntry {
pub async fn from_code(id: &ModuleId, code: Vc<Code>, chunk_path: &str) -> Result<Self> {
let map = &*code.generate_source_map().await?;
let map = map.as_content().map(|f| f.content().clone());
/// serde_qs can't serialize a lone enum when it's [serde::untagged].
#[derive(Serialize)]
struct Id<'a> {
id: &'a ModuleId,
}
let id = serde_qs::to_string(&Id { id }).unwrap();
Ok(EcmascriptModuleEntry {
// Cloning a rope is cheap.
code: code.await?.source_code().clone(),
url: format!("{}?{}", chunk_path, id),
map,
})
}
}
pub mod merged_update;
pub mod update;
pub mod version;
...@@ -8,7 +8,7 @@ use turbopack_core::version::{ ...@@ -8,7 +8,7 @@ use turbopack_core::version::{
VersionedContentMerger, VersionedContentMerger,
}; };
use super::{content::EcmascriptDevChunkListContent, version::EcmascriptDevChunkListVersion}; use super::version::ChunkListVersion;
/// Update of a chunk list from one version to another. /// Update of a chunk list from one version to another.
#[derive(Serialize)] #[derive(Serialize)]
...@@ -47,24 +47,27 @@ impl ChunkListUpdate<'_> { ...@@ -47,24 +47,27 @@ impl ChunkListUpdate<'_> {
} }
/// Computes the update of a chunk list from one version to another. /// Computes the update of a chunk list from one version to another.
pub(super) async fn update_chunk_list( ///
content: ResolvedVc<EcmascriptDevChunkListContent>, /// Runtime-agnostic (takes plain paths + [`VersionedContent`]) so the browser
/// and node chunking contexts can share one implementation instead of each
/// duplicating the merge-by-[`VersionedContentMerger`] logic.
pub async fn update_chunk_list(
chunks_contents: &FxIndexMap<String, ResolvedVc<Box<dyn VersionedContent>>>,
to_version: Vc<ChunkListVersion>,
from_version: ResolvedVc<Box<dyn Version>>, from_version: ResolvedVc<Box<dyn Version>>,
) -> Result<Vc<Update>> { ) -> Result<Vc<Update>> {
let to_version = content.version(); let from_version =
let from_version = if let Some(from) = if let Some(from) = ResolvedVc::try_downcast_type::<ChunkListVersion>(from_version) {
ResolvedVc::try_downcast_type::<EcmascriptDevChunkListVersion>(from_version) from
{ } else {
from // It's likely `from_version` is `NotFoundVersion`.
} else { return Ok(Update::Total(TotalUpdate {
// It's likely `from_version` is `NotFoundVersion`. to: Vc::upcast::<Box<dyn Version>>(to_version)
return Ok(Update::Total(TotalUpdate { .into_trait_ref()
to: Vc::upcast::<Box<dyn Version>>(to_version) .await?,
.into_trait_ref() })
.await?, .cell());
}) };
.cell());
};
let to = to_version.await?; let to = to_version.await?;
let from = from_version.await?; let from = from_version.await?;
...@@ -76,19 +79,12 @@ pub(super) async fn update_chunk_list( ...@@ -76,19 +79,12 @@ pub(super) async fn update_chunk_list(
return Ok(Update::None.cell()); return Ok(Update::None.cell());
} }
let content = content.await?; // Group mergeable chunks by merger so their updates collapse into one
// `EcmascriptMergedUpdate`; everything else is diffed individually by path.
// There are two kind of updates nested within a chunk list update:
// * merged updates; and
// * single chunk updates.
// In order to compute merged updates, we first need to group mergeable chunks
// by common mergers. Then, we compute the update of each group separately.
// Single chunk updates are computed separately and only require a stable chunk
// path to identify the chunk across versions.
let mut by_merger = FxIndexMap::<_, Vec<_>>::default(); let mut by_merger = FxIndexMap::<_, Vec<_>>::default();
let mut by_path = FxIndexMap::<_, _>::default(); let mut by_path = FxIndexMap::<_, _>::default();
for (chunk_path, chunk_content) in &content.chunks_contents { for (chunk_path, chunk_content) in chunks_contents {
if let Some(mergeable) = if let Some(mergeable) =
ResolvedVc::try_sidecast::<Box<dyn MergeableVersionedContent>>(*chunk_content) ResolvedVc::try_sidecast::<Box<dyn MergeableVersionedContent>>(*chunk_content)
{ {
......
...@@ -2,15 +2,21 @@ use anyhow::Result; ...@@ -2,15 +2,21 @@ use anyhow::Result;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, ResolvedVc, TraitRef, TryJoinIterExt, Vc}; use turbo_tasks::{FxIndexMap, ResolvedVc, TraitRef, TryJoinIterExt, Vc};
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64}; use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_core::version::{Version, VersionedContentMerger}; use turbopack_core::version::{
MergeableVersionedContent, Version, VersionedContent, VersionedContentMerger,
};
type VersionTraitRef = TraitRef<Box<dyn Version>>; type VersionTraitRef = TraitRef<Box<dyn Version>>;
/// The version of a [`EcmascriptDevChunkListContent`]. /// The version of a chunk list content.
/// ///
/// [`EcmascriptDevChunkListContent`]: super::content::EcmascriptDevChunkListContent /// Tracks versions of individual chunks by path and by merger. Chunks that
/// implement [`MergeableVersionedContent`] are grouped by their merger and
/// their versions are merged. Other chunks are tracked by path.
///
/// [`MergeableVersionedContent`]: turbopack_core::version::MergeableVersionedContent
#[turbo_tasks::value(serialization = "skip", shared)] #[turbo_tasks::value(serialization = "skip", shared)]
pub(super) struct EcmascriptDevChunkListVersion { pub struct ChunkListVersion {
/// A map from chunk path to its version. /// A map from chunk path to its version.
#[turbo_tasks(trace_ignore)] #[turbo_tasks(trace_ignore)]
pub by_path: FxIndexMap<String, VersionTraitRef>, pub by_path: FxIndexMap<String, VersionTraitRef>,
...@@ -24,7 +30,7 @@ pub(super) struct EcmascriptDevChunkListVersion { ...@@ -24,7 +30,7 @@ pub(super) struct EcmascriptDevChunkListVersion {
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Version for EcmascriptDevChunkListVersion { impl Version for ChunkListVersion {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn id(&self) -> Result<Vc<RcStr>> { async fn id(&self) -> Result<Vc<RcStr>> {
let by_path = { let by_path = {
...@@ -66,3 +72,48 @@ impl Version for EcmascriptDevChunkListVersion { ...@@ -66,3 +72,48 @@ impl Version for EcmascriptDevChunkListVersion {
Ok(Vc::cell(hash.into())) Ok(Vc::cell(hash.into()))
} }
} }
/// Computes a [`ChunkListVersion`] from a map of chunk paths to their
/// [`VersionedContent`].
///
/// Chunks that implement [`MergeableVersionedContent`] are grouped by their
/// merger and their versions are merged. Other chunks are tracked by path.
///
/// [`VersionedContent`]: turbopack_core::version::VersionedContent
/// [`MergeableVersionedContent`]: turbopack_core::version::MergeableVersionedContent
pub async fn compute_chunk_list_version(
chunks_contents: &FxIndexMap<String, ResolvedVc<Box<dyn VersionedContent>>>,
) -> Result<Vc<ChunkListVersion>> {
let mut by_merger = FxIndexMap::<_, Vec<_>>::default();
let mut by_path = FxIndexMap::<_, _>::default();
for (chunk_path, chunk_content) in chunks_contents {
if let Some(mergeable) =
ResolvedVc::try_sidecast::<Box<dyn MergeableVersionedContent>>(*chunk_content)
{
let merger = mergeable.get_merger().to_resolved().await?;
by_merger.entry(merger).or_default().push(*chunk_content);
} else {
by_path.insert(
chunk_path.clone(),
chunk_content.version().into_trait_ref().await?,
);
}
}
let by_merger = by_merger
.into_iter()
.map(|(merger, contents)| (merger, Vc::cell(contents)))
.map(async |(merger, contents)| {
Ok((
merger,
merger.merge(contents).version().into_trait_ref().await?,
))
})
.try_join()
.await?
.into_iter()
.collect();
Ok(ChunkListVersion { by_path, by_merger }.cell())
}
...@@ -12,6 +12,7 @@ pub mod annotations; ...@@ -12,6 +12,7 @@ pub mod annotations;
pub mod async_chunk; pub mod async_chunk;
pub mod bytes_source_transform; pub mod bytes_source_transform;
pub mod chunk; pub mod chunk;
pub mod chunk_list;
pub mod code_gen; pub mod code_gen;
pub mod embed_js; pub mod embed_js;
mod errors; mod errors;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment