Unverified Commit 1a304e37 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Keep HMR instructions typed until serialization (#96569)

Stacked on #96686.

## What?

Keep HMR update instructions as typed Rust values until they cross the
client protocol serialization boundary. The JSON wire format remains
unchanged.

## Why?

HMR aggregation currently operates on `serde_json::Value`, so it has to
inspect serialized `type` fields and clone values out of JSON objects.
This loses type information before aggregation and makes unsupported
instruction variants easy to overlook.

Keeping instructions typed lets aggregation use concrete equality and
hashing, preserves stable ordering while deduplicating updates, and
falls back to a total update when it encounters an instruction type it
cannot aggregate.

## How?

- Add `UpdateInstructionValue`, a type-erased serializable value with
typed downcasting and equality.
- Represent chunk-list and merged ECMAScript updates with owned,
hashable values backed by frozen collections.
- Aggregate typed instructions with first-seen ordering for merged
updates and last-writer-wins values for chunk updates.
- Serialize instructions only when producing the HMR protocol message.
- Add unit coverage for downcasting, equality, hashing, aggregation
behavior, and the existing JSON protocol shape.

## Verification

- Unit tests cover the new typed values and aggregation behavior.
- Serialization tests confirm the client-facing HMR messages are
unchanged.
- CI

<!-- NEXT_JS_LLM -->
parent f603d4ab
...@@ -10528,6 +10528,7 @@ dependencies = [ ...@@ -10528,6 +10528,7 @@ dependencies = [
"const_format", "const_format",
"data-encoding", "data-encoding",
"either", "either",
"erased-serde",
"indexmap 2.13.0", "indexmap 2.13.0",
"num-bigint", "num-bigint",
"patricia_tree", "patricia_tree",
...@@ -10703,6 +10704,7 @@ dependencies = [ ...@@ -10703,6 +10704,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"turbo-rcstr", "turbo-rcstr",
"turbo-tasks",
"turbopack-cli-utils", "turbopack-cli-utils",
"turbopack-core", "turbopack-core",
] ]
......
use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TraitRef, TryJoinIterExt, Vc, FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TraitRef, TryJoinIterExt, Vc,
...@@ -10,7 +7,14 @@ use turbo_tasks::{ ...@@ -10,7 +7,14 @@ use turbo_tasks::{
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64}; use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_browser::ecmascript::list::content::EcmascriptDevChunkListContent; use turbopack_browser::ecmascript::list::content::EcmascriptDevChunkListContent;
use turbopack_core::version::{PartialUpdate, Update, Version, VersionState, VersionedContent}; use turbopack_core::{
update_instruction::UpdateInstruction,
version::{PartialUpdate, Update, Version, VersionState, VersionedContent},
};
use turbopack_ecmascript::chunk_list::{
merged_update::EcmascriptMergedUpdate,
update::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction},
};
use turbopack_nodejs::ecmascript::node::entry::chunk_list_content::EcmascriptBuildNodeChunkListContent; use turbopack_nodejs::ecmascript::node::entry::chunk_list_content::EcmascriptBuildNodeChunkListContent;
use crate::versioned_content_map::VersionedContentMap; use crate::versioned_content_map::VersionedContentMap;
...@@ -103,38 +107,30 @@ impl AggregateHmrVersion { ...@@ -103,38 +107,30 @@ impl AggregateHmrVersion {
/// Aggregates per-entry HMR instructions into a single combined `ChunkListUpdate`. /// Aggregates per-entry HMR instructions into a single combined `ChunkListUpdate`.
#[derive(Default)] #[derive(Default)]
pub struct ChunkListUpdateBuilder { pub struct ChunkListUpdateBuilder {
chunks: FxHashMap<String, serde_json::Value>, chunks: FxIndexMap<RcStr, ChunkUpdate>,
merged: FxIndexSet<serde_json::Value>, merged: FxIndexSet<EcmascriptMergedUpdate>,
} }
impl ChunkListUpdateBuilder { impl ChunkListUpdateBuilder {
pub fn add_instruction(&mut self, instruction: &serde_json::Value) { pub fn add_instruction(&mut self, instruction: &UpdateInstruction) {
let Some(obj) = instruction.as_object() else { let instruction = instruction
return; .downcast_ref::<EcmascriptUpdateInstruction>()
}; .expect("aggregate HMR only accepts ECMAScript update instructions");
match obj.get("type").and_then(|v| v.as_str()) {
Some("ChunkListUpdate") => { match instruction {
if let Some(chunks) = obj.get("chunks").and_then(|v| v.as_object()) { EcmascriptUpdateInstruction::ChunkList(update) => {
for (k, v) in chunks { for (chunk_path, update) in &update.chunks {
self.chunks.insert(k.clone(), v.clone()); self.chunks.insert(chunk_path.clone(), update.clone());
}
} }
if let Some(merged) = obj.get("merged").and_then(|v| v.as_array()) { for update in &update.merged {
for update in merged {
self.push_merged(update); self.push_merged(update);
} }
} }
} EcmascriptUpdateInstruction::Merged(update) => self.push_merged(update),
Some("EcmascriptMergedUpdate") => {
self.push_merged(instruction);
}
// Unknown instruction shapes are ignored; the caller already
// escalates `Total`/`Missing` updates to a full restart.
_ => {}
} }
} }
fn push_merged(&mut self, update: &serde_json::Value) { fn push_merged(&mut self, update: &EcmascriptMergedUpdate) {
self.merged.insert(update.clone()); self.merged.insert(update.clone());
} }
...@@ -143,26 +139,13 @@ impl ChunkListUpdateBuilder { ...@@ -143,26 +139,13 @@ impl ChunkListUpdateBuilder {
} }
pub fn build(self, to: TraitRef<Box<dyn Version>>) -> Update { pub fn build(self, to: TraitRef<Box<dyn Version>>) -> Update {
let mut instruction = serde_json::Map::new();
instruction.insert(
"type".to_string(),
serde_json::Value::String("ChunkListUpdate".to_string()),
);
if !self.chunks.is_empty() {
instruction.insert(
"chunks".to_string(),
serde_json::Value::Object(self.chunks.into_iter().collect()),
);
}
if !self.merged.is_empty() {
instruction.insert(
"merged".to_string(),
serde_json::Value::Array(self.merged.into_iter().collect()),
);
}
Update::Partial(PartialUpdate { Update::Partial(PartialUpdate {
to, to,
instruction: Arc::new(serde_json::Value::Object(instruction)), instruction: ChunkListUpdate {
chunks: self.chunks,
merged: self.merged.into_iter().collect(),
}
.into_instruction(),
}) })
} }
} }
...@@ -222,3 +205,82 @@ pub async fn diff_chunks_against( ...@@ -222,3 +205,82 @@ pub async fn diff_chunks_against(
has_new_chunks, has_new_chunks,
}) })
} }
#[cfg(test)]
mod tests {
use turbo_tasks::{FxIndexMap, FxIndexSet};
use turbopack_core::update_instruction::UpdateInstruction;
use turbopack_ecmascript::chunk_list::{
merged_update::{
EcmascriptMergedChunkDeleted, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate,
},
update::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction},
};
use super::ChunkListUpdateBuilder;
fn merged(chunk_path: &str) -> EcmascriptMergedUpdate {
EcmascriptMergedUpdate {
entries: Default::default(),
chunks: [(
chunk_path.into(),
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: Default::default(),
}),
)]
.into_iter()
.collect(),
}
}
#[test]
fn deduplicates_merged_updates_in_first_seen_order() {
let first = merged("first.js");
let second = merged("second.js");
let mut builder = ChunkListUpdateBuilder::default();
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(first.clone()),
));
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(second.clone()),
));
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(first.clone()),
));
assert_eq!(builder.merged, FxIndexSet::from_iter([first, second]));
}
#[test]
fn chunk_updates_use_last_writer_and_stable_order() {
let mut builder = ChunkListUpdateBuilder::default();
let first = ChunkListUpdate {
chunks: FxIndexMap::from_iter([
("a.js".into(), ChunkUpdate::Total),
("b.js".into(), ChunkUpdate::Added),
]),
merged: vec![],
};
let second = ChunkListUpdate {
chunks: FxIndexMap::from_iter([
("a.js".into(), ChunkUpdate::Deleted),
("c.js".into(), ChunkUpdate::Total),
]),
merged: vec![],
};
builder.add_instruction(&first.into_instruction());
builder.add_instruction(&second.into_instruction());
assert_eq!(
builder
.chunks
.keys()
.map(|path| path.as_str())
.collect::<Vec<_>>(),
["a.js", "b.js", "c.js"]
);
assert_eq!(builder.chunks["a.js"], ChunkUpdate::Deleted);
}
}
...@@ -2,6 +2,7 @@ use std::{ ...@@ -2,6 +2,7 @@ use std::{
borrow::Cow, borrow::Cow,
cmp::{Ordering, min}, cmp::{Ordering, min},
fmt, fmt,
hash::{Hash, Hasher},
io::{BufRead, Read, Result as IoResult, Write}, io::{BufRead, Read, Result as IoResult, Write},
mem, mem,
ops::{AddAssign, Deref}, ops::{AddAssign, Deref},
...@@ -22,7 +23,7 @@ use bytes::Bytes; ...@@ -22,7 +23,7 @@ use bytes::Bytes;
use futures::Stream; use futures::Stream;
use tokio::io::{AsyncRead, ReadBuf}; use tokio::io::{AsyncRead, ReadBuf};
use triomphe::Arc; use triomphe::Arc;
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher}; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64};
static EMPTY_BUF: &[u8] = &[]; static EMPTY_BUF: &[u8] = &[];
...@@ -491,6 +492,12 @@ impl PartialEq for Rope { ...@@ -491,6 +492,12 @@ impl PartialEq for Rope {
impl Eq for Rope {} impl Eq for Rope {}
impl Hash for Rope {
fn hash<H: Hasher>(&self, state: &mut H) {
hash_xxh3_hash64(self.content_hash()).hash(state);
}
}
impl Ord for Rope { impl Ord for Rope {
fn cmp(&self, other: &Self) -> Ordering { fn cmp(&self, other: &Self) -> Ordering {
if Arc::ptr_eq(&self.data, &other.data) { if Arc::ptr_eq(&self.data, &other.data) {
...@@ -1124,6 +1131,24 @@ mod test { ...@@ -1124,6 +1131,24 @@ mod test {
assert_eq!(hash_xxh3_hash64(rope.content_hash()), hasher.finish()); assert_eq!(hash_xxh3_hash64(rope.content_hash()), hasher.finish());
} }
#[test]
fn standard_hash_uses_content() {
use std::{
collections::hash_map::DefaultHasher,
hash::{Hash, Hasher},
};
let original = Rope::from("same content");
let copied = Rope::from(original.to_bytes().into_owned());
let mut original_hasher = DefaultHasher::new();
let mut copied_hasher = DefaultHasher::new();
original.hash(&mut original_hasher);
copied.hash(&mut copied_hasher);
assert_eq!(original, copied);
assert_eq!(original_hasher.finish(), copied_hasher.finish());
}
#[test] #[test]
fn iteration() { fn iteration() {
let shared = Rope::from("def"); let shared = Rope::from("def");
......
...@@ -36,7 +36,6 @@ enum CurrentChunkMethodWithData { ...@@ -36,7 +36,6 @@ enum CurrentChunkMethodWithData {
DocumentCurrentScript, DocumentCurrentScript,
} }
/// Contents of an [`EcmascriptDevChunkList`].
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct EcmascriptDevChunkListContent { pub struct EcmascriptDevChunkListContent {
current_chunk_method: CurrentChunkMethodWithData, current_chunk_method: CurrentChunkMethodWithData,
......
...@@ -23,6 +23,7 @@ bytes-str = { workspace = true } ...@@ -23,6 +23,7 @@ bytes-str = { workspace = true }
const_format = { workspace = true } const_format = { workspace = true }
data-encoding = { workspace = true } data-encoding = { workspace = true }
either = { workspace = true } either = { workspace = true }
erased-serde = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
num-bigint = "0.4" num-bigint = "0.4"
patricia_tree = { version = "0.10.1", features = ["serde"] } patricia_tree = { version = "0.10.1", features = ["serde"] }
......
...@@ -41,6 +41,7 @@ pub mod source_map; ...@@ -41,6 +41,7 @@ pub mod source_map;
pub mod source_pos; pub mod source_pos;
pub mod source_transform; pub mod source_transform;
pub mod target; pub mod target;
pub mod update_instruction;
mod utils; mod utils;
pub mod version; pub mod version;
pub mod virtual_output; pub mod virtual_output;
......
use std::{any::Any, fmt::Debug, sync::Arc};
use serde::Serialize;
use turbo_tasks::{
NonLocalValue,
debug::ValueDebugFormat,
trace::{TraceRawVcs, TraceRawVcsContext},
};
trait ErasedUpdateInstruction:
erased_serde::Serialize + Debug + Send + Sync + NonLocalValue + 'static
{
fn as_any(&self) -> &dyn Any;
fn dyn_eq(&self, other: &dyn Any) -> bool;
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext);
}
impl<T> ErasedUpdateInstruction for T
where
T: Serialize + Eq + Debug + Send + Sync + NonLocalValue + TraceRawVcs + 'static,
{
fn as_any(&self) -> &dyn Any {
self
}
fn dyn_eq(&self, other: &dyn Any) -> bool {
other.downcast_ref::<Self>() == Some(self)
}
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
TraceRawVcs::trace_raw_vcs(self, trace_context);
}
}
erased_serde::serialize_trait_object!(ErasedUpdateInstruction);
#[derive(Clone, Debug, Serialize, ValueDebugFormat, NonLocalValue)]
#[serde(transparent)]
pub struct UpdateInstruction(Arc<dyn ErasedUpdateInstruction>);
impl PartialEq for UpdateInstruction {
fn eq(&self, other: &Self) -> bool {
self.0.dyn_eq(other.0.as_any())
}
}
impl Eq for UpdateInstruction {}
impl UpdateInstruction {
pub fn new<T>(instruction: T) -> Self
where
T: Serialize + Eq + Debug + Send + Sync + NonLocalValue + TraceRawVcs + 'static,
{
Self(Arc::new(instruction))
}
pub fn downcast_ref<T: 'static>(&self) -> Option<&T> {
self.0.as_any().downcast_ref()
}
}
impl TraceRawVcs for UpdateInstruction {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
ErasedUpdateInstruction::trace_raw_vcs(self.0.as_ref(), trace_context);
}
}
#[cfg(test)]
mod tests {
use serde::Serialize;
use turbo_tasks::{NonLocalValue, trace::TraceRawVcs};
use super::UpdateInstruction;
#[derive(Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
struct TestInstruction {
value: u32,
}
#[test]
fn serializes_without_an_extra_wrapper() {
let instruction = UpdateInstruction::new(TestInstruction { value: 42 });
assert_eq!(
serde_json::to_value(&instruction).unwrap(),
serde_json::json!({ "value": 42 })
);
}
#[test]
fn downcasts_by_concrete_type() {
let instruction = UpdateInstruction::new(TestInstruction { value: 42 });
assert_eq!(
instruction
.downcast_ref::<TestInstruction>()
.map(|instruction| instruction.value),
Some(42)
);
assert_eq!(
instruction,
UpdateInstruction::new(TestInstruction { value: 42 })
);
}
}
use std::sync::Arc;
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
...@@ -213,8 +211,7 @@ pub struct PartialUpdate { ...@@ -213,8 +211,7 @@ pub struct PartialUpdate {
pub to: TraitRef<Box<dyn Version>>, pub to: TraitRef<Box<dyn Version>>,
/// The instructions to be passed to a remote system in order to update the /// The instructions to be passed to a remote system in order to update the
/// versioned object. /// versioned object.
#[turbo_tasks(trace_ignore)] pub instruction: crate::update_instruction::UpdateInstruction,
pub instruction: Arc<serde_json::Value>,
} }
/// [`Version`] implementation that hashes a file at a given path and returns /// [`Version`] implementation that hashes a file at a given path and returns
......
...@@ -14,8 +14,11 @@ workspace = true ...@@ -14,8 +14,11 @@ workspace = true
[dependencies] [dependencies]
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true }
turbo-rcstr = { workspace = true } turbo-rcstr = { workspace = true }
turbopack-cli-utils = { workspace = true } turbopack-cli-utils = { workspace = true }
turbopack-core = { workspace = true } turbopack-core = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
turbo-tasks = { workspace = true }
use std::{collections::BTreeMap, fmt::Display, path::PathBuf}; use std::{collections::BTreeMap, fmt::Display, path::PathBuf};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbopack_cli_utils::issue::{LogOptions, format_issue}; use turbopack_cli_utils::issue::{LogOptions, format_issue};
use turbopack_core::{ use turbopack_core::{
issue::{IssueSeverity, IssueStage, PlainIssue, StyledString}, issue::{IssueSeverity, IssueStage, PlainIssue, StyledString},
source_pos::SourcePos, source_pos::SourcePos,
update_instruction::UpdateInstruction,
}; };
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
...@@ -78,7 +78,7 @@ impl<'a> ClientUpdateInstruction<'a> { ...@@ -78,7 +78,7 @@ impl<'a> ClientUpdateInstruction<'a> {
pub fn partial( pub fn partial(
resource: &'a ResourceIdentifier, resource: &'a ResourceIdentifier,
instruction: &'a Value, instruction: &'a UpdateInstruction,
issues: &'a [Issue<'a>], issues: &'a [Issue<'a>],
) -> Self { ) -> Self {
Self::new( Self::new(
...@@ -106,7 +106,7 @@ impl<'a> ClientUpdateInstruction<'a> { ...@@ -106,7 +106,7 @@ impl<'a> ClientUpdateInstruction<'a> {
pub enum ClientUpdateInstructionType<'a> { pub enum ClientUpdateInstructionType<'a> {
Restart, Restart,
NotFound, NotFound,
Partial { instruction: &'a Value }, Partial { instruction: &'a UpdateInstruction },
Issues, Issues,
} }
...@@ -187,3 +187,50 @@ impl<'a> From<&'a PlainIssue> for Issue<'a> { ...@@ -187,3 +187,50 @@ impl<'a> From<&'a PlainIssue> for Issue<'a> {
} }
} }
} }
#[cfg(test)]
mod tests {
use serde::Serialize;
use serde_json::json;
use turbo_rcstr::rcstr;
use turbo_tasks::{NonLocalValue, trace::TraceRawVcs};
use turbopack_core::update_instruction::UpdateInstruction;
use super::{ClientUpdateInstruction, ResourceIdentifier};
#[derive(Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
struct TestInstruction(serde_json::Value);
#[test]
fn partial_instruction_wire_format_is_unchanged() {
let resource = ResourceIdentifier {
path: rcstr!("server/app.js"),
headers: None,
};
let instruction = UpdateInstruction::new(TestInstruction(json!({
"type": "ecmascriptMerged",
"chunks": {},
})));
assert_eq!(
serde_json::to_value(ClientUpdateInstruction::partial(
&resource,
&instruction,
&[],
))
.unwrap(),
json!({
"resource": {
"path": "server/app.js",
"headers": null,
},
"type": "partial",
"instruction": {
"type": "ecmascriptMerged",
"chunks": {},
},
"issues": [],
})
);
}
}
...@@ -12,34 +12,36 @@ ...@@ -12,34 +12,36 @@
use anyhow::Result; use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use turbo_tasks::{FxIndexMap, FxIndexSet, Vc}; use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::rope::Rope; use turbo_tasks_fs::rope::Rope;
use turbopack_core::{chunk::ModuleId, code_builder::Code, source_map::GenerateSourceMap}; use turbopack_core::{chunk::ModuleId, code_builder::Code, source_map::GenerateSourceMap};
/// A merged update covering one or more ecmascript chunks that share a merger. /// A merged update covering one or more ecmascript chunks that share a merger.
#[derive(Serialize, Default)] #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde( #[serde(
tag = "type", tag = "type",
rename = "EcmascriptMergedUpdate", rename = "EcmascriptMergedUpdate",
rename_all = "camelCase" rename_all = "camelCase"
)] )]
pub struct EcmascriptMergedUpdate<'a> { pub struct EcmascriptMergedUpdate {
/// A map from module id to its latest module entry (code + source map url). /// A map from module id to its latest module entry (code + source map url).
#[serde(skip_serializing_if = "FxIndexMap::is_empty")] #[serde(skip_serializing_if = "FrozenMap::is_empty")]
pub entries: FxIndexMap<ModuleId, EcmascriptModuleEntry>, pub entries: FrozenMap<ModuleId, EcmascriptModuleEntry>,
/// A map from chunk path to the update for that chunk. /// A map from chunk path to the update for that chunk.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")] #[serde(skip_serializing_if = "FrozenMap::is_empty")]
pub chunks: FxIndexMap<&'a str, EcmascriptMergedChunkUpdate>, pub chunks: FrozenMap<RcStr, EcmascriptMergedChunkUpdate>,
} }
impl EcmascriptMergedUpdate<'_> { impl EcmascriptMergedUpdate {
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
self.entries.is_empty() && self.chunks.is_empty() self.entries.is_empty() && self.chunks.is_empty()
} }
} }
/// Per-chunk portion of an [`EcmascriptMergedUpdate`]. /// Per-chunk portion of an [`EcmascriptMergedUpdate`].
#[derive(Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type", rename_all = "camelCase")] #[serde(tag = "type", rename_all = "camelCase")]
pub enum EcmascriptMergedChunkUpdate { pub enum EcmascriptMergedChunkUpdate {
Added(EcmascriptMergedChunkAdded), Added(EcmascriptMergedChunkAdded),
...@@ -48,41 +50,41 @@ pub enum EcmascriptMergedChunkUpdate { ...@@ -48,41 +50,41 @@ pub enum EcmascriptMergedChunkUpdate {
} }
/// A chunk that was newly added in this version. /// A chunk that was newly added in this version.
#[derive(Serialize, Default)] #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkAdded { pub struct EcmascriptMergedChunkAdded {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")] #[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub modules: FxIndexSet<ModuleId>, pub modules: FrozenSet<ModuleId>,
} }
/// A chunk that was removed in this version. /// A chunk that was removed in this version.
#[derive(Serialize, Default)] #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkDeleted { pub struct EcmascriptMergedChunkDeleted {
// Technically, this is redundant, since the client will already know all // Technically, this is redundant, since the client will already know all
// modules in the chunk from the previous version. However, it's useful for // modules in the chunk from the previous version. However, it's useful for
// merging updates without access to an initial state. // merging updates without access to an initial state.
#[serde(skip_serializing_if = "FxIndexSet::is_empty")] #[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub modules: FxIndexSet<ModuleId>, pub modules: FrozenSet<ModuleId>,
} }
/// A chunk that was present in both versions and whose module membership /// A chunk that was present in both versions and whose module membership
/// changed. /// changed.
#[derive(Serialize, Default)] #[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkPartial { pub struct EcmascriptMergedChunkPartial {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")] #[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub added: FxIndexSet<ModuleId>, pub added: FrozenSet<ModuleId>,
#[serde(skip_serializing_if = "FxIndexSet::is_empty")] #[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub deleted: FxIndexSet<ModuleId>, pub deleted: FrozenSet<ModuleId>,
} }
/// The code (and source map) for a single module in a merged update. /// The code (and source map) for a single module in a merged update.
#[derive(Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
pub struct EcmascriptModuleEntry { pub struct EcmascriptModuleEntry {
#[serde(with = "turbo_tasks_fs::rope::ser_as_string")] #[serde(with = "turbo_tasks_fs::rope::ser_as_string")]
pub code: Rope, pub code: Rope,
pub url: String, pub url: RcStr,
#[serde(with = "turbo_tasks_fs::rope::ser_option_as_string")] #[serde(with = "turbo_tasks_fs::rope::ser_option_as_string")]
pub map: Option<Rope>, pub map: Option<Rope>,
} }
...@@ -102,7 +104,7 @@ impl EcmascriptModuleEntry { ...@@ -102,7 +104,7 @@ impl EcmascriptModuleEntry {
Ok(EcmascriptModuleEntry { Ok(EcmascriptModuleEntry {
// Cloning a rope is cheap. // Cloning a rope is cheap.
code: code.await?.source_code().clone(), code: code.await?.source_code().clone(),
url: format!("{}?{}", chunk_path, id), url: format!("{}?{}", chunk_path, id).into(),
map, map,
}) })
} }
......
use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use serde::Serialize; use serde::Serialize;
use turbo_tasks::{FxIndexMap, ResolvedVc, TraitRef, Vc}; use turbo_rcstr::RcStr;
use turbopack_core::version::{ use turbo_tasks::{FxIndexMap, NonLocalValue, ResolvedVc, TraitRef, Vc, trace::TraceRawVcs};
use turbopack_core::{
update_instruction::UpdateInstruction,
version::{
MergeableVersionedContent, PartialUpdate, TotalUpdate, Update, Version, VersionedContent, MergeableVersionedContent, PartialUpdate, TotalUpdate, Update, Version, VersionedContent,
VersionedContentMerger, VersionedContentMerger,
},
}; };
use super::version::ChunkListVersion; use super::{merged_update::EcmascriptMergedUpdate, version::ChunkListVersion};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(untagged)]
pub enum EcmascriptUpdateInstruction {
ChunkList(ChunkListUpdate),
Merged(EcmascriptMergedUpdate),
}
/// Update of a chunk list from one version to another. /// Update of a chunk list from one version to another.
#[derive(Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type")] #[serde(tag = "type", rename = "ChunkListUpdate", rename_all = "camelCase")]
#[serde(rename_all = "camelCase")] pub struct ChunkListUpdate {
struct ChunkListUpdate<'a> {
/// A map from chunk path to a corresponding update of that chunk. /// A map from chunk path to a corresponding update of that chunk.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")] #[serde(skip_serializing_if = "FxIndexMap::is_empty")]
chunks: FxIndexMap<&'a str, ChunkUpdate>, pub chunks: FxIndexMap<RcStr, ChunkUpdate>,
/// List of merged updates since the last version. /// List of merged updates since the last version.
#[serde(skip_serializing_if = "Vec::is_empty")] #[serde(skip_serializing_if = "Vec::is_empty")]
merged: Vec<Arc<serde_json::Value>>, pub merged: Vec<EcmascriptMergedUpdate>,
}
impl ChunkListUpdate {
pub fn into_instruction(self) -> UpdateInstruction {
UpdateInstruction::new(EcmascriptUpdateInstruction::ChunkList(self))
}
} }
/// Update of a chunk from one version to another. /// Update of a chunk from one version to another.
#[derive(Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type")] #[serde(tag = "type")]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
enum ChunkUpdate { pub enum ChunkUpdate {
/// The chunk was updated and must be reloaded. /// The chunk was updated and must be reloaded.
Total, Total,
/// The chunk was updated and can be merged with the previous version. /// The chunk was updated and can be merged with the previous version.
Partial { instruction: Arc<serde_json::Value> }, Partial { instruction: EcmascriptMergedUpdate },
/// The chunk was added. /// The chunk was added.
Added, Added,
/// The chunk was deleted. /// The chunk was deleted.
Deleted, Deleted,
} }
impl ChunkListUpdate<'_> {
/// Returns `true` if this update is empty.
fn is_empty(&self) -> bool {
let ChunkListUpdate { chunks, merged } = self;
chunks.is_empty() && merged.is_empty()
}
}
/// Computes the update of a chunk list from one version to another. /// Computes the update of a chunk list from one version to another.
/// ///
/// Runtime-agnostic (takes plain paths + [`VersionedContent`]) so the browser /// Runtime-agnostic (takes plain paths + [`VersionedContent`]) so the browser
...@@ -105,25 +111,26 @@ pub async fn update_chunk_list( ...@@ -105,25 +111,26 @@ pub async fn update_chunk_list(
match &*chunk_update { match &*chunk_update {
Update::Total(_) => { Update::Total(_) => {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Total); chunks.insert(chunk_path.clone().into(), ChunkUpdate::Total);
} }
Update::Partial(partial) => { Update::Partial(partial) => {
let instruction = expect_merged_instruction_from_partial(partial);
chunks.insert( chunks.insert(
chunk_path.as_ref(), chunk_path.clone().into(),
ChunkUpdate::Partial { ChunkUpdate::Partial {
instruction: partial.instruction.clone(), instruction: instruction.clone(),
}, },
); );
} }
Update::Missing | Update::None => {} Update::Missing | Update::None => {}
} }
} else { } else {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Deleted); chunks.insert(chunk_path.clone().into(), ChunkUpdate::Deleted);
} }
} }
for chunk_path in by_path.keys() { for chunk_path in by_path.keys() {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Added); chunks.insert((*chunk_path).clone().into(), ChunkUpdate::Added);
} }
let mut merged = vec![]; let mut merged = vec![];
...@@ -147,24 +154,73 @@ pub async fn update_chunk_list( ...@@ -147,24 +154,73 @@ pub async fn update_chunk_list(
.cell()); .cell());
} }
Update::Partial(partial) => { Update::Partial(partial) => {
merged.push(partial.instruction.clone()); let instruction = expect_merged_instruction_from_partial(partial);
merged.push(instruction.clone());
} }
Update::Missing | Update::None => {} Update::Missing | Update::None => {}
} }
} }
} }
let update = ChunkListUpdate { chunks, merged }; let update = if chunks.is_empty() && merged.is_empty() {
let update = if update.is_empty() {
Update::None Update::None
} else { } else {
Update::Partial(PartialUpdate { Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_version) to: Vc::upcast::<Box<dyn Version>>(to_version)
.into_trait_ref() .into_trait_ref()
.await?, .await?,
instruction: Arc::new(serde_json::to_value(&update)?), instruction: ChunkListUpdate { chunks, merged }.into_instruction(),
}) })
}; };
Ok(update.cell()) Ok(update.cell())
} }
fn expect_merged_instruction_from_partial(partial: &PartialUpdate) -> &EcmascriptMergedUpdate {
let Some(EcmascriptUpdateInstruction::Merged(instruction)) = partial
.instruction
.downcast_ref::<EcmascriptUpdateInstruction>(
) else {
panic!("ECMAScript partial updates must contain a merged instruction");
};
instruction
}
#[cfg(test)]
mod tests {
use serde_json::json;
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_tasks::FxIndexMap;
use super::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction};
use crate::chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate,
};
#[test]
fn instruction_wire_format() {
let instruction = EcmascriptUpdateInstruction::ChunkList(ChunkListUpdate {
chunks: FxIndexMap::from_iter([("app.js".into(), ChunkUpdate::Total)]),
merged: vec![EcmascriptMergedUpdate {
entries: FrozenMap::default(),
chunks: FrozenMap::from_iter([(
"app.js".into(),
EcmascriptMergedChunkUpdate::Added(EcmascriptMergedChunkAdded {
modules: FrozenSet::default(),
}),
)]),
}],
});
assert_eq!(
serde_json::to_value(instruction).unwrap(),
json!({
"type": "ChunkListUpdate",
"chunks": { "app.js": { "type": "total" } },
"merged": [{
"type": "EcmascriptMergedUpdate",
"chunks": { "app.js": { "type": "added" } },
}],
})
);
}
}
use std::sync::Arc;
use anyhow::Result; use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc}; use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_tasks::{FxIndexMap, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{ use turbopack_core::{
chunk::ModuleId, chunk::ModuleId,
code_builder::Code, code_builder::Code,
update_instruction::UpdateInstruction,
version::{PartialUpdate, TotalUpdate, Update, Version}, version::{PartialUpdate, TotalUpdate, Update, Version},
}; };
use crate::{ use crate::{
chunk::EcmascriptChunkContentEntries, chunk::EcmascriptChunkContentEntries,
chunk_list::merged_update::{ chunk_list::{
merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial, EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
}, },
update::EcmascriptUpdateInstruction,
},
hmr::{ hmr::{
EcmascriptHmrChunkContent, EcmascriptHmrChunkContent,
content::EcmascriptMergedChunkContent, content::EcmascriptMergedChunkContent,
...@@ -145,21 +148,24 @@ async fn partial_chunk_update( ...@@ -145,21 +148,24 @@ async fn partial_chunk_update(
unreachable!("caller filters out EcmascriptChunkUpdate::None"); unreachable!("caller filters out EcmascriptChunkUpdate::None");
}; };
let mut partial = EcmascriptMergedChunkPartial::default(); let mut added_modules = FxIndexSet::default();
for (id, AddedModule { hash, code }) in added { for (id, AddedModule { hash, code }) in added {
partial.added.insert(id.clone()); added_modules.insert(id.clone());
insert_entry_unless_shipped(entries, from_versions, id, hash, *code, chunk_path).await?; insert_entry_unless_shipped(entries, from_versions, id, hash, *code, chunk_path).await?;
} }
partial.deleted.extend(deleted.into_keys());
for (id, code) in modified { for (id, code) in modified {
let entry = EcmascriptModuleEntry::from_code(&id, *code, chunk_path).await?; let entry = EcmascriptModuleEntry::from_code(&id, *code, chunk_path).await?;
entries.insert(id, entry); entries.insert(id, entry);
} }
Ok(EcmascriptMergedChunkUpdate::Partial(partial)) Ok(EcmascriptMergedChunkUpdate::Partial(
EcmascriptMergedChunkPartial {
added: FrozenSet::from(added_modules),
deleted: deleted.into_keys().collect(),
},
))
} }
/// Builds the payload for a chunk that wasn't present in the previous version. /// Builds the payload for a chunk that wasn't present in the previous version.
...@@ -169,10 +175,10 @@ async fn added_chunk_update( ...@@ -169,10 +175,10 @@ async fn added_chunk_update(
from_versions: &[ReadRef<EcmascriptChunkVersion>], from_versions: &[ReadRef<EcmascriptChunkVersion>],
entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>, entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>,
) -> Result<EcmascriptMergedChunkUpdate> { ) -> Result<EcmascriptMergedChunkUpdate> {
let mut added = EcmascriptMergedChunkAdded::default(); let mut modules = FxIndexSet::default();
for (id, entry) in chunk_entries.iter() { for (id, entry) in chunk_entries.iter() {
added.modules.insert(id.clone()); modules.insert(id.clone());
insert_entry_unless_shipped( insert_entry_unless_shipped(
entries, entries,
from_versions, from_versions,
...@@ -184,7 +190,11 @@ async fn added_chunk_update( ...@@ -184,7 +190,11 @@ async fn added_chunk_update(
.await?; .await?;
} }
Ok(EcmascriptMergedChunkUpdate::Added(added)) Ok(EcmascriptMergedChunkUpdate::Added(
EcmascriptMergedChunkAdded {
modules: FrozenSet::from(modules),
},
))
} }
/// Computes a single [`Update`] covering every chunk in a merged chunk content. /// Computes a single [`Update`] covering every chunk in a merged chunk content.
...@@ -235,7 +245,8 @@ pub async fn update_ecmascript_merged_chunk( ...@@ -235,7 +245,8 @@ pub async fn update_ecmascript_merged_chunk(
.try_join() .try_join()
.await?; .await?;
let mut merged_update = EcmascriptMergedUpdate::default(); let mut merged_entries = FxIndexMap::default();
let mut chunks = FxIndexMap::default();
for (content, entries, to_version) in &to_contents { for (content, entries, to_version) in &to_contents {
let chunk_path = to_version.chunk_path.as_str(); let chunk_path = to_version.chunk_path.as_str();
...@@ -247,40 +258,34 @@ pub async fn update_ecmascript_merged_chunk( ...@@ -247,40 +258,34 @@ pub async fn update_ecmascript_merged_chunk(
{ {
EcmascriptChunkUpdate::None => continue, EcmascriptChunkUpdate::None => continue,
update => { update => {
partial_chunk_update( partial_chunk_update(update, chunk_path, from_versions, &mut merged_entries)
update,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await? .await?
} }
} }
} }
None => { None => {
added_chunk_update( added_chunk_update(entries, chunk_path, from_versions, &mut merged_entries).await?
entries,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await?
} }
}; };
merged_update.chunks.insert(chunk_path, chunk_update); chunks.insert(to_version.chunk_path.clone(), chunk_update);
} }
for (chunk_path, chunk_version) in from_versions_by_chunk_path { for chunk_version in from_versions_by_chunk_path.into_values() {
let hashes = &chunk_version.entries_hashes; let hashes = &chunk_version.entries_hashes;
merged_update.chunks.insert( chunks.insert(
chunk_path, chunk_version.chunk_path.clone(),
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted { EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: hashes.keys().cloned().collect(), modules: hashes.keys().cloned().collect(),
}), }),
); );
} }
let merged_update = EcmascriptMergedUpdate {
entries: FrozenMap::from(merged_entries),
chunks: FrozenMap::from(chunks),
};
Ok(if merged_update.is_empty() { Ok(if merged_update.is_empty() {
Update::None Update::None
} else { } else {
...@@ -288,7 +293,7 @@ pub async fn update_ecmascript_merged_chunk( ...@@ -288,7 +293,7 @@ pub async fn update_ecmascript_merged_chunk(
to: Vc::upcast::<Box<dyn Version>>(to_merged_version) to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref() .into_trait_ref()
.await?, .await?,
instruction: Arc::new(serde_json::to_value(&merged_update)?), instruction: UpdateInstruction::new(EcmascriptUpdateInstruction::Merged(merged_update)),
}) })
}) })
} }
...@@ -100,7 +100,7 @@ impl EcmascriptBuildNodeChunkListContent { ...@@ -100,7 +100,7 @@ impl EcmascriptBuildNodeChunkListContent {
/// Builds a chunk list content directly from a fixed set of `chunks`, /// Builds a chunk list content directly from a fixed set of `chunks`,
/// without expanding async-loader references. Used by /// without expanding async-loader references. Used by
/// [`super::chunk_list::EcmascriptBuildNodeChunkList`] to track chunks /// `super::chunk_list::EcmascriptBuildNodeChunkList` to track chunks
/// (e.g. client-component SSR chunks) that are already fully enumerated by /// (e.g. client-component SSR chunks) that are already fully enumerated by
/// the caller. /// the caller.
#[turbo_tasks::function] #[turbo_tasks::function]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment