Unverified Commit 1a304e37 authored by Will Binns-Smith's avatar Will Binns-Smith Committed by GitHub

Keep HMR instructions typed until serialization (#96569)

Stacked on #96686.

## What?

Keep HMR update instructions as typed Rust values until they cross the
client protocol serialization boundary. The JSON wire format remains
unchanged.

## Why?

HMR aggregation currently operates on `serde_json::Value`, so it has to
inspect serialized `type` fields and clone values out of JSON objects.
This loses type information before aggregation and makes unsupported
instruction variants easy to overlook.

Keeping instructions typed lets aggregation use concrete equality and
hashing, preserves stable ordering while deduplicating updates, and
falls back to a total update when it encounters an instruction type it
cannot aggregate.

## How?

- Add `UpdateInstructionValue`, a type-erased serializable value with
typed downcasting and equality.
- Represent chunk-list and merged ECMAScript updates with owned,
hashable values backed by frozen collections.
- Aggregate typed instructions with first-seen ordering for merged
updates and last-writer-wins values for chunk updates.
- Serialize instructions only when producing the HMR protocol message.
- Add unit coverage for downcasting, equality, hashing, aggregation
behavior, and the existing JSON protocol shape.

## Verification

- Unit tests cover the new typed values and aggregation behavior.
- Serialization tests confirm the client-facing HMR messages are
unchanged.
- CI

<!-- NEXT_JS_LLM -->
parent f603d4ab
......@@ -10528,6 +10528,7 @@ dependencies = [
"const_format",
"data-encoding",
"either",
"erased-serde",
"indexmap 2.13.0",
"num-bigint",
"patricia_tree",
......@@ -10703,6 +10704,7 @@ dependencies = [
"serde",
"serde_json",
"turbo-rcstr",
"turbo-tasks",
"turbopack-cli-utils",
"turbopack-core",
]
......
use std::sync::Arc;
use anyhow::Result;
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TraitRef, TryJoinIterExt, Vc,
......@@ -10,7 +7,14 @@ use turbo_tasks::{
use turbo_tasks_fs::FileSystemPath;
use turbo_tasks_hash::{Xxh3Hash64Hasher, encode_base64};
use turbopack_browser::ecmascript::list::content::EcmascriptDevChunkListContent;
use turbopack_core::version::{PartialUpdate, Update, Version, VersionState, VersionedContent};
use turbopack_core::{
update_instruction::UpdateInstruction,
version::{PartialUpdate, Update, Version, VersionState, VersionedContent},
};
use turbopack_ecmascript::chunk_list::{
merged_update::EcmascriptMergedUpdate,
update::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction},
};
use turbopack_nodejs::ecmascript::node::entry::chunk_list_content::EcmascriptBuildNodeChunkListContent;
use crate::versioned_content_map::VersionedContentMap;
......@@ -103,38 +107,30 @@ impl AggregateHmrVersion {
/// Aggregates per-entry HMR instructions into a single combined `ChunkListUpdate`.
#[derive(Default)]
pub struct ChunkListUpdateBuilder {
chunks: FxHashMap<String, serde_json::Value>,
merged: FxIndexSet<serde_json::Value>,
chunks: FxIndexMap<RcStr, ChunkUpdate>,
merged: FxIndexSet<EcmascriptMergedUpdate>,
}
impl ChunkListUpdateBuilder {
pub fn add_instruction(&mut self, instruction: &serde_json::Value) {
let Some(obj) = instruction.as_object() else {
return;
};
match obj.get("type").and_then(|v| v.as_str()) {
Some("ChunkListUpdate") => {
if let Some(chunks) = obj.get("chunks").and_then(|v| v.as_object()) {
for (k, v) in chunks {
self.chunks.insert(k.clone(), v.clone());
}
pub fn add_instruction(&mut self, instruction: &UpdateInstruction) {
let instruction = instruction
.downcast_ref::<EcmascriptUpdateInstruction>()
.expect("aggregate HMR only accepts ECMAScript update instructions");
match instruction {
EcmascriptUpdateInstruction::ChunkList(update) => {
for (chunk_path, update) in &update.chunks {
self.chunks.insert(chunk_path.clone(), update.clone());
}
if let Some(merged) = obj.get("merged").and_then(|v| v.as_array()) {
for update in merged {
self.push_merged(update);
}
for update in &update.merged {
self.push_merged(update);
}
}
Some("EcmascriptMergedUpdate") => {
self.push_merged(instruction);
}
// Unknown instruction shapes are ignored; the caller already
// escalates `Total`/`Missing` updates to a full restart.
_ => {}
EcmascriptUpdateInstruction::Merged(update) => self.push_merged(update),
}
}
fn push_merged(&mut self, update: &serde_json::Value) {
fn push_merged(&mut self, update: &EcmascriptMergedUpdate) {
self.merged.insert(update.clone());
}
......@@ -143,26 +139,13 @@ impl ChunkListUpdateBuilder {
}
pub fn build(self, to: TraitRef<Box<dyn Version>>) -> Update {
let mut instruction = serde_json::Map::new();
instruction.insert(
"type".to_string(),
serde_json::Value::String("ChunkListUpdate".to_string()),
);
if !self.chunks.is_empty() {
instruction.insert(
"chunks".to_string(),
serde_json::Value::Object(self.chunks.into_iter().collect()),
);
}
if !self.merged.is_empty() {
instruction.insert(
"merged".to_string(),
serde_json::Value::Array(self.merged.into_iter().collect()),
);
}
Update::Partial(PartialUpdate {
to,
instruction: Arc::new(serde_json::Value::Object(instruction)),
instruction: ChunkListUpdate {
chunks: self.chunks,
merged: self.merged.into_iter().collect(),
}
.into_instruction(),
})
}
}
......@@ -222,3 +205,82 @@ pub async fn diff_chunks_against(
has_new_chunks,
})
}
#[cfg(test)]
mod tests {
use turbo_tasks::{FxIndexMap, FxIndexSet};
use turbopack_core::update_instruction::UpdateInstruction;
use turbopack_ecmascript::chunk_list::{
merged_update::{
EcmascriptMergedChunkDeleted, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate,
},
update::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction},
};
use super::ChunkListUpdateBuilder;
fn merged(chunk_path: &str) -> EcmascriptMergedUpdate {
EcmascriptMergedUpdate {
entries: Default::default(),
chunks: [(
chunk_path.into(),
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: Default::default(),
}),
)]
.into_iter()
.collect(),
}
}
#[test]
fn deduplicates_merged_updates_in_first_seen_order() {
let first = merged("first.js");
let second = merged("second.js");
let mut builder = ChunkListUpdateBuilder::default();
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(first.clone()),
));
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(second.clone()),
));
builder.add_instruction(&UpdateInstruction::new(
EcmascriptUpdateInstruction::Merged(first.clone()),
));
assert_eq!(builder.merged, FxIndexSet::from_iter([first, second]));
}
#[test]
fn chunk_updates_use_last_writer_and_stable_order() {
let mut builder = ChunkListUpdateBuilder::default();
let first = ChunkListUpdate {
chunks: FxIndexMap::from_iter([
("a.js".into(), ChunkUpdate::Total),
("b.js".into(), ChunkUpdate::Added),
]),
merged: vec![],
};
let second = ChunkListUpdate {
chunks: FxIndexMap::from_iter([
("a.js".into(), ChunkUpdate::Deleted),
("c.js".into(), ChunkUpdate::Total),
]),
merged: vec![],
};
builder.add_instruction(&first.into_instruction());
builder.add_instruction(&second.into_instruction());
assert_eq!(
builder
.chunks
.keys()
.map(|path| path.as_str())
.collect::<Vec<_>>(),
["a.js", "b.js", "c.js"]
);
assert_eq!(builder.chunks["a.js"], ChunkUpdate::Deleted);
}
}
......@@ -2,6 +2,7 @@ use std::{
borrow::Cow,
cmp::{Ordering, min},
fmt,
hash::{Hash, Hasher},
io::{BufRead, Read, Result as IoResult, Write},
mem,
ops::{AddAssign, Deref},
......@@ -22,7 +23,7 @@ use bytes::Bytes;
use futures::Stream;
use tokio::io::{AsyncRead, ReadBuf};
use triomphe::Arc;
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher};
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64};
static EMPTY_BUF: &[u8] = &[];
......@@ -491,6 +492,12 @@ impl PartialEq for Rope {
impl Eq for Rope {}
impl Hash for Rope {
fn hash<H: Hasher>(&self, state: &mut H) {
hash_xxh3_hash64(self.content_hash()).hash(state);
}
}
impl Ord for Rope {
fn cmp(&self, other: &Self) -> Ordering {
if Arc::ptr_eq(&self.data, &other.data) {
......@@ -1124,6 +1131,24 @@ mod test {
assert_eq!(hash_xxh3_hash64(rope.content_hash()), hasher.finish());
}
#[test]
fn standard_hash_uses_content() {
use std::{
collections::hash_map::DefaultHasher,
hash::{Hash, Hasher},
};
let original = Rope::from("same content");
let copied = Rope::from(original.to_bytes().into_owned());
let mut original_hasher = DefaultHasher::new();
let mut copied_hasher = DefaultHasher::new();
original.hash(&mut original_hasher);
copied.hash(&mut copied_hasher);
assert_eq!(original, copied);
assert_eq!(original_hasher.finish(), copied_hasher.finish());
}
#[test]
fn iteration() {
let shared = Rope::from("def");
......
......@@ -36,7 +36,6 @@ enum CurrentChunkMethodWithData {
DocumentCurrentScript,
}
/// Contents of an [`EcmascriptDevChunkList`].
#[turbo_tasks::value]
pub struct EcmascriptDevChunkListContent {
current_chunk_method: CurrentChunkMethodWithData,
......
......@@ -23,6 +23,7 @@ bytes-str = { workspace = true }
const_format = { workspace = true }
data-encoding = { workspace = true }
either = { workspace = true }
erased-serde = { workspace = true }
indexmap = { workspace = true }
num-bigint = "0.4"
patricia_tree = { version = "0.10.1", features = ["serde"] }
......
......@@ -41,6 +41,7 @@ pub mod source_map;
pub mod source_pos;
pub mod source_transform;
pub mod target;
pub mod update_instruction;
mod utils;
pub mod version;
pub mod virtual_output;
......
use std::{any::Any, fmt::Debug, sync::Arc};
use serde::Serialize;
use turbo_tasks::{
NonLocalValue,
debug::ValueDebugFormat,
trace::{TraceRawVcs, TraceRawVcsContext},
};
trait ErasedUpdateInstruction:
erased_serde::Serialize + Debug + Send + Sync + NonLocalValue + 'static
{
fn as_any(&self) -> &dyn Any;
fn dyn_eq(&self, other: &dyn Any) -> bool;
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext);
}
impl<T> ErasedUpdateInstruction for T
where
T: Serialize + Eq + Debug + Send + Sync + NonLocalValue + TraceRawVcs + 'static,
{
fn as_any(&self) -> &dyn Any {
self
}
fn dyn_eq(&self, other: &dyn Any) -> bool {
other.downcast_ref::<Self>() == Some(self)
}
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
TraceRawVcs::trace_raw_vcs(self, trace_context);
}
}
erased_serde::serialize_trait_object!(ErasedUpdateInstruction);
#[derive(Clone, Debug, Serialize, ValueDebugFormat, NonLocalValue)]
#[serde(transparent)]
pub struct UpdateInstruction(Arc<dyn ErasedUpdateInstruction>);
impl PartialEq for UpdateInstruction {
fn eq(&self, other: &Self) -> bool {
self.0.dyn_eq(other.0.as_any())
}
}
impl Eq for UpdateInstruction {}
impl UpdateInstruction {
pub fn new<T>(instruction: T) -> Self
where
T: Serialize + Eq + Debug + Send + Sync + NonLocalValue + TraceRawVcs + 'static,
{
Self(Arc::new(instruction))
}
pub fn downcast_ref<T: 'static>(&self) -> Option<&T> {
self.0.as_any().downcast_ref()
}
}
impl TraceRawVcs for UpdateInstruction {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
ErasedUpdateInstruction::trace_raw_vcs(self.0.as_ref(), trace_context);
}
}
#[cfg(test)]
mod tests {
use serde::Serialize;
use turbo_tasks::{NonLocalValue, trace::TraceRawVcs};
use super::UpdateInstruction;
#[derive(Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
struct TestInstruction {
value: u32,
}
#[test]
fn serializes_without_an_extra_wrapper() {
let instruction = UpdateInstruction::new(TestInstruction { value: 42 });
assert_eq!(
serde_json::to_value(&instruction).unwrap(),
serde_json::json!({ "value": 42 })
);
}
#[test]
fn downcasts_by_concrete_type() {
let instruction = UpdateInstruction::new(TestInstruction { value: 42 });
assert_eq!(
instruction
.downcast_ref::<TestInstruction>()
.map(|instruction| instruction.value),
Some(42)
);
assert_eq!(
instruction,
UpdateInstruction::new(TestInstruction { value: 42 })
);
}
}
use std::sync::Arc;
use anyhow::{Context, Result, bail};
use turbo_rcstr::RcStr;
use turbo_tasks::{
......@@ -213,8 +211,7 @@ pub struct PartialUpdate {
pub to: TraitRef<Box<dyn Version>>,
/// The instructions to be passed to a remote system in order to update the
/// versioned object.
#[turbo_tasks(trace_ignore)]
pub instruction: Arc<serde_json::Value>,
pub instruction: crate::update_instruction::UpdateInstruction,
}
/// [`Version`] implementation that hashes a file at a given path and returns
......
......@@ -14,8 +14,11 @@ workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
turbo-rcstr = { workspace = true }
turbopack-cli-utils = { workspace = true }
turbopack-core = { workspace = true }
[dev-dependencies]
serde_json = { workspace = true }
turbo-tasks = { workspace = true }
use std::{collections::BTreeMap, fmt::Display, path::PathBuf};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use turbo_rcstr::RcStr;
use turbopack_cli_utils::issue::{LogOptions, format_issue};
use turbopack_core::{
issue::{IssueSeverity, IssueStage, PlainIssue, StyledString},
source_pos::SourcePos,
update_instruction::UpdateInstruction,
};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
......@@ -78,7 +78,7 @@ impl<'a> ClientUpdateInstruction<'a> {
pub fn partial(
resource: &'a ResourceIdentifier,
instruction: &'a Value,
instruction: &'a UpdateInstruction,
issues: &'a [Issue<'a>],
) -> Self {
Self::new(
......@@ -106,7 +106,7 @@ impl<'a> ClientUpdateInstruction<'a> {
pub enum ClientUpdateInstructionType<'a> {
Restart,
NotFound,
Partial { instruction: &'a Value },
Partial { instruction: &'a UpdateInstruction },
Issues,
}
......@@ -187,3 +187,50 @@ impl<'a> From<&'a PlainIssue> for Issue<'a> {
}
}
}
#[cfg(test)]
mod tests {
use serde::Serialize;
use serde_json::json;
use turbo_rcstr::rcstr;
use turbo_tasks::{NonLocalValue, trace::TraceRawVcs};
use turbopack_core::update_instruction::UpdateInstruction;
use super::{ClientUpdateInstruction, ResourceIdentifier};
#[derive(Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
struct TestInstruction(serde_json::Value);
#[test]
fn partial_instruction_wire_format_is_unchanged() {
let resource = ResourceIdentifier {
path: rcstr!("server/app.js"),
headers: None,
};
let instruction = UpdateInstruction::new(TestInstruction(json!({
"type": "ecmascriptMerged",
"chunks": {},
})));
assert_eq!(
serde_json::to_value(ClientUpdateInstruction::partial(
&resource,
&instruction,
&[],
))
.unwrap(),
json!({
"resource": {
"path": "server/app.js",
"headers": null,
},
"type": "partial",
"instruction": {
"type": "ecmascriptMerged",
"chunks": {},
},
"issues": [],
})
);
}
}
......@@ -12,34 +12,36 @@
use anyhow::Result;
use serde::Serialize;
use turbo_tasks::{FxIndexMap, FxIndexSet, Vc};
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, Vc, trace::TraceRawVcs};
use turbo_tasks_fs::rope::Rope;
use turbopack_core::{chunk::ModuleId, code_builder::Code, source_map::GenerateSourceMap};
/// A merged update covering one or more ecmascript chunks that share a merger.
#[derive(Serialize, Default)]
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(
tag = "type",
rename = "EcmascriptMergedUpdate",
rename_all = "camelCase"
)]
pub struct EcmascriptMergedUpdate<'a> {
pub struct EcmascriptMergedUpdate {
/// A map from module id to its latest module entry (code + source map url).
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
pub entries: FxIndexMap<ModuleId, EcmascriptModuleEntry>,
#[serde(skip_serializing_if = "FrozenMap::is_empty")]
pub entries: FrozenMap<ModuleId, EcmascriptModuleEntry>,
/// A map from chunk path to the update for that chunk.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
pub chunks: FxIndexMap<&'a str, EcmascriptMergedChunkUpdate>,
#[serde(skip_serializing_if = "FrozenMap::is_empty")]
pub chunks: FrozenMap<RcStr, EcmascriptMergedChunkUpdate>,
}
impl EcmascriptMergedUpdate<'_> {
impl EcmascriptMergedUpdate {
pub fn is_empty(&self) -> bool {
self.entries.is_empty() && self.chunks.is_empty()
}
}
/// Per-chunk portion of an [`EcmascriptMergedUpdate`].
#[derive(Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type", rename_all = "camelCase")]
pub enum EcmascriptMergedChunkUpdate {
Added(EcmascriptMergedChunkAdded),
......@@ -48,41 +50,41 @@ pub enum EcmascriptMergedChunkUpdate {
}
/// A chunk that was newly added in this version.
#[derive(Serialize, Default)]
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkAdded {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub modules: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub modules: FrozenSet<ModuleId>,
}
/// A chunk that was removed in this version.
#[derive(Serialize, Default)]
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkDeleted {
// Technically, this is redundant, since the client will already know all
// modules in the chunk from the previous version. However, it's useful for
// merging updates without access to an initial state.
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub modules: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub modules: FrozenSet<ModuleId>,
}
/// A chunk that was present in both versions and whose module membership
/// changed.
#[derive(Serialize, Default)]
#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(rename_all = "camelCase")]
pub struct EcmascriptMergedChunkPartial {
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub added: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FxIndexSet::is_empty")]
pub deleted: FxIndexSet<ModuleId>,
#[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub added: FrozenSet<ModuleId>,
#[serde(skip_serializing_if = "FrozenSet::is_empty")]
pub deleted: FrozenSet<ModuleId>,
}
/// The code (and source map) for a single module in a merged update.
#[derive(Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, TraceRawVcs, NonLocalValue)]
pub struct EcmascriptModuleEntry {
#[serde(with = "turbo_tasks_fs::rope::ser_as_string")]
pub code: Rope,
pub url: String,
pub url: RcStr,
#[serde(with = "turbo_tasks_fs::rope::ser_option_as_string")]
pub map: Option<Rope>,
}
......@@ -102,7 +104,7 @@ impl EcmascriptModuleEntry {
Ok(EcmascriptModuleEntry {
// Cloning a rope is cheap.
code: code.await?.source_code().clone(),
url: format!("{}?{}", chunk_path, id),
url: format!("{}?{}", chunk_path, id).into(),
map,
})
}
......
use std::sync::Arc;
use anyhow::Result;
use serde::Serialize;
use turbo_tasks::{FxIndexMap, ResolvedVc, TraitRef, Vc};
use turbopack_core::version::{
MergeableVersionedContent, PartialUpdate, TotalUpdate, Update, Version, VersionedContent,
VersionedContentMerger,
use turbo_rcstr::RcStr;
use turbo_tasks::{FxIndexMap, NonLocalValue, ResolvedVc, TraitRef, Vc, trace::TraceRawVcs};
use turbopack_core::{
update_instruction::UpdateInstruction,
version::{
MergeableVersionedContent, PartialUpdate, TotalUpdate, Update, Version, VersionedContent,
VersionedContentMerger,
},
};
use super::version::ChunkListVersion;
use super::{merged_update::EcmascriptMergedUpdate, version::ChunkListVersion};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(untagged)]
pub enum EcmascriptUpdateInstruction {
ChunkList(ChunkListUpdate),
Merged(EcmascriptMergedUpdate),
}
/// Update of a chunk list from one version to another.
#[derive(Serialize)]
#[serde(tag = "type")]
#[serde(rename_all = "camelCase")]
struct ChunkListUpdate<'a> {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type", rename = "ChunkListUpdate", rename_all = "camelCase")]
pub struct ChunkListUpdate {
/// A map from chunk path to a corresponding update of that chunk.
#[serde(skip_serializing_if = "FxIndexMap::is_empty")]
chunks: FxIndexMap<&'a str, ChunkUpdate>,
pub chunks: FxIndexMap<RcStr, ChunkUpdate>,
/// List of merged updates since the last version.
#[serde(skip_serializing_if = "Vec::is_empty")]
merged: Vec<Arc<serde_json::Value>>,
pub merged: Vec<EcmascriptMergedUpdate>,
}
impl ChunkListUpdate {
pub fn into_instruction(self) -> UpdateInstruction {
UpdateInstruction::new(EcmascriptUpdateInstruction::ChunkList(self))
}
}
/// Update of a chunk from one version to another.
#[derive(Serialize)]
#[derive(Clone, Debug, PartialEq, Eq, Serialize, TraceRawVcs, NonLocalValue)]
#[serde(tag = "type")]
#[serde(rename_all = "camelCase")]
enum ChunkUpdate {
pub enum ChunkUpdate {
/// The chunk was updated and must be reloaded.
Total,
/// The chunk was updated and can be merged with the previous version.
Partial { instruction: Arc<serde_json::Value> },
Partial { instruction: EcmascriptMergedUpdate },
/// The chunk was added.
Added,
/// The chunk was deleted.
Deleted,
}
impl ChunkListUpdate<'_> {
/// Returns `true` if this update is empty.
fn is_empty(&self) -> bool {
let ChunkListUpdate { chunks, merged } = self;
chunks.is_empty() && merged.is_empty()
}
}
/// Computes the update of a chunk list from one version to another.
///
/// Runtime-agnostic (takes plain paths + [`VersionedContent`]) so the browser
......@@ -105,25 +111,26 @@ pub async fn update_chunk_list(
match &*chunk_update {
Update::Total(_) => {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Total);
chunks.insert(chunk_path.clone().into(), ChunkUpdate::Total);
}
Update::Partial(partial) => {
let instruction = expect_merged_instruction_from_partial(partial);
chunks.insert(
chunk_path.as_ref(),
chunk_path.clone().into(),
ChunkUpdate::Partial {
instruction: partial.instruction.clone(),
instruction: instruction.clone(),
},
);
}
Update::Missing | Update::None => {}
}
} else {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Deleted);
chunks.insert(chunk_path.clone().into(), ChunkUpdate::Deleted);
}
}
for chunk_path in by_path.keys() {
chunks.insert(chunk_path.as_ref(), ChunkUpdate::Added);
chunks.insert((*chunk_path).clone().into(), ChunkUpdate::Added);
}
let mut merged = vec![];
......@@ -147,24 +154,73 @@ pub async fn update_chunk_list(
.cell());
}
Update::Partial(partial) => {
merged.push(partial.instruction.clone());
let instruction = expect_merged_instruction_from_partial(partial);
merged.push(instruction.clone());
}
Update::Missing | Update::None => {}
}
}
}
let update = ChunkListUpdate { chunks, merged };
let update = if update.is_empty() {
let update = if chunks.is_empty() && merged.is_empty() {
Update::None
} else {
Update::Partial(PartialUpdate {
to: Vc::upcast::<Box<dyn Version>>(to_version)
.into_trait_ref()
.await?,
instruction: Arc::new(serde_json::to_value(&update)?),
instruction: ChunkListUpdate { chunks, merged }.into_instruction(),
})
};
Ok(update.cell())
}
fn expect_merged_instruction_from_partial(partial: &PartialUpdate) -> &EcmascriptMergedUpdate {
let Some(EcmascriptUpdateInstruction::Merged(instruction)) = partial
.instruction
.downcast_ref::<EcmascriptUpdateInstruction>(
) else {
panic!("ECMAScript partial updates must contain a merged instruction");
};
instruction
}
#[cfg(test)]
mod tests {
use serde_json::json;
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_tasks::FxIndexMap;
use super::{ChunkListUpdate, ChunkUpdate, EcmascriptUpdateInstruction};
use crate::chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate,
};
#[test]
fn instruction_wire_format() {
let instruction = EcmascriptUpdateInstruction::ChunkList(ChunkListUpdate {
chunks: FxIndexMap::from_iter([("app.js".into(), ChunkUpdate::Total)]),
merged: vec![EcmascriptMergedUpdate {
entries: FrozenMap::default(),
chunks: FrozenMap::from_iter([(
"app.js".into(),
EcmascriptMergedChunkUpdate::Added(EcmascriptMergedChunkAdded {
modules: FrozenSet::default(),
}),
)]),
}],
});
assert_eq!(
serde_json::to_value(instruction).unwrap(),
json!({
"type": "ChunkListUpdate",
"chunks": { "app.js": { "type": "total" } },
"merged": [{
"type": "EcmascriptMergedUpdate",
"chunks": { "app.js": { "type": "added" } },
}],
})
);
}
}
use std::sync::Arc;
use anyhow::Result;
use turbo_tasks::{FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_tasks::{FxIndexMap, FxIndexSet, ReadRef, ResolvedVc, TryJoinIterExt, Vc};
use turbopack_core::{
chunk::ModuleId,
code_builder::Code,
update_instruction::UpdateInstruction,
version::{PartialUpdate, TotalUpdate, Update, Version},
};
use crate::{
chunk::EcmascriptChunkContentEntries,
chunk_list::merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
chunk_list::{
merged_update::{
EcmascriptMergedChunkAdded, EcmascriptMergedChunkDeleted, EcmascriptMergedChunkPartial,
EcmascriptMergedChunkUpdate, EcmascriptMergedUpdate, EcmascriptModuleEntry,
},
update::EcmascriptUpdateInstruction,
},
hmr::{
EcmascriptHmrChunkContent,
......@@ -145,21 +148,24 @@ async fn partial_chunk_update(
unreachable!("caller filters out EcmascriptChunkUpdate::None");
};
let mut partial = EcmascriptMergedChunkPartial::default();
let mut added_modules = FxIndexSet::default();
for (id, AddedModule { hash, code }) in added {
partial.added.insert(id.clone());
added_modules.insert(id.clone());
insert_entry_unless_shipped(entries, from_versions, id, hash, *code, chunk_path).await?;
}
partial.deleted.extend(deleted.into_keys());
for (id, code) in modified {
let entry = EcmascriptModuleEntry::from_code(&id, *code, chunk_path).await?;
entries.insert(id, entry);
}
Ok(EcmascriptMergedChunkUpdate::Partial(partial))
Ok(EcmascriptMergedChunkUpdate::Partial(
EcmascriptMergedChunkPartial {
added: FrozenSet::from(added_modules),
deleted: deleted.into_keys().collect(),
},
))
}
/// Builds the payload for a chunk that wasn't present in the previous version.
......@@ -169,10 +175,10 @@ async fn added_chunk_update(
from_versions: &[ReadRef<EcmascriptChunkVersion>],
entries: &mut FxIndexMap<ModuleId, EcmascriptModuleEntry>,
) -> Result<EcmascriptMergedChunkUpdate> {
let mut added = EcmascriptMergedChunkAdded::default();
let mut modules = FxIndexSet::default();
for (id, entry) in chunk_entries.iter() {
added.modules.insert(id.clone());
modules.insert(id.clone());
insert_entry_unless_shipped(
entries,
from_versions,
......@@ -184,7 +190,11 @@ async fn added_chunk_update(
.await?;
}
Ok(EcmascriptMergedChunkUpdate::Added(added))
Ok(EcmascriptMergedChunkUpdate::Added(
EcmascriptMergedChunkAdded {
modules: FrozenSet::from(modules),
},
))
}
/// Computes a single [`Update`] covering every chunk in a merged chunk content.
......@@ -235,7 +245,8 @@ pub async fn update_ecmascript_merged_chunk(
.try_join()
.await?;
let mut merged_update = EcmascriptMergedUpdate::default();
let mut merged_entries = FxIndexMap::default();
let mut chunks = FxIndexMap::default();
for (content, entries, to_version) in &to_contents {
let chunk_path = to_version.chunk_path.as_str();
......@@ -247,40 +258,34 @@ pub async fn update_ecmascript_merged_chunk(
{
EcmascriptChunkUpdate::None => continue,
update => {
partial_chunk_update(
update,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await?
partial_chunk_update(update, chunk_path, from_versions, &mut merged_entries)
.await?
}
}
}
None => {
added_chunk_update(
entries,
chunk_path,
from_versions,
&mut merged_update.entries,
)
.await?
added_chunk_update(entries, chunk_path, from_versions, &mut merged_entries).await?
}
};
merged_update.chunks.insert(chunk_path, chunk_update);
chunks.insert(to_version.chunk_path.clone(), chunk_update);
}
for (chunk_path, chunk_version) in from_versions_by_chunk_path {
for chunk_version in from_versions_by_chunk_path.into_values() {
let hashes = &chunk_version.entries_hashes;
merged_update.chunks.insert(
chunk_path,
chunks.insert(
chunk_version.chunk_path.clone(),
EcmascriptMergedChunkUpdate::Deleted(EcmascriptMergedChunkDeleted {
modules: hashes.keys().cloned().collect(),
}),
);
}
let merged_update = EcmascriptMergedUpdate {
entries: FrozenMap::from(merged_entries),
chunks: FrozenMap::from(chunks),
};
Ok(if merged_update.is_empty() {
Update::None
} else {
......@@ -288,7 +293,7 @@ pub async fn update_ecmascript_merged_chunk(
to: Vc::upcast::<Box<dyn Version>>(to_merged_version)
.into_trait_ref()
.await?,
instruction: Arc::new(serde_json::to_value(&merged_update)?),
instruction: UpdateInstruction::new(EcmascriptUpdateInstruction::Merged(merged_update)),
})
})
}
......@@ -100,7 +100,7 @@ impl EcmascriptBuildNodeChunkListContent {
/// Builds a chunk list content directly from a fixed set of `chunks`,
/// without expanding async-loader references. Used by
/// [`super::chunk_list::EcmascriptBuildNodeChunkList`] to track chunks
/// `super::chunk_list::EcmascriptBuildNodeChunkList` to track chunks
/// (e.g. client-component SSR chunks) that are already fully enumerated by
/// the caller.
#[turbo_tasks::function]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment