Commit 2ffcc59e authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Merge `EcmascriptChunkUpdate`s before sending them to the client (vercel/turbo#3975)

This diff:
* introduces the `VersionedContentMerger` trait, which allows for
merging the updates of versioned contents within the same chunk group;
* implements this for `EcmascriptChunkContent`/`EcmascriptChunkUpdate`,
turning them into
`EcmascriptMergedChunkContent`/`EcmascriptMergedChunkUpdate`;
* creates a new `ChunkList` asset which is capable of merging chunk
updates of chunks within the same chunk group, and create such an asset
for dynamic chunks (through manifest/loader_item.rs) and chunk group
files assets.

This fixes a bunch of edge cases related to HMR:
* HMR of dynamic imports now works;
* Chunks getting added/deleted/renamed now also works with HMR, since
we're listening to updates at the chunk group level;
* CSS chunks get reloaded in the right order, with respect for
precedence.

There are still known edge cases with HMR:
* CSS chunks added through HMR are not inserted at the right position to
respect precedence (WEB-652).
* Update aggregation is disabled because we don't want a critical issue
to stop all HMR (WEB-582) . This would be fixed by applying aggregated
updates when dismissing the error modal, but there are some edge cases
with this too (e.g. what happens when an HMR update also causes an error
on top of an existing error).
parent e6103ae6
...@@ -37,7 +37,7 @@ import { headersFromEntries } from "@vercel/turbopack-next/internal/headers"; ...@@ -37,7 +37,7 @@ import { headersFromEntries } from "@vercel/turbopack-next/internal/headers";
import { parse, ParsedUrlQuery } from "node:querystring"; import { parse, ParsedUrlQuery } from "node:querystring";
globalThis.__next_require__ = (data) => { globalThis.__next_require__ = (data) => {
const [, , ssr_id] = JSON.parse(data); const [, , , ssr_id] = JSON.parse(data);
return __turbopack_require__(ssr_id); return __turbopack_require__(ssr_id);
}; };
globalThis.__next_chunk_load__ = () => Promise.resolve(); globalThis.__next_chunk_load__ = () => Promise.resolve();
......
...@@ -19,7 +19,8 @@ window.next = { ...@@ -19,7 +19,8 @@ window.next = {
}; };
globalThis.__next_require__ = (data) => { globalThis.__next_require__ = (data) => {
const [client_id] = JSON.parse(data); const [client_id, chunks, chunkListPath] = JSON.parse(data);
__turbopack_register_chunk_list__(chunkListPath, chunks);
return __turbopack_require__(client_id); return __turbopack_require__(client_id);
}; };
globalThis.__next_chunk_load__ = __turbopack_load__; globalThis.__next_chunk_load__ = __turbopack_load__;
......
...@@ -5,3 +5,4 @@ export = Anything; ...@@ -5,3 +5,4 @@ export = Anything;
export const __turbopack_module_id__: string | number; export const __turbopack_module_id__: string | number;
export const chunks: string[]; export const chunks: string[];
export const chunkListPath: string;
...@@ -5,6 +5,8 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/ ...@@ -5,6 +5,8 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/
import { __turbopack_module_id__ as id } from "CLIENT_MODULE"; import { __turbopack_module_id__ as id } from "CLIENT_MODULE";
// @ts-expect-error CLIENT_CHUNKS is provided by rust // @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS"; import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";
export default createProxy(JSON.stringify([client_id, chunks, id])); export default createProxy(
JSON.stringify([client_id, chunks, chunkListPath, id])
);
import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy"; import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";
// @ts-expect-error CLIENT_CHUNKS is provided by rust // @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS"; import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";
export default createProxy(JSON.stringify([client_id, chunks])); export default createProxy(JSON.stringify([client_id, chunks, chunkListPath]));
...@@ -18,7 +18,7 @@ subscribeToUpdate( ...@@ -18,7 +18,7 @@ subscribeToUpdate(
}, },
}, },
(update) => { (update) => {
if (update.type === "restart") { if (update.type === "restart" || update.type === "notFound") {
location.reload(); location.reload();
} }
} }
......
...@@ -97,7 +97,7 @@ function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) { ...@@ -97,7 +97,7 @@ function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) {
path: "_next/static/development/_devPagesManifest.json", path: "_next/static/development/_devPagesManifest.json",
}, },
(update) => { (update) => {
if (["restart", "partial"].includes(update.type)) { if (["restart", "notFound", "partial"].includes(update.type)) {
return; return;
} }
......
declare global { declare global {
function __turbopack_require__(name: any): any; function __turbopack_require__(name: any): any;
function __turbopack_load__(path: string): any; function __turbopack_load__(path: string): any;
function __turbopack_register_chunk_list__(
chunkListPath: string,
chunksPaths: string[]
): any;
function __webpack_require__(name: any): any; function __webpack_require__(name: any): any;
var __webpack_public_path__: string | undefined; var __webpack_public_path__: string | undefined;
var __DEV_MIDDLEWARE_MATCHERS: any[]; var __DEV_MIDDLEWARE_MATCHERS: any[];
......
...@@ -411,7 +411,7 @@ async fn create_app_source_for_directory( ...@@ -411,7 +411,7 @@ async fn create_app_source_for_directory(
segments: layouts, segments: layouts,
} => { } => {
let LayoutSegment { target, .. } = *segment.await?; let LayoutSegment { target, .. } = *segment.await?;
let pathname = pathname_for_path(server_root, url, false); let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname); let params_matcher = NextParamsMatcherVc::new(pathname);
sources.push(create_node_rendered_source( sources.push(create_node_rendered_source(
...@@ -443,7 +443,7 @@ async fn create_app_source_for_directory( ...@@ -443,7 +443,7 @@ async fn create_app_source_for_directory(
route, route,
.. ..
} => { } => {
let pathname = pathname_for_path(server_root, url, false); let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname); let params_matcher = NextParamsMatcherVc::new(pathname);
sources.push(create_node_api_source( sources.push(create_node_api_source(
......
...@@ -95,6 +95,7 @@ impl Transition for NextClientTransition { ...@@ -95,6 +95,7 @@ impl Transition for NextClientTransition {
asset: asset.into(), asset: asset.into(),
chunking_context: self.client_chunking_context, chunking_context: self.client_chunking_context,
base_path: self.server_root.join("_next"), base_path: self.server_root.join("_next"),
server_root: self.server_root,
runtime_entries: Some(runtime_entries), runtime_entries: Some(runtime_entries),
}; };
......
use anyhow::Result; use anyhow::{bail, Result};
use indoc::formatdoc;
use serde_json::Value; use serde_json::Value;
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc}; use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc};
use turbo_tasks_fs::FileSystemPathVc; use turbo_tasks_fs::FileSystemPathVc;
...@@ -13,14 +14,15 @@ use turbopack::ecmascript::{ ...@@ -13,14 +14,15 @@ use turbopack::ecmascript::{
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc}, asset::{Asset, AssetContentVc, AssetVc},
chunk::{ chunk::{
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkVc, ChunkableAsset, Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContextVc, ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext,
ChunkingType, ChunkingTypeOptionVc, ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc,
}, },
ident::AssetIdentVc, ident::AssetIdentVc,
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc}, reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc}, resolve::{ResolveResult, ResolveResultVc},
}; };
use turbopack_ecmascript::utils::stringify_js_pretty;
use super::in_chunking_context_asset::InChunkingContextAsset; use super::in_chunking_context_asset::InChunkingContextAsset;
...@@ -93,6 +95,25 @@ struct WithChunksChunkItem { ...@@ -93,6 +95,25 @@ struct WithChunksChunkItem {
inner: WithChunksAssetVc, inner: WithChunksAssetVc,
} }
#[turbo_tasks::value_impl]
impl WithChunksChunkItemVc {
#[turbo_tasks::function]
async fn chunk_list_path(self) -> Result<FileSystemPathVc> {
let this = self.await?;
Ok(this.inner_context.chunk_list_path(this.inner.ident()))
}
#[turbo_tasks::function]
async fn chunk_group(self) -> Result<ChunkGroupVc> {
let this = self.await?;
let inner = this.inner.await?;
Ok(ChunkGroupVc::from_asset(
inner.asset.into(),
this.inner_context,
))
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl EcmascriptChunkItem for WithChunksChunkItem { impl EcmascriptChunkItem for WithChunksChunkItem {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -101,29 +122,40 @@ impl EcmascriptChunkItem for WithChunksChunkItem { ...@@ -101,29 +122,40 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn content(&self) -> Result<EcmascriptChunkItemContentVc> { async fn content(self_vc: WithChunksChunkItemVc) -> Result<EcmascriptChunkItemContentVc> {
let inner = self.inner.await?; let this = self_vc.await?;
let group = ChunkGroupVc::from_asset(inner.asset.into(), self.inner_context); let inner = this.inner.await?;
let group = self_vc.chunk_group();
let chunks = group.chunks().await?; let chunks = group.chunks().await?;
let server_root = inner.server_root.await?; let server_root = inner.server_root.await?;
let mut client_chunks = Vec::new(); let mut client_chunks = Vec::new();
let chunk_list_path = self_vc.chunk_list_path().await?;
let chunk_list_path = if let Some(path) = server_root.get_path_to(&chunk_list_path) {
path
} else {
bail!("could not get path to chunk list");
};
for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? { for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? {
if let Some(path) = server_root.get_path_to(&chunk_path) { if let Some(path) = server_root.get_path_to(&chunk_path) {
client_chunks.push(Value::String(path.to_string())); client_chunks.push(Value::String(path.to_string()));
} }
} }
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.inner_context).id().await?); let module_id = stringify_js(&*inner.asset.as_chunk_item(this.inner_context).id().await?);
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
inner_code: format!( inner_code: formatdoc! {
"__turbopack_esm__({{ r#"
default: () => {}, __turbopack_esm__({{
chunks: () => chunks default: () => {},
}}); chunks: () => {},
const chunks = {}; chunkListPath: () => {},
", }});
"#,
module_id, module_id,
Value::Array(client_chunks) stringify_js_pretty(&client_chunks),
) stringify_js(&chunk_list_path),
}
.into(), .into(),
..Default::default() ..Default::default()
} }
...@@ -139,18 +171,27 @@ impl ChunkItem for WithChunksChunkItem { ...@@ -139,18 +171,27 @@ impl ChunkItem for WithChunksChunkItem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> { async fn references(self_vc: WithChunksChunkItemVc) -> Result<AssetReferencesVc> {
let inner = self.inner.await?; let this = self_vc.await?;
Ok(AssetReferencesVc::cell(vec![WithChunksAssetReference { let inner = this.inner.await?;
asset: InChunkingContextAsset { Ok(AssetReferencesVc::cell(vec![
asset: inner.asset, WithChunksAssetReference {
chunking_context: self.inner_context, asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: this.inner_context,
}
.cell()
.into(),
} }
.cell() .cell()
.into(), .into(),
} ChunkListReferenceVc::new(
.cell() inner.server_root,
.into()])) self_vc.chunk_group(),
self_vc.chunk_list_path(),
)
.into(),
]))
} }
} }
......
...@@ -21,6 +21,7 @@ use turbopack_core::{ ...@@ -21,6 +21,7 @@ use turbopack_core::{
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc}, reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc}, resolve::{ResolveResult, ResolveResultVc},
}; };
use turbopack_ecmascript::utils::stringify_js_pretty;
use crate::next_client_chunks::in_chunking_context_asset::InChunkingContextAsset; use crate::next_client_chunks::in_chunking_context_asset::InChunkingContextAsset;
...@@ -129,15 +130,17 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem { ...@@ -129,15 +130,17 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem {
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.context).id().await?); let module_id = stringify_js(&*inner.asset.as_chunk_item(self.context).id().await?);
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
inner_code: formatdoc!( inner_code: formatdoc!(
// We store the chunks in a binding, otherwise a new array would be created every
// time the export binding is read.
r#" r#"
__turbopack_esm__({{ __turbopack_esm__({{
default: () => __turbopack_import__({}), default: () => __turbopack_import__({}),
chunks: () => chunks chunks: () => chunks,
}}); }});
const chunks = {}; const chunks = {};
"#, "#,
module_id, module_id,
stringify_js(&client_chunks) stringify_js_pretty(&client_chunks),
) )
.into(), .into(),
..Default::default() ..Default::default()
...@@ -154,13 +157,14 @@ impl ChunkItem for WithClientChunksChunkItem { ...@@ -154,13 +157,14 @@ impl ChunkItem for WithClientChunksChunkItem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> { async fn references(self_vc: WithClientChunksChunkItemVc) -> Result<AssetReferencesVc> {
let inner = self.inner.await?; let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![ Ok(AssetReferencesVc::cell(vec![
WithClientChunksAssetReference { WithClientChunksAssetReference {
asset: InChunkingContextAsset { asset: InChunkingContextAsset {
asset: inner.asset, asset: inner.asset,
chunking_context: self.context, chunking_context: this.context,
} }
.cell() .cell()
.into(), .into(),
......
...@@ -112,6 +112,7 @@ impl Transition for NextEdgeTransition { ...@@ -112,6 +112,7 @@ impl Transition for NextEdgeTransition {
asset: new_asset.into(), asset: new_asset.into(),
chunking_context: self.edge_chunking_context, chunking_context: self.edge_chunking_context,
base_path: self.output_path, base_path: self.output_path,
server_root: self.output_path,
runtime_entries: None, runtime_entries: None,
}; };
......
...@@ -2,7 +2,7 @@ use std::io::Write; ...@@ -2,7 +2,7 @@ use std::io::Write;
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use indexmap::indexmap; use indexmap::indexmap;
use turbo_tasks::{primitives::StringVc, Value}; use turbo_tasks::{primitives::StringVc, TryJoinIterExt, Value};
use turbo_tasks_fs::{rope::RopeBuilder, File, FileContent, FileSystemPathVc}; use turbo_tasks_fs::{rope::RopeBuilder, File, FileContent, FileSystemPathVc};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc}, asset::{Asset, AssetContentVc, AssetVc},
...@@ -116,17 +116,28 @@ impl Asset for PageLoaderAsset { ...@@ -116,17 +116,28 @@ impl Asset for PageLoaderAsset {
let this = &*self_vc.await?; let this = &*self_vc.await?;
let chunks = self_vc.get_page_chunks().await?; let chunks = self_vc.get_page_chunks().await?;
let server_root = this.server_root.await?;
let mut data = Vec::with_capacity(chunks.len());
for chunk in chunks.iter() { let chunk_paths: Vec<_> = chunks
let path = chunk.path().await?; .iter()
data.push(serde_json::Value::String(path.path.clone())); .map(|chunk| {
} let server_root = server_root.clone();
async move {
Ok(server_root
.get_path_to(&*chunk.path().await?)
.map(|path| path.to_string()))
}
})
.try_join()
.await?
.into_iter()
.flatten()
.collect();
let content = format!( let content = format!(
"__turbopack_load_page_chunks__({}, {})\n", "__turbopack_load_page_chunks__({}, {})\n",
stringify_js(&this.pathname.await?), stringify_js(&this.pathname.await?),
serde_json::Value::Array(data) stringify_js(&chunk_paths)
); );
Ok(AssetContentVc::from(File::from(content))) Ok(AssetContentVc::from(File::from(content)))
......
...@@ -346,7 +346,7 @@ async fn create_page_source_for_file( ...@@ -346,7 +346,7 @@ async fn create_page_source_for_file(
Value::new(ClientContextType::Pages { pages_dir }), Value::new(ClientContextType::Pages { pages_dir }),
); );
let pathname = pathname_for_path(server_root, server_path, true); let pathname = pathname_for_path(server_root, server_path, true, false);
let route_matcher = NextParamsMatcherVc::new(pathname); let route_matcher = NextParamsMatcherVc::new(pathname);
Ok(if is_api_path { Ok(if is_api_path {
...@@ -370,8 +370,9 @@ async fn create_page_source_for_file( ...@@ -370,8 +370,9 @@ async fn create_page_source_for_file(
runtime_entries, runtime_entries,
) )
} else { } else {
let data_pathname = pathname_for_path(server_root, server_path, true, true);
let data_route_matcher = let data_route_matcher =
NextPrefixSuffixParamsMatcherVc::new(pathname, "_next/data/development/", ".json"); NextPrefixSuffixParamsMatcherVc::new(data_pathname, "_next/data/development/", ".json");
let ssr_entry = SsrEntry { let ssr_entry = SsrEntry {
context: server_context, context: server_context,
......
...@@ -28,6 +28,7 @@ pub async fn pathname_for_path( ...@@ -28,6 +28,7 @@ pub async fn pathname_for_path(
server_root: FileSystemPathVc, server_root: FileSystemPathVc,
server_path: FileSystemPathVc, server_path: FileSystemPathVc,
has_extension: bool, has_extension: bool,
data: bool,
) -> Result<StringVc> { ) -> Result<StringVc> {
let server_path_value = &*server_path.await?; let server_path_value = &*server_path.await?;
let path = if let Some(path) = server_root.await?.get_path_to(server_path_value) { let path = if let Some(path) = server_root.await?.get_path_to(server_path_value) {
...@@ -46,10 +47,14 @@ pub async fn pathname_for_path( ...@@ -46,10 +47,14 @@ pub async fn pathname_for_path(
} else { } else {
path path
}; };
let path = if path == "index" { let path = if data {
"" path
} else { } else {
path.strip_suffix("/index").unwrap_or(path) if path == "index" {
""
} else {
path.strip_suffix("/index").unwrap_or(path)
}
}; };
Ok(StringVc::cell(path.to_string())) Ok(StringVc::cell(path.to_string()))
......
...@@ -66,18 +66,23 @@ pub async fn create_web_entry_source( ...@@ -66,18 +66,23 @@ pub async fn create_web_entry_source(
}) })
.try_join() .try_join()
.await?; .await?;
let chunks: Vec<_> = entries
let chunk_groups: Vec<_> = entries
.into_iter() .into_iter()
.flatten() .flatten()
.enumerate() .enumerate()
.map(|(i, module)| async move { .map(|(i, module)| async move {
if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? { if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? {
Ok(ecmascript let chunk = ecmascript
.as_evaluated_chunk(chunking_context, (i == 0).then_some(runtime_entries))) .as_evaluated_chunk(chunking_context, (i == 0).then_some(runtime_entries));
let chunk_group = ChunkGroupVc::from_chunk(chunk);
Ok(chunk_group)
} else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? { } else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? {
// TODO this is missing runtime code, so it's probably broken and we should also // TODO this is missing runtime code, so it's probably broken and we should also
// add an ecmascript chunk with the runtime code // add an ecmascript chunk with the runtime code
Ok(chunkable.as_chunk(chunking_context)) Ok(ChunkGroupVc::from_chunk(
chunkable.as_chunk(chunking_context),
))
} else { } else {
// TODO convert into a serve-able asset // TODO convert into a serve-able asset
Err(anyhow!( Err(anyhow!(
...@@ -89,11 +94,7 @@ pub async fn create_web_entry_source( ...@@ -89,11 +94,7 @@ pub async fn create_web_entry_source(
.try_join() .try_join()
.await?; .await?;
let entry_asset = DevHtmlAssetVc::new( let entry_asset = DevHtmlAssetVc::new(server_root.join("index.html"), chunk_groups).into();
server_root.join("index.html"),
chunks.into_iter().map(ChunkGroupVc::from_chunk).collect(),
)
.into();
let graph = if eager_compile { let graph = if eager_compile {
AssetGraphContentSourceVc::new_eager(server_root, entry_asset) AssetGraphContentSourceVc::new_eager(server_root, entry_asset)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment