Commit 2ffcc59e authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Merge `EcmascriptChunkUpdate`s before sending them to the client (vercel/turbo#3975)

This diff:
* introduces the `VersionedContentMerger` trait, which allows for
merging the updates of versioned contents within the same chunk group;
* implements this for `EcmascriptChunkContent`/`EcmascriptChunkUpdate`,
turning them into
`EcmascriptMergedChunkContent`/`EcmascriptMergedChunkUpdate`;
* creates a new `ChunkList` asset which is capable of merging chunk
updates of chunks within the same chunk group, and create such an asset
for dynamic chunks (through manifest/loader_item.rs) and chunk group
files assets.

This fixes a bunch of edge cases related to HMR:
* HMR of dynamic imports now works;
* Chunks getting added/deleted/renamed now also works with HMR, since
we're listening to updates at the chunk group level;
* CSS chunks get reloaded in the right order, with respect for
precedence.

There are still known edge cases with HMR:
* CSS chunks added through HMR are not inserted at the right position to
respect precedence (WEB-652).
* Update aggregation is disabled because we don't want a critical issue
to stop all HMR (WEB-582) . This would be fixed by applying aggregated
updates when dismissing the error modal, but there are some edge cases
with this too (e.g. what happens when an HMR update also causes an error
on top of an existing error).
parent e6103ae6
......@@ -37,7 +37,7 @@ import { headersFromEntries } from "@vercel/turbopack-next/internal/headers";
import { parse, ParsedUrlQuery } from "node:querystring";
globalThis.__next_require__ = (data) => {
const [, , ssr_id] = JSON.parse(data);
const [, , , ssr_id] = JSON.parse(data);
return __turbopack_require__(ssr_id);
};
globalThis.__next_chunk_load__ = () => Promise.resolve();
......
......@@ -19,7 +19,8 @@ window.next = {
};
globalThis.__next_require__ = (data) => {
const [client_id] = JSON.parse(data);
const [client_id, chunks, chunkListPath] = JSON.parse(data);
__turbopack_register_chunk_list__(chunkListPath, chunks);
return __turbopack_require__(client_id);
};
globalThis.__next_chunk_load__ = __turbopack_load__;
......
......@@ -5,3 +5,4 @@ export = Anything;
export const __turbopack_module_id__: string | number;
export const chunks: string[];
export const chunkListPath: string;
......@@ -5,6 +5,8 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/
import { __turbopack_module_id__ as id } from "CLIENT_MODULE";
// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";
export default createProxy(JSON.stringify([client_id, chunks, id]));
export default createProxy(
JSON.stringify([client_id, chunks, chunkListPath, id])
);
import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";
// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";
export default createProxy(JSON.stringify([client_id, chunks]));
export default createProxy(JSON.stringify([client_id, chunks, chunkListPath]));
......@@ -18,7 +18,7 @@ subscribeToUpdate(
},
},
(update) => {
if (update.type === "restart") {
if (update.type === "restart" || update.type === "notFound") {
location.reload();
}
}
......
......@@ -97,7 +97,7 @@ function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) {
path: "_next/static/development/_devPagesManifest.json",
},
(update) => {
if (["restart", "partial"].includes(update.type)) {
if (["restart", "notFound", "partial"].includes(update.type)) {
return;
}
......
declare global {
function __turbopack_require__(name: any): any;
function __turbopack_load__(path: string): any;
function __turbopack_register_chunk_list__(
chunkListPath: string,
chunksPaths: string[]
): any;
function __webpack_require__(name: any): any;
var __webpack_public_path__: string | undefined;
var __DEV_MIDDLEWARE_MATCHERS: any[];
......
......@@ -411,7 +411,7 @@ async fn create_app_source_for_directory(
segments: layouts,
} => {
let LayoutSegment { target, .. } = *segment.await?;
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);
sources.push(create_node_rendered_source(
......@@ -443,7 +443,7 @@ async fn create_app_source_for_directory(
route,
..
} => {
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);
sources.push(create_node_api_source(
......
......@@ -95,6 +95,7 @@ impl Transition for NextClientTransition {
asset: asset.into(),
chunking_context: self.client_chunking_context,
base_path: self.server_root.join("_next"),
server_root: self.server_root,
runtime_entries: Some(runtime_entries),
};
......
use anyhow::Result;
use anyhow::{bail, Result};
use indoc::formatdoc;
use serde_json::Value;
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc};
use turbo_tasks_fs::FileSystemPathVc;
......@@ -13,14 +14,15 @@ use turbopack::ecmascript::{
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
chunk::{
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContextVc,
ChunkingType, ChunkingTypeOptionVc,
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext,
ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc,
},
ident::AssetIdentVc,
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;
use super::in_chunking_context_asset::InChunkingContextAsset;
......@@ -93,6 +95,25 @@ struct WithChunksChunkItem {
inner: WithChunksAssetVc,
}
#[turbo_tasks::value_impl]
impl WithChunksChunkItemVc {
#[turbo_tasks::function]
async fn chunk_list_path(self) -> Result<FileSystemPathVc> {
let this = self.await?;
Ok(this.inner_context.chunk_list_path(this.inner.ident()))
}
#[turbo_tasks::function]
async fn chunk_group(self) -> Result<ChunkGroupVc> {
let this = self.await?;
let inner = this.inner.await?;
Ok(ChunkGroupVc::from_asset(
inner.asset.into(),
this.inner_context,
))
}
}
#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for WithChunksChunkItem {
#[turbo_tasks::function]
......@@ -101,29 +122,40 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
}
#[turbo_tasks::function]
async fn content(&self) -> Result<EcmascriptChunkItemContentVc> {
let inner = self.inner.await?;
let group = ChunkGroupVc::from_asset(inner.asset.into(), self.inner_context);
async fn content(self_vc: WithChunksChunkItemVc) -> Result<EcmascriptChunkItemContentVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
let group = self_vc.chunk_group();
let chunks = group.chunks().await?;
let server_root = inner.server_root.await?;
let mut client_chunks = Vec::new();
let chunk_list_path = self_vc.chunk_list_path().await?;
let chunk_list_path = if let Some(path) = server_root.get_path_to(&chunk_list_path) {
path
} else {
bail!("could not get path to chunk list");
};
for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? {
if let Some(path) = server_root.get_path_to(&chunk_path) {
client_chunks.push(Value::String(path.to_string()));
}
}
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.inner_context).id().await?);
let module_id = stringify_js(&*inner.asset.as_chunk_item(this.inner_context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: format!(
"__turbopack_esm__({{
default: () => {},
chunks: () => chunks
}});
const chunks = {};
",
inner_code: formatdoc! {
r#"
__turbopack_esm__({{
default: () => {},
chunks: () => {},
chunkListPath: () => {},
}});
"#,
module_id,
Value::Array(client_chunks)
)
stringify_js_pretty(&client_chunks),
stringify_js(&chunk_list_path),
}
.into(),
..Default::default()
}
......@@ -139,18 +171,27 @@ impl ChunkItem for WithChunksChunkItem {
}
#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
Ok(AssetReferencesVc::cell(vec![WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.inner_context,
async fn references(self_vc: WithChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: this.inner_context,
}
.cell()
.into(),
}
.cell()
.into(),
}
.cell()
.into()]))
ChunkListReferenceVc::new(
inner.server_root,
self_vc.chunk_group(),
self_vc.chunk_list_path(),
)
.into(),
]))
}
}
......
......@@ -21,6 +21,7 @@ use turbopack_core::{
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;
use crate::next_client_chunks::in_chunking_context_asset::InChunkingContextAsset;
......@@ -129,15 +130,17 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem {
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: formatdoc!(
// We store the chunks in a binding, otherwise a new array would be created every
// time the export binding is read.
r#"
__turbopack_esm__({{
default: () => __turbopack_import__({}),
chunks: () => chunks
chunks: () => chunks,
}});
const chunks = {};
"#,
module_id,
stringify_js(&client_chunks)
stringify_js_pretty(&client_chunks),
)
.into(),
..Default::default()
......@@ -154,13 +157,14 @@ impl ChunkItem for WithClientChunksChunkItem {
}
#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
async fn references(self_vc: WithClientChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithClientChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.context,
chunking_context: this.context,
}
.cell()
.into(),
......
......@@ -112,6 +112,7 @@ impl Transition for NextEdgeTransition {
asset: new_asset.into(),
chunking_context: self.edge_chunking_context,
base_path: self.output_path,
server_root: self.output_path,
runtime_entries: None,
};
......
......@@ -2,7 +2,7 @@ use std::io::Write;
use anyhow::{bail, Result};
use indexmap::indexmap;
use turbo_tasks::{primitives::StringVc, Value};
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, Value};
use turbo_tasks_fs::{rope::RopeBuilder, File, FileContent, FileSystemPathVc};
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
......@@ -116,17 +116,28 @@ impl Asset for PageLoaderAsset {
let this = &*self_vc.await?;
let chunks = self_vc.get_page_chunks().await?;
let mut data = Vec::with_capacity(chunks.len());
for chunk in chunks.iter() {
let path = chunk.path().await?;
data.push(serde_json::Value::String(path.path.clone()));
}
let server_root = this.server_root.await?;
let chunk_paths: Vec<_> = chunks
.iter()
.map(|chunk| {
let server_root = server_root.clone();
async move {
Ok(server_root
.get_path_to(&*chunk.path().await?)
.map(|path| path.to_string()))
}
})
.try_join()
.await?
.into_iter()
.flatten()
.collect();
let content = format!(
"__turbopack_load_page_chunks__({}, {})\n",
stringify_js(&this.pathname.await?),
serde_json::Value::Array(data)
stringify_js(&chunk_paths)
);
Ok(AssetContentVc::from(File::from(content)))
......
......@@ -346,7 +346,7 @@ async fn create_page_source_for_file(
Value::new(ClientContextType::Pages { pages_dir }),
);
let pathname = pathname_for_path(server_root, server_path, true);
let pathname = pathname_for_path(server_root, server_path, true, false);
let route_matcher = NextParamsMatcherVc::new(pathname);
Ok(if is_api_path {
......@@ -370,8 +370,9 @@ async fn create_page_source_for_file(
runtime_entries,
)
} else {
let data_pathname = pathname_for_path(server_root, server_path, true, true);
let data_route_matcher =
NextPrefixSuffixParamsMatcherVc::new(pathname, "_next/data/development/", ".json");
NextPrefixSuffixParamsMatcherVc::new(data_pathname, "_next/data/development/", ".json");
let ssr_entry = SsrEntry {
context: server_context,
......
......@@ -28,6 +28,7 @@ pub async fn pathname_for_path(
server_root: FileSystemPathVc,
server_path: FileSystemPathVc,
has_extension: bool,
data: bool,
) -> Result<StringVc> {
let server_path_value = &*server_path.await?;
let path = if let Some(path) = server_root.await?.get_path_to(server_path_value) {
......@@ -46,10 +47,14 @@ pub async fn pathname_for_path(
} else {
path
};
let path = if path == "index" {
""
let path = if data {
path
} else {
path.strip_suffix("/index").unwrap_or(path)
if path == "index" {
""
} else {
path.strip_suffix("/index").unwrap_or(path)
}
};
Ok(StringVc::cell(path.to_string()))
......
......@@ -66,18 +66,23 @@ pub async fn create_web_entry_source(
})
.try_join()
.await?;
let chunks: Vec<_> = entries
let chunk_groups: Vec<_> = entries
.into_iter()
.flatten()
.enumerate()
.map(|(i, module)| async move {
if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? {
Ok(ecmascript
.as_evaluated_chunk(chunking_context, (i == 0).then_some(runtime_entries)))
let chunk = ecmascript
.as_evaluated_chunk(chunking_context, (i == 0).then_some(runtime_entries));
let chunk_group = ChunkGroupVc::from_chunk(chunk);
Ok(chunk_group)
} else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? {
// TODO this is missing runtime code, so it's probably broken and we should also
// add an ecmascript chunk with the runtime code
Ok(chunkable.as_chunk(chunking_context))
Ok(ChunkGroupVc::from_chunk(
chunkable.as_chunk(chunking_context),
))
} else {
// TODO convert into a serve-able asset
Err(anyhow!(
......@@ -89,11 +94,7 @@ pub async fn create_web_entry_source(
.try_join()
.await?;
let entry_asset = DevHtmlAssetVc::new(
server_root.join("index.html"),
chunks.into_iter().map(ChunkGroupVc::from_chunk).collect(),
)
.into();
let entry_asset = DevHtmlAssetVc::new(server_root.join("index.html"), chunk_groups).into();
let graph = if eager_compile {
AssetGraphContentSourceVc::new_eager(server_root, entry_asset)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment