Unverified Commit ce42173e authored by Alex Kirszenberg's avatar Alex Kirszenberg Committed by GitHub

Chunking Refactor pt. 2 (#47961)

See https://github.com/vercel/turbo/pull/4450

This PR updates Turbopack to turbopack-230411.2:

## Miscellaneous

* https://github.com/vercel/turbo/pull/4450 <!-- Alex Kirszenberg -
Chunking Refactor pt. 2 -->
parent cd55220a
This diff is collapsed.
...@@ -47,11 +47,11 @@ swc_emotion = { version = "0.29.10" } ...@@ -47,11 +47,11 @@ swc_emotion = { version = "0.29.10" }
testing = { version = "0.31.31" } testing = { version = "0.31.31" }
# Turbo crates # Turbo crates
turbo-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230406.2" } turbo-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230411.2" }
# [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros.. # [TODO]: need to refactor embed_directory! macro usages, as well as resolving turbo_tasks::function, macros..
turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230406.2" } turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230411.2" }
# [TODO]: need to refactor embed_directory! macro usage in next-core # [TODO]: need to refactor embed_directory! macro usage in next-core
turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230406.2" } turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-230411.2" }
# General Deps # General Deps
......
...@@ -10,7 +10,7 @@ ...@@ -10,7 +10,7 @@
"check": "tsc --noEmit" "check": "tsc --noEmit"
}, },
"dependencies": { "dependencies": {
"@vercel/turbopack-dev-runtime": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2", "@vercel/turbopack-dev-runtime": "https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2",
"anser": "^2.1.1", "anser": "^2.1.1",
"css.escape": "^1.5.1", "css.escape": "^1.5.1",
"next": "*", "next": "*",
......
...@@ -39,7 +39,7 @@ import layoutEntry, { chunks as layoutEntryClientChunks } from "@vercel/turbopac ...@@ -39,7 +39,7 @@ import layoutEntry, { chunks as layoutEntryClientChunks } from "@vercel/turbopac
globalThis.__next_require__ = (data) => { globalThis.__next_require__ = (data) => {
const [, , , ssr_id] = JSON.parse(data); const [, , ssr_id] = JSON.parse(data);
return __turbopack_require__(ssr_id); return __turbopack_require__(ssr_id);
}; };
globalThis.__next_chunk_load__ = () => Promise.resolve(); globalThis.__next_chunk_load__ = () => Promise.resolve();
......
...@@ -3,7 +3,7 @@ import React, { use } from "react"; ...@@ -3,7 +3,7 @@ import React, { use } from "react";
import type { ReactElement } from "react"; import type { ReactElement } from "react";
import { version } from "next/package.json"; import { version } from "next/package.json";
import { createFromReadableStream } from "next/dist/compiled/react-server-dom-webpack/client"; import { createFromReadableStream } from "next/dist/compiled/react-server-dom-webpack/client";
import { callServer } from 'next/dist/client/app-call-server'; import { callServer } from "next/dist/client/app-call-server";
import { HeadManagerContext } from "next/dist/shared/lib/head-manager-context"; import { HeadManagerContext } from "next/dist/shared/lib/head-manager-context";
...@@ -19,8 +19,7 @@ window.next = { ...@@ -19,8 +19,7 @@ window.next = {
}; };
globalThis.__next_require__ = (data) => { globalThis.__next_require__ = (data) => {
const [client_id, chunks, chunkListPath] = JSON.parse(data); const [client_id] = JSON.parse(data);
__turbopack_register_chunk_list__(chunkListPath, chunks);
return __turbopack_require__(client_id); return __turbopack_require__(client_id);
}; };
globalThis.__next_chunk_load__ = __turbopack_load__; globalThis.__next_chunk_load__ = __turbopack_load__;
......
...@@ -5,4 +5,3 @@ export = Anything; ...@@ -5,4 +5,3 @@ export = Anything;
export const __turbopack_module_id__: string | number; export const __turbopack_module_id__: string | number;
export const chunks: string[]; export const chunks: string[];
export const chunkListPath: string;
...@@ -3,8 +3,6 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/ ...@@ -3,8 +3,6 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/
("TURBOPACK { chunking-type: isolatedParallel }"); ("TURBOPACK { chunking-type: isolatedParallel }");
import { __turbopack_module_id__ as id } from "CLIENT_MODULE"; import { __turbopack_module_id__ as id } from "CLIENT_MODULE";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS"; import client_id, { chunks } from "CLIENT_CHUNKS";
export default createProxy( export default createProxy(JSON.stringify([client_id, chunks, id]));
JSON.stringify([client_id, chunks, chunkListPath, id])
);
import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy"; import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS"; import client_id, { chunks } from "CLIENT_CHUNKS";
export default createProxy(JSON.stringify([client_id, chunks, chunkListPath])); export default createProxy(JSON.stringify([client_id, chunks]));
declare global { declare global {
function __turbopack_require__(name: any): any; function __turbopack_require__(name: any): any;
function __turbopack_load__(path: string): any; function __turbopack_load__(path: string): any;
function __turbopack_register_chunk_list__(
chunkListPath: string,
chunksPaths: string[]
): any;
function __webpack_require__(name: any): any; function __webpack_require__(name: any): any;
var __webpack_public_path__: string | undefined; var __webpack_public_path__: string | undefined;
var __DEV_MIDDLEWARE_MATCHERS: any[]; var __DEV_MIDDLEWARE_MATCHERS: any[];
......
...@@ -80,5 +80,4 @@ declare module "CLIENT_CHUNKS" { ...@@ -80,5 +80,4 @@ declare module "CLIENT_CHUNKS" {
export default moduleId; export default moduleId;
export const chunks: any[]; export const chunks: any[];
export const chunkListPath: string;
} }
...@@ -5,7 +5,7 @@ use turbo_binding::{ ...@@ -5,7 +5,7 @@ use turbo_binding::{
turbo::{tasks_env::ProcessEnvVc, tasks_fs::FileSystemPathVc}, turbo::{tasks_env::ProcessEnvVc, tasks_fs::FileSystemPathVc},
turbopack::{ turbopack::{
core::{ core::{
chunk::ChunkGroupVc, chunk::{ChunkableAsset, ChunkingContext},
compile_time_info::CompileTimeInfoVc, compile_time_info::CompileTimeInfoVc,
context::AssetContextVc, context::AssetContextVc,
resolve::{options::ImportMap, origin::PlainResolveOriginVc}, resolve::{options::ImportMap, origin::PlainResolveOriginVc},
...@@ -85,10 +85,9 @@ pub async fn get_fallback_page( ...@@ -85,10 +85,9 @@ pub async fn get_fallback_page(
Ok(DevHtmlAssetVc::new( Ok(DevHtmlAssetVc::new(
dev_server_root.join("fallback.html"), dev_server_root.join("fallback.html"),
vec![ChunkGroupVc::evaluated( vec![chunking_context.evaluated_chunk_group(
chunking_context, module.as_root_chunk(chunking_context),
module.into(), runtime_entries.with_entry(module.into()),
runtime_entries,
)], )],
)) ))
} }
...@@ -3,19 +3,20 @@ use std::io::Write; ...@@ -3,19 +3,20 @@ use std::io::Write;
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use indoc::writedoc; use indoc::writedoc;
use turbo_binding::{ use turbo_binding::{
turbo::tasks::TryJoinIterExt, turbo::{
tasks::{primitives::StringVc, TryJoinIterExt, Value},
tasks_fs::rope::RopeBuilder,
},
turbopack::{ turbopack::{
core::{ core::{
asset::{Asset, AssetContentVc, AssetVc}, asset::{Asset, AssetContentVc, AssetVc, AssetsVc},
chunk::{ chunk::{
availability_info::AvailabilityInfo, ChunkGroupReferenceVc, ChunkGroupVc, availability_info::AvailabilityInfo, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc,
ChunkItem, ChunkItemVc, ChunkVc, ChunkableAsset, ChunkableAssetVc, ChunkingContext, ChunkVc, ChunkableAsset, ChunkableAssetVc, ChunkingContext, ChunkingContextVc,
ChunkingContextVc,
}, },
ident::AssetIdentVc, ident::AssetIdentVc,
reference::AssetReferencesVc, reference::AssetReferencesVc,
}, },
dev::{ChunkListReferenceVc, DevChunkingContextVc},
ecmascript::{ ecmascript::{
chunk::{ chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc,
...@@ -27,8 +28,6 @@ use turbo_binding::{ ...@@ -27,8 +28,6 @@ use turbo_binding::{
}, },
}, },
}; };
use turbo_tasks::{primitives::StringVc, Value};
use turbo_tasks_fs::rope::RopeBuilder;
#[turbo_tasks::function] #[turbo_tasks::function]
fn modifier() -> StringVc { fn modifier() -> StringVc {
...@@ -56,18 +55,13 @@ impl Asset for WithChunksAsset { ...@@ -56,18 +55,13 @@ impl Asset for WithChunksAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(self_vc: WithChunksAssetVc) -> Result<AssetReferencesVc> { async fn references(self_vc: WithChunksAssetVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?; let this = self_vc.await?;
let chunk_group = self_vc.chunk_group(); let entry_chunk = self_vc.entry_chunk();
let mut references = vec![ChunkGroupReferenceVc::new(chunk_group).into()];
if let Some(context) = DevChunkingContextVc::resolve_from(this.chunking_context).await? {
references.push(
ChunkListReferenceVc::new(context, chunk_group.entry(), chunk_group.chunks())
.into(),
);
}
Ok(AssetReferencesVc::cell(references)) Ok(AssetReferencesVc::cell(vec![ChunkGroupReferenceVc::new(
this.chunking_context,
entry_chunk,
)
.into()]))
} }
} }
...@@ -113,15 +107,15 @@ impl EcmascriptChunkPlaceable for WithChunksAsset { ...@@ -113,15 +107,15 @@ impl EcmascriptChunkPlaceable for WithChunksAsset {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WithChunksAssetVc { impl WithChunksAssetVc {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn chunk_group(self) -> Result<ChunkGroupVc> { async fn entry_chunk(self) -> Result<ChunkVc> {
let this = self.await?; let this = self.await?;
Ok(ChunkGroupVc::from_asset( Ok(this.asset.as_root_chunk(this.chunking_context))
this.asset.into(), }
this.chunking_context,
Value::new(AvailabilityInfo::Root { #[turbo_tasks::function]
current_availability_root: this.asset.into(), async fn chunks(self) -> Result<AssetsVc> {
}), let this = self.await?;
)) Ok(this.chunking_context.chunk_group(self.entry_chunk()))
} }
} }
...@@ -145,8 +139,7 @@ impl EcmascriptChunkItem for WithChunksChunkItem { ...@@ -145,8 +139,7 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
bail!("the chunking context is not an EcmascriptChunkingContextVc"); bail!("the chunking context is not an EcmascriptChunkingContextVc");
}; };
let group = self.inner.chunk_group(); let chunks = self.inner.chunks().await?;
let chunks = group.chunks().await?;
let server_root = inner_chunking_context.output_root().await?; let server_root = inner_chunking_context.output_root().await?;
let mut client_chunks = Vec::new(); let mut client_chunks = Vec::new();
...@@ -174,29 +167,10 @@ impl EcmascriptChunkItem for WithChunksChunkItem { ...@@ -174,29 +167,10 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
__turbopack_esm__({{ __turbopack_esm__({{
default: () => {}, default: () => {},
chunks: () => chunks, chunks: () => chunks,
"#,
StringifyJs(&module_id),
)?;
if let Some(dev_chunking_context) =
DevChunkingContextVc::resolve_from(inner.chunking_context).await?
{
let chunk_list_path = dev_chunking_context
.chunk_list_path(group.entry().ident())
.await?;
if let Some(path) = server_root.get_path_to(&chunk_list_path) {
writeln!(code, " chunkListPath: () => {}", StringifyJs(&path))?;
} else {
bail!("could not get path to chunk list");
}
}
writedoc!(
code,
r#"
}}); }});
const chunks = {:#}; const chunks = {:#};
"#, "#,
StringifyJs(&module_id),
StringifyJs(&client_chunks), StringifyJs(&client_chunks),
)?; )?;
......
...@@ -6,7 +6,7 @@ use turbo_binding::{ ...@@ -6,7 +6,7 @@ use turbo_binding::{
core::{ core::{
asset::{Asset, AssetContentVc, AssetVc}, asset::{Asset, AssetContentVc, AssetVc},
chunk::{ chunk::{
availability_info::AvailabilityInfo, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkVc, availability_info::AvailabilityInfo, ChunkItem, ChunkItemVc, ChunkVc,
ChunkableAsset, ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAsset, ChunkableAssetReference, ChunkableAssetReferenceVc,
ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunkingType, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, ChunkingType,
ChunkingTypeOptionVc, ChunkingTypeOptionVc,
...@@ -121,14 +121,10 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem { ...@@ -121,14 +121,10 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn content(&self) -> Result<EcmascriptChunkItemContentVc> { async fn content(&self) -> Result<EcmascriptChunkItemContentVc> {
let inner = self.inner.await?; let inner = self.inner.await?;
let group = ChunkGroupVc::from_asset( let chunks = self
inner.asset.into(), .context
self.context.into(), .chunk_group(inner.asset.as_root_chunk(self.context.into()))
Value::new(AvailabilityInfo::Root { .await?;
current_availability_root: inner.asset.into(),
}),
);
let chunks = group.chunks().await?;
let server_root = inner.server_root.await?; let server_root = inner.server_root.await?;
let mut asset_paths = vec![]; let mut asset_paths = vec![];
......
...@@ -7,7 +7,7 @@ use turbo_binding::{ ...@@ -7,7 +7,7 @@ use turbo_binding::{
turbopack::{ turbopack::{
core::{ core::{
asset::{Asset, AssetContentVc, AssetVc, AssetsVc}, asset::{Asset, AssetContentVc, AssetVc, AssetsVc},
chunk::{ChunkGroupVc, ChunkingContextVc, EvaluatableAssetsVc}, chunk::{ChunkableAsset, ChunkingContext, ChunkingContextVc, EvaluatableAssetsVc},
context::{AssetContext, AssetContextVc}, context::{AssetContext, AssetContextVc},
ident::AssetIdentVc, ident::AssetIdentVc,
reference::{AssetReferencesVc, SingleAssetReferenceVc}, reference::{AssetReferencesVc, SingleAssetReferenceVc},
...@@ -100,13 +100,10 @@ impl PageLoaderAssetVc { ...@@ -100,13 +100,10 @@ impl PageLoaderAssetVc {
}), }),
); );
let chunk_group = ChunkGroupVc::evaluated( Ok(this.client_chunking_context.evaluated_chunk_group(
this.client_chunking_context, asset.as_root_chunk(this.client_chunking_context),
asset.into(), EvaluatableAssetsVc::one(asset.into()),
EvaluatableAssetsVc::empty(), ))
);
Ok(chunk_group.chunks())
} }
} }
......
...@@ -6,7 +6,7 @@ use turbo_binding::{ ...@@ -6,7 +6,7 @@ use turbo_binding::{
}, },
turbopack::{ turbopack::{
core::{ core::{
chunk::{ChunkGroupVc, ChunkableAsset, ChunkableAssetVc}, chunk::{ChunkableAsset, ChunkableAssetVc, ChunkingContext},
compile_time_defines, compile_time_defines,
compile_time_info::{CompileTimeDefinesVc, CompileTimeInfo, CompileTimeInfoVc}, compile_time_info::{CompileTimeDefinesVc, CompileTimeInfo, CompileTimeInfoVc},
environment::{ environment::{
...@@ -147,16 +147,15 @@ pub async fn create_web_entry_source( ...@@ -147,16 +147,15 @@ pub async fn create_web_entry_source(
.flatten() .flatten()
.map(|module| async move { .map(|module| async move {
if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? { if let Some(ecmascript) = EcmascriptModuleAssetVc::resolve_from(module).await? {
let chunk_group = let chunk_group = chunking_context.evaluated_chunk_group(
ChunkGroupVc::evaluated(chunking_context, ecmascript.into(), runtime_entries); ecmascript.as_root_chunk(chunking_context),
runtime_entries.with_entry(ecmascript.into()),
);
Ok(chunk_group) Ok(chunk_group)
} else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? { } else if let Some(chunkable) = ChunkableAssetVc::resolve_from(module).await? {
// TODO this is missing runtime code, so it's probably broken and we should also // TODO this is missing runtime code, so it's probably broken and we should also
// add an ecmascript chunk with the runtime code // add an ecmascript chunk with the runtime code
Ok(ChunkGroupVc::from_chunk( Ok(chunking_context.chunk_group(chunkable.as_root_chunk(chunking_context)))
chunking_context,
chunkable.as_root_chunk(chunking_context),
))
} else { } else {
// TODO convert into a serve-able asset // TODO convert into a serve-able asset
Err(anyhow!( Err(anyhow!(
......
...@@ -1012,7 +1012,7 @@ importers: ...@@ -1012,7 +1012,7 @@ importers:
'@types/react': ^18.0.26 '@types/react': ^18.0.26
'@types/react-dom': ^18.0.9 '@types/react-dom': ^18.0.9
'@vercel/ncc': ^0.36.0 '@vercel/ncc': ^0.36.0
'@vercel/turbopack-dev-runtime': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2 '@vercel/turbopack-dev-runtime': https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2
anser: ^2.1.1 anser: ^2.1.1
css.escape: ^1.5.1 css.escape: ^1.5.1
find-up: ^6.3.0 find-up: ^6.3.0
...@@ -1024,7 +1024,7 @@ importers: ...@@ -1024,7 +1024,7 @@ importers:
stacktrace-parser: ^0.1.10 stacktrace-parser: ^0.1.10
strip-ansi: ^7.0.1 strip-ansi: ^7.0.1
dependencies: dependencies:
'@vercel/turbopack-dev-runtime': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2_react-refresh@0.12.0' '@vercel/turbopack-dev-runtime': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2_react-refresh@0.12.0'
anser: 2.1.1 anser: 2.1.1
css.escape: 1.5.1 css.escape: 1.5.1
next: link:../../../../next next: link:../../../../next
...@@ -25577,9 +25577,9 @@ packages: ...@@ -25577,9 +25577,9 @@ packages:
/zwitch/2.0.4: /zwitch/2.0.4:
resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==}
'@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2_react-refresh@0.12.0': '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2_react-refresh@0.12.0':
resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2} resolution: {tarball: https://gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2}
id: '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230406.2' id: '@gitpkg.vercel.app/vercel/turbo/crates/turbopack-dev/js?turbopack-230411.2'
name: '@vercel/turbopack-dev-runtime' name: '@vercel/turbopack-dev-runtime'
version: 0.0.0 version: 0.0.0
dependencies: dependencies:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment