Commit d09b769e authored by hrmny's avatar hrmny Committed by Janka Uryga

feat(turbopack): add support for esm externals in app dir (#64918)

Writes the async flag to the client reference manifest to support ESM
externals in app dir.

The `app-ssr` context can't have esm externals as that might cause a
module to be async in ssr but not on the client.

Closes PACK-2967
Fixes #64525
parent dafdc81b
...@@ -846,13 +846,13 @@ impl AppEndpoint { ...@@ -846,13 +846,13 @@ impl AppEndpoint {
{ {
entry_client_chunks.extend(chunks.await?.iter().copied()); entry_client_chunks.extend(chunks.await?.iter().copied());
} }
for chunks in client_references_chunks_ref for (chunks, _) in client_references_chunks_ref
.client_component_client_chunks .client_component_client_chunks
.values() .values()
{ {
client_assets.extend(chunks.await?.iter().copied()); client_assets.extend(chunks.await?.iter().copied());
} }
for chunks in client_references_chunks_ref for (chunks, _) in client_references_chunks_ref
.client_component_ssr_chunks .client_component_ssr_chunks
.values() .values()
{ {
...@@ -929,7 +929,7 @@ impl AppEndpoint { ...@@ -929,7 +929,7 @@ impl AppEndpoint {
// initialization // initialization
let client_references_chunks = &*client_references_chunks.await?; let client_references_chunks = &*client_references_chunks.await?;
for &ssr_chunks in client_references_chunks for (ssr_chunks, _) in client_references_chunks
.client_component_ssr_chunks .client_component_ssr_chunks
.values() .values()
{ {
......
...@@ -32,8 +32,10 @@ fn client_modules_ssr_modifier() -> Vc<String> { ...@@ -32,8 +32,10 @@ fn client_modules_ssr_modifier() -> Vc<String> {
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct ClientReferencesChunks { pub struct ClientReferencesChunks {
pub client_component_client_chunks: IndexMap<ClientReferenceType, Vc<OutputAssets>>, pub client_component_client_chunks:
pub client_component_ssr_chunks: IndexMap<ClientReferenceType, Vc<OutputAssets>>, IndexMap<ClientReferenceType, (Vc<OutputAssets>, AvailabilityInfo)>,
pub client_component_ssr_chunks:
IndexMap<ClientReferenceType, (Vc<OutputAssets>, AvailabilityInfo)>,
pub layout_segment_client_chunks: IndexMap<Vc<NextServerComponentModule>, Vc<OutputAssets>>, pub layout_segment_client_chunks: IndexMap<Vc<NextServerComponentModule>, Vc<OutputAssets>>,
} }
...@@ -66,23 +68,47 @@ pub async fn get_app_client_references_chunks( ...@@ -66,23 +68,47 @@ pub async fn get_app_client_references_chunks(
) => { ) => {
let ecmascript_client_reference_ref = let ecmascript_client_reference_ref =
ecmascript_client_reference.await?; ecmascript_client_reference.await?;
(
client_chunking_context.root_chunk_group_assets(Vc::upcast( let client_chunk_group = client_chunking_context
.root_chunk_group(Vc::upcast(
ecmascript_client_reference_ref.client_module, ecmascript_client_reference_ref.client_module,
)), ))
ssr_chunking_context.map(|ssr_chunking_context| { .await?;
ssr_chunking_context.root_chunk_group_assets(Vc::upcast(
ecmascript_client_reference_ref.ssr_module, (
(
client_chunk_group.assets,
client_chunk_group.availability_info,
),
if let Some(ssr_chunking_context) = ssr_chunking_context {
let ssr_chunk_group = ssr_chunking_context
.root_chunk_group(Vc::upcast(
ecmascript_client_reference_ref.ssr_module,
))
.await?;
Some((
ssr_chunk_group.assets,
ssr_chunk_group.availability_info,
)) ))
}), } else {
None
},
) )
} }
ClientReferenceType::CssClientReference(css_client_reference) => { ClientReferenceType::CssClientReference(css_client_reference) => {
let css_client_reference_ref = css_client_reference.await?; let css_client_reference_ref = css_client_reference.await?;
( let client_chunk_group = client_chunking_context
client_chunking_context.root_chunk_group_assets(Vc::upcast( .root_chunk_group(Vc::upcast(
css_client_reference_ref.client_module, css_client_reference_ref.client_module,
)), ))
.await?;
(
(
client_chunk_group.assets,
client_chunk_group.availability_info,
),
None, None,
) )
} }
...@@ -165,6 +191,7 @@ pub async fn get_app_client_references_chunks( ...@@ -165,6 +191,7 @@ pub async fn get_app_client_references_chunks(
}) })
.try_flat_join() .try_flat_join()
.await?; .await?;
let ssr_chunk_group = if !ssr_modules.is_empty() { let ssr_chunk_group = if !ssr_modules.is_empty() {
let ssr_entry_module = IncludeModulesModule::new( let ssr_entry_module = IncludeModulesModule::new(
base_ident.with_modifier(client_modules_ssr_modifier()), base_ident.with_modifier(client_modules_ssr_modifier()),
...@@ -184,6 +211,7 @@ pub async fn get_app_client_references_chunks( ...@@ -184,6 +211,7 @@ pub async fn get_app_client_references_chunks(
} else { } else {
None None
}; };
let client_modules = client_reference_types let client_modules = client_reference_types
.iter() .iter()
.map(|client_reference_ty| async move { .map(|client_reference_ty| async move {
...@@ -240,8 +268,10 @@ pub async fn get_app_client_references_chunks( ...@@ -240,8 +268,10 @@ pub async fn get_app_client_references_chunks(
if let ClientReferenceType::EcmascriptClientReference(_) = if let ClientReferenceType::EcmascriptClientReference(_) =
client_reference_ty client_reference_ty
{ {
client_component_client_chunks client_component_client_chunks.insert(
.insert(client_reference_ty, client_chunks); client_reference_ty,
(client_chunks, client_chunk_group.availability_info),
);
} }
} }
} }
...@@ -261,7 +291,10 @@ pub async fn get_app_client_references_chunks( ...@@ -261,7 +291,10 @@ pub async fn get_app_client_references_chunks(
if let ClientReferenceType::EcmascriptClientReference(_) = if let ClientReferenceType::EcmascriptClientReference(_) =
client_reference_ty client_reference_ty
{ {
client_component_ssr_chunks.insert(client_reference_ty, ssr_chunks); client_component_ssr_chunks.insert(
client_reference_ty,
(ssr_chunks, ssr_chunk_group.availability_info),
);
} }
} }
} }
......
...@@ -5,7 +5,10 @@ use turbo_tasks_fs::{File, FileSystemPath}; ...@@ -5,7 +5,10 @@ use turbo_tasks_fs::{File, FileSystemPath};
use turbopack_binding::turbopack::{ use turbopack_binding::turbopack::{
core::{ core::{
asset::AssetContent, asset::AssetContent,
chunk::{ChunkItemExt, ChunkableModule, ModuleId as TurbopackModuleId}, chunk::{
availability_info::AvailabilityInfo, ChunkItem, ChunkItemExt, ChunkableModule,
ModuleId as TurbopackModuleId,
},
output::OutputAsset, output::OutputAsset,
virtual_output::VirtualOutputAsset, virtual_output::VirtualOutputAsset,
}, },
...@@ -66,61 +69,70 @@ impl ClientReferenceManifest { ...@@ -66,61 +69,70 @@ impl ClientReferenceManifest {
.to_string() .to_string()
.await?; .await?;
let client_module_id = ecmascript_client_reference let client_chunk_item = ecmascript_client_reference
.client_module .client_module
.as_chunk_item(Vc::upcast(client_chunking_context)) .as_chunk_item(Vc::upcast(client_chunking_context));
.id()
.await?; let client_module_id = client_chunk_item.id().await?;
let (client_chunks_paths, client_is_async) =
if let Some((client_chunks, client_availability_info)) =
client_references_chunks
.client_component_client_chunks
.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
let client_chunks_paths = client_chunks
.iter()
.map(|chunk| chunk.ident().path())
.try_join()
.await?;
let chunk_paths = client_chunks_paths
.iter()
.filter_map(|chunk_path| client_relative_path.get_path_to(chunk_path))
.map(ToString::to_string)
// It's possible that a chunk also emits CSS files, that will
// be handled separatedly.
.filter(|path| path.ends_with(".js"))
// .map(RcStr::from) // BACKPORT: no RcStr
.collect::<Vec<_>>();
let is_async =
is_item_async(client_availability_info, client_chunk_item).await?;
(chunk_paths, is_async)
} else {
(Vec::new(), false)
};
let client_chunks_paths = if let Some(client_chunks) = client_references_chunks
.client_component_client_chunks
.get(&app_client_reference_ty)
{
let client_chunks = client_chunks.await?;
let client_chunks_paths = client_chunks
.iter()
.map(|chunk| chunk.ident().path())
.try_join()
.await?;
client_chunks_paths
.iter()
.filter_map(|chunk_path| client_relative_path.get_path_to(chunk_path))
.map(ToString::to_string)
// It's possible that a chunk also emits CSS files, that will
// be handled separatedly.
.filter(|path| path.ends_with(".js"))
.collect::<Vec<_>>()
} else {
Vec::new()
};
entry_manifest.client_modules.module_exports.insert( entry_manifest.client_modules.module_exports.insert(
get_client_reference_module_key(&server_path, "*"), get_client_reference_module_key(&server_path, "*"),
ManifestNodeEntry { ManifestNodeEntry {
name: "*".to_string(), name: "*".to_string(),
id: (&*client_module_id).into(), id: (&*client_module_id).into(),
chunks: client_chunks_paths, chunks: client_chunks_paths,
// TODO(WEB-434) r#async: client_is_async,
r#async: false,
}, },
); );
if let Some(ssr_chunking_context) = ssr_chunking_context { if let Some(ssr_chunking_context) = ssr_chunking_context {
let ssr_module_id = ecmascript_client_reference let ssr_chunk_item = ecmascript_client_reference
.ssr_module .ssr_module
.as_chunk_item(Vc::upcast(ssr_chunking_context)) .as_chunk_item(Vc::upcast(ssr_chunking_context));
.id()
.await?;
let ssr_chunks_paths = if runtime == NextRuntime::Edge { let ssr_module_id = ssr_chunk_item.id().await?;
let (ssr_chunks_paths, ssr_is_async) = if runtime == NextRuntime::Edge {
// the chunks get added to the middleware-manifest.json instead // the chunks get added to the middleware-manifest.json instead
// of this file because the // of this file because the
// edge runtime doesn't support dynamically // edge runtime doesn't support dynamically
// loading chunks. // loading chunks.
Vec::new() (Vec::new(), false)
} else if let Some(ssr_chunks) = client_references_chunks } else if let Some((ssr_chunks, ssr_availability_info)) =
.client_component_ssr_chunks client_references_chunks
.get(&app_client_reference_ty) .client_component_ssr_chunks
.get(&app_client_reference_ty)
{ {
let ssr_chunks = ssr_chunks.await?; let ssr_chunks = ssr_chunks.await?;
...@@ -130,14 +142,20 @@ impl ClientReferenceManifest { ...@@ -130,14 +142,20 @@ impl ClientReferenceManifest {
.try_join() .try_join()
.await?; .await?;
ssr_chunks_paths let chunk_paths = ssr_chunks_paths
.iter() .iter()
.filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path)) .filter_map(|chunk_path| node_root_ref.get_path_to(chunk_path))
.map(ToString::to_string) .map(ToString::to_string)
.collect::<Vec<_>>() // .map(RcStr::from) // BACKPORT: no RcStr
.collect::<Vec<_>>();
let is_async = is_item_async(ssr_availability_info, ssr_chunk_item).await?;
(chunk_paths, is_async)
} else { } else {
Vec::new() (Vec::new(), false)
}; };
let mut ssr_manifest_node = ManifestNode::default(); let mut ssr_manifest_node = ManifestNode::default();
ssr_manifest_node.module_exports.insert( ssr_manifest_node.module_exports.insert(
"*".to_string(), "*".to_string(),
...@@ -145,8 +163,7 @@ impl ClientReferenceManifest { ...@@ -145,8 +163,7 @@ impl ClientReferenceManifest {
name: "*".to_string(), name: "*".to_string(),
id: (&*ssr_module_id).into(), id: (&*ssr_module_id).into(),
chunks: ssr_chunks_paths, chunks: ssr_chunks_paths,
// TODO(WEB-434) r#async: ssr_is_async,
r#async: false,
}, },
); );
...@@ -250,3 +267,18 @@ pub fn get_client_reference_module_key(server_path: &str, export_name: &str) -> ...@@ -250,3 +267,18 @@ pub fn get_client_reference_module_key(server_path: &str, export_name: &str) ->
format!("{}#{}", server_path, export_name) format!("{}#{}", server_path, export_name)
} }
} }
async fn is_item_async(
availability_info: &AvailabilityInfo,
chunk_item: Vc<Box<dyn ChunkItem>>,
) -> Result<bool> {
let Some(available_chunk_items) = availability_info.available_chunk_items() else {
return Ok(false);
};
let Some(info) = &*available_chunk_items.get(chunk_item).await? else {
return Ok(false);
};
Ok(info.is_async)
}
...@@ -146,14 +146,15 @@ pub async fn get_server_resolve_options_context( ...@@ -146,14 +146,15 @@ pub async fn get_server_resolve_options_context(
.cloned(), .cloned(),
); );
let ty = ty.into_value();
let server_component_externals_plugin = ExternalCjsModulesResolvePlugin::new( let server_component_externals_plugin = ExternalCjsModulesResolvePlugin::new(
project_path, project_path,
project_path.root(), project_path.root(),
ExternalPredicate::Only(Vc::cell(external_packages)).cell(), ExternalPredicate::Only(Vc::cell(external_packages)).cell(),
// TODO(sokra) esmExternals support // app-ssr can't have esm externals as that would make the module async on the server only
false, *next_config.import_externals().await? && !matches!(ty, ServerContextType::AppSSR { .. }),
); );
let ty = ty.into_value();
let mut custom_conditions = vec![mode.await?.condition().to_string()]; let mut custom_conditions = vec![mode.await?.condition().to_string()];
custom_conditions.extend( custom_conditions.extend(
......
...@@ -16,7 +16,7 @@ async function resolveStreamResponse(response: any, onData?: any) { ...@@ -16,7 +16,7 @@ async function resolveStreamResponse(response: any, onData?: any) {
} }
describe('app dir - external dependency', () => { describe('app dir - external dependency', () => {
const { next, skipped, isTurbopack } = nextTestSetup({ const { next, skipped } = nextTestSetup({
files: __dirname, files: __dirname,
dependencies: { dependencies: {
swr: 'latest', swr: 'latest',
...@@ -46,7 +46,7 @@ describe('app dir - external dependency', () => { ...@@ -46,7 +46,7 @@ describe('app dir - external dependency', () => {
expect(result).toContain('Server subpath: subpath.default') expect(result).toContain('Server subpath: subpath.default')
expect(result).toContain('Client: index.default') expect(result).toContain('Client: index.default')
expect(result).toContain('Client subpath: subpath.default') expect(result).toContain('Client subpath: subpath.default')
expect(result).toContain('opt-out-react-version: 18.3.1') expect(result).not.toContain('opt-out-react-version: 18.3.0-canary')
}) })
}) })
...@@ -286,10 +286,7 @@ describe('app dir - external dependency', () => { ...@@ -286,10 +286,7 @@ describe('app dir - external dependency', () => {
browser.elementByCss('#dual-pkg-outout button').click() browser.elementByCss('#dual-pkg-outout button').click()
await check(async () => { await check(async () => {
const text = await browser.elementByCss('#dual-pkg-outout p').text() const text = await browser.elementByCss('#dual-pkg-outout p').text()
// TODO: enable esm externals for app router in turbopack for actions expect(text).toBe('dual-pkg-optout:mjs')
expect(text).toBe(
isTurbopack ? 'dual-pkg-optout:cjs' : 'dual-pkg-optout:mjs'
)
return 'success' return 'success'
}, /success/) }, /success/)
}) })
......
...@@ -8,65 +8,63 @@ describe('esm-externals', () => { ...@@ -8,65 +8,63 @@ describe('esm-externals', () => {
const { next, isTurbopack } = nextTestSetup({ const { next, isTurbopack } = nextTestSetup({
files: __dirname, files: __dirname,
}) })
// Pages // Pages
{ describe.each(['/static', '/ssr', '/ssg'])('pages url %s', (url) => {
const urls = ['/static', '/ssr', '/ssg'] // For invalid esm packages that have "import" pointing to a non-esm-flagged module
// webpack is using the CJS version instead, but Turbopack is opting out of
// externalizing and bundling the non-esm-flagged module.
const expectedHtml = isTurbopack
? 'Hello World+World+World+World+World+World'
: url === '/static'
? 'Hello World+World+Alternative+World+World+World'
: 'Hello World+World+Alternative+World+World+Alternative'
for (const url of urls) { // On the client side, webpack always bundles, so it uses the non-esm-flagged module too.
// For invalid esm packages that have "import" pointing to a non-esm-flagged module const expectedText =
// webpack is using the CJS version instead but Turbopack is opting out of isTurbopack || url === '/static'
// externalizing and bundling the non-esm-flagged module. ? 'Hello World+World+World+World+World+World'
const expectedHtml = isTurbopack : 'Hello World+World+World+World+World+Alternative'
? /Hello World\+World\+World\+World\+World\+World/
: url === '/static'
? /Hello World\+World\+Alternative\+World\+World\+World/
: /Hello World\+World\+Alternative\+World\+World\+Alternative/
// On client side, webpack always bundlings so it uses the non-esm-flagged module too.
const expectedText =
isTurbopack || url === '/static'
? /Hello World\+World\+World\+World\+World\+World/
: /Hello World\+World\+World\+World\+World\+Alternative/
it(`should return the correct SSR HTML for ${url}`, async () => { it('should return the correct SSR HTML', async () => {
const res = await next.fetch(url) const $ = await next.render$(url)
const html = await res.text() const body = $('body > div > div').html()
expect(normalize(html)).toMatch(expectedHtml) expect(normalize(body)).toEqual(expectedHtml)
}) })
it(`should render the correct page for ${url}`, async () => { it('should render the correct page', async () => {
const browser = await next.browser(url) const browser = await next.browser(url)
expect(await browser.elementByCss('body').text()).toMatch(expectedText) expect(await browser.elementByCss('body > div').text()).toEqual(
}) expectedText
} )
} })
})
// App dir // App dir
{ describe.each(['/server', '/client'])('app dir url %s', (url) => {
// TODO App Dir doesn't use esmExternals: true correctly for Turbopack
// so we only verify that the page doesn't crash, but ignore the actual content
// const expectedHtml = isTurbopack ? /Hello World\+World\+World/ : /Hello World\+World\+Alternative/
const expectedHtml = isTurbopack const expectedHtml = isTurbopack
? /Hello Wrong\+Wrong\+Alternative/ ? url === '/client'
: /Hello World\+World\+Alternative/ ? 'Hello Wrong+Wrong+Alternative'
const urls = ['/server', '/client'] : 'Hello World+World+World'
: 'Hello World+World+Alternative'
for (const url of urls) { const expectedText = isTurbopack
const expectedText = ? 'Hello World+World+World'
url !== '/server' : url === '/client'
? /Hello World\+World\+World/ ? 'Hello World+World+World'
: isTurbopack : 'Hello World+World+Alternative'
? /Hello Wrong\+Wrong\+Alternative/
: /Hello World\+World\+Alternative/
it(`should return the correct SSR HTML for ${url}`, async () => {
const res = await next.fetch(url)
const html = await res.text()
expect(normalize(html)).toMatch(expectedHtml)
})
it(`should render the correct page for ${url}`, async () => { it('should return the correct SSR HTML', async () => {
const browser = await next.browser(url) const $ = await next.render$(url)
expect(await browser.elementByCss('body').text()).toMatch(expectedText) const body = $('body > div').html()
}) expect(normalize(body)).toEqual(expectedHtml)
} })
}
it('should render the correct page', async () => {
const browser = await next.browser(url)
expect(await browser.elementByCss('body > div').text()).toEqual(
expectedText
)
})
})
}) })
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment