Unverified Commit 09296125 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Add bundling support for worker_threads (#87746)

Add support for bundling worker_threads workers

Previously we only supported tracing dependencies of worker threads.  However, this could trigger issues if the callsite was getting bundled since we would believe that the file was both bundled and unbundled.

The solution is to support bundling modules using worker threads and implementing the threads.  This requires a new reference and loader module type which follows the patter of async-loaders and webworkers. 

An interesting consideration is about file extensions.  `new Worker(...)` takes a relative or absolute filepath, in node 23+ that means you can load typescript files directly, so we support the same.  If you want to load a `.ts` file you need to write `foo.ts` we don't do any pattern rewriting.

This fixes a bug reported by an X user: https://x.com/strugglercss/status/2002504736860484008?s=20.  The problem with the previous approach is that we would record the reference as 'traced', but if you did `new Worker(__filename,...)` then we would end up with 'traced references' to a file that was bundled.  This caused issues during tracing when we tried to add CachedExternalModule references to the nft json files

```
Error [TurbopackInternalError]: NftJsonAsset: cannot handle filepath url

Debug info:
- Execution of get_all_written_entrypoints_with_issues_operation failed
- Execution of EntrypointsOperation::new failed
- Execution of all_entrypoints_write_to_disk_operation failed
- Execution of Project::emit_all_output_assets failed
- Execution of emit_assets failed
- Execution of emit failed
- Execution of <NftJsonAsset as Asset>::content failed
- NftJsonAsset: cannot handle filepath url
    at <unknown> (TurbopackInternalError: NftJsonAsset: cannot handle filepath url) {
```

there `url` was due to a `require('url')` statement in a bundled file this caused us to produce an external shim, but a `new Worker(__filename)` caused us to also create a traced reference to the same file.  The problem there was simply that the `ident()` for a `CachedExternalModule` contains a dummy filepath that broke the nft.json asset production.   The root issue here is that the `FilePathModuleReference` was using the wrong `AssetContext` to construct the referenced module which lead to the wrong files being traced.   After investigation @mischnic and I determined that that was an insufficient fix since we really do need to rewrite the reference.   Proper bundling support fixes this directly and the pattern here paves the way for more worker support in the future.

This doesn't seem like the only way a synthetic module (like an extern module) might end up as a dependency of traced module, so a more robust solution is probably in order.
parent 05406f6e
edition = "2024"
max_width = 100 max_width = 100
comment_width = 100 comment_width = 100
......
...@@ -14,10 +14,7 @@ use turbo_tasks::{Completion, ResolvedVc, Vc}; ...@@ -14,10 +14,7 @@ use turbo_tasks::{Completion, ResolvedVc, Vc};
use turbo_tasks_fs::{self, File, FileContent, FileSystemPath}; use turbo_tasks_fs::{self, File, FileContent, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::AssetContent, asset::AssetContent,
chunk::{ chunk::{ChunkingContextExt, EntryChunkGroupResult, availability_info::AvailabilityInfo},
ChunkingContext, ChunkingContextExt, EntryChunkGroupResult,
availability_info::AvailabilityInfo,
},
context::AssetContext, context::AssetContext,
module::Module, module::Module,
module_graph::{ module_graph::{
...@@ -138,7 +135,7 @@ impl MiddlewareEndpoint { ...@@ -138,7 +135,7 @@ impl MiddlewareEndpoint {
}; };
let EntryChunkGroupResult { asset: chunk, .. } = *chunking_context let EntryChunkGroupResult { asset: chunk, .. } = *chunking_context
.entry_chunk_group( .root_entry_chunk_group(
this.project this.project
.node_root() .node_root()
.await? .await?
...@@ -147,7 +144,6 @@ impl MiddlewareEndpoint { ...@@ -147,7 +144,6 @@ impl MiddlewareEndpoint {
module_graph, module_graph,
OutputAssets::empty(), OutputAssets::empty(),
OutputAssets::empty(), OutputAssets::empty(),
AvailabilityInfo::root(),
) )
.await?; .await?;
Ok(*chunk) Ok(*chunk)
......
...@@ -76,30 +76,27 @@ impl OutputAsset for NftJsonAsset { ...@@ -76,30 +76,27 @@ impl OutputAsset for NftJsonAsset {
} }
} }
#[turbo_tasks::value(transparent)]
pub struct OutputSpecifier(Option<RcStr>);
fn get_output_specifier( fn get_output_specifier(
path_ref: &FileSystemPath, chunk_path: &FileSystemPath,
ident_folder: &FileSystemPath, ident_folder: &FileSystemPath,
ident_folder_in_project_fs: &FileSystemPath, ident_folder_in_project_fs: &FileSystemPath,
output_root: &FileSystemPath, output_root: &FileSystemPath,
project_root: &FileSystemPath, project_root: &FileSystemPath,
) -> Result<RcStr> { ) -> Option<RcStr> {
// include assets in the outputs such as referenced chunks // include assets in the outputs such as referenced chunks
if path_ref.is_inside_ref(output_root) { if chunk_path.is_inside_ref(output_root) {
return Ok(ident_folder.get_relative_path_to(path_ref).unwrap()); return Some(ident_folder.get_relative_path_to(chunk_path).unwrap());
} }
// include assets in the project root such as images and traced references (externals) // include assets in the project root such as images and traced references (externals)
if path_ref.is_inside_ref(project_root) { if chunk_path.is_inside_ref(project_root) {
return Ok(ident_folder_in_project_fs return Some(
.get_relative_path_to(path_ref) ident_folder_in_project_fs
.unwrap()); .get_relative_path_to(chunk_path)
.unwrap(),
);
} }
None
// This should effectively be unreachable
bail!("NftJsonAsset: cannot handle filepath {path_ref}");
} }
/// Apply outputFileTracingIncludes patterns to find additional files /// Apply outputFileTracingIncludes patterns to find additional files
...@@ -285,13 +282,21 @@ impl Asset for NftJsonAsset { ...@@ -285,13 +282,21 @@ impl Asset for NftJsonAsset {
} }
} }
let specifier = get_output_specifier( let Some(specifier) = get_output_specifier(
&referenced_chunk_path, &referenced_chunk_path,
&ident_folder, &ident_folder,
&ident_folder_in_project_fs, &ident_folder_in_project_fs,
&output_root_ref, &output_root_ref,
&project_root_ref, &project_root_ref,
)?; ) else {
// This should effectively be unreachable
bail!(
"NftJsonAsset: cannot handle filepath '{chunk_path}' for \
{referenced_chunk:?} it is not under the output_root: \
'{output_root_ref}' or the project_root: '{project_root_ref}'",
chunk_path = referenced_chunk_path.value_to_string().await?
);
};
result.insert(specifier); result.insert(specifier);
} }
......
import { Worker } from 'node:worker_threads'
import { NextResponse } from 'next/server'
export async function GET() {
try {
// Test with a relative path instead of __filename
const worker = new Worker('./app/worker-dir/simple-worker.ts')
const message = await new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker timeout'))
}, 5000)
worker.on('message', (msg) => {
clearTimeout(timeout)
resolve(msg)
})
worker.on('error', (err) => {
clearTimeout(timeout)
reject(err)
})
worker.on('exit', (code) => {
if (code !== 0) {
clearTimeout(timeout)
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
await worker.terminate()
return NextResponse.json({ success: true, message })
} catch (error) {
return NextResponse.json(
{ success: false, error: String(error) },
{ status: 500 }
)
}
}
import { Worker, isMainThread, parentPort } from 'node:worker_threads'
import { NextResponse } from 'next/server'
// This file tests self-referencing worker threads using __filename
// This pattern is used by libraries like @duckdb/duckdb-wasm
if (!isMainThread && parentPort) {
// Worker thread - handle messages
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort!.postMessage('pong')
}
})
}
export async function GET() {
if (!isMainThread) {
// If we're in a worker, don't try to create another worker
return NextResponse.json({ error: 'Already in worker' }, { status: 500 })
}
// Log __filename for debugging
console.log('__filename:', __filename)
console.log('typeof __filename:', typeof __filename)
try {
// Create a worker using __filename - this is the pattern that triggers the cycle bug
const worker = new Worker(__filename)
const message = await new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker timeout'))
}, 5000)
worker.on('message', (msg) => {
clearTimeout(timeout)
resolve(msg)
})
worker.on('error', (err) => {
clearTimeout(timeout)
reject(err)
})
worker.on('exit', (code) => {
if (code !== 0) {
clearTimeout(timeout)
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
await worker.terminate()
return NextResponse.json({ success: true, message })
} catch (error) {
return NextResponse.json(
{ success: false, error: String(error) },
{ status: 500 }
)
}
}
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html>
<body>{children}</body>
</html>
)
}
import { parentPort } from 'node:worker_threads'
if (parentPort) {
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort!.postMessage('pong from simple worker')
}
})
}
import { nextTestSetup } from 'e2e-utils'
describe('node-worker-threads', () => {
const { next, skipped, isTurbopack } = nextTestSetup({
files: __dirname,
skipDeployment: true,
})
if (skipped) {
return
}
// These tests are Turbopack-specific since they rely on Turbopack's worker bundling
if (!isTurbopack) {
it.skip('webpack doesnt support bundling worker-threads', () => {})
return
}
it('should handle simple worker with relative path', async () => {
const res = await next.fetch('/api/simple-worker-test')
const data = await res.json()
console.log('Simple worker response:', data)
expect(res.status).toBe(200)
expect(data.success).toBe(true)
expect(data.message).toBe('pong from simple worker')
})
it('should handle self-referencing worker with __filename', async () => {
const res = await next.fetch('/api/worker-test')
const data = await res.json()
console.log('Self-ref worker response:', data)
expect(res.status).toBe(200)
expect(data.success).toBe(true)
expect(data.message).toBe('pong')
})
})
...@@ -354,10 +354,7 @@ impl BrowserChunkingContext { ...@@ -354,10 +354,7 @@ impl BrowserChunkingContext {
} }
} }
} }
#[turbo_tasks::value_impl]
impl BrowserChunkingContext { impl BrowserChunkingContext {
#[turbo_tasks::function]
fn generate_evaluate_chunk( fn generate_evaluate_chunk(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
...@@ -374,8 +371,6 @@ impl BrowserChunkingContext { ...@@ -374,8 +371,6 @@ impl BrowserChunkingContext {
module_graph, module_graph,
)) ))
} }
#[turbo_tasks::function]
fn generate_chunk_list_register_chunk( fn generate_chunk_list_register_chunk(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
...@@ -391,26 +386,30 @@ impl BrowserChunkingContext { ...@@ -391,26 +386,30 @@ impl BrowserChunkingContext {
source, source,
)) ))
} }
#[turbo_tasks::function]
async fn generate_chunk( async fn generate_chunk(
self: Vc<Self>, self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>, chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<Vc<Box<dyn OutputAsset>>> { ) -> Result<ResolvedVc<Box<dyn OutputAsset>>> {
Ok( Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk) if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{ {
Vc::upcast(EcmascriptBrowserChunk::new(self, *ecmascript_chunk)) ResolvedVc::upcast(
EcmascriptBrowserChunk::new(self, *ecmascript_chunk)
.to_resolved()
.await?,
)
} else if let Some(output_asset) = } else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk) ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{ {
*output_asset output_asset
} else { } else {
bail!("Unable to generate output asset for chunk"); bail!("Unable to generate output asset for chunk");
}, },
) )
} }
}
#[turbo_tasks::value_impl]
impl BrowserChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn current_chunk_method(&self) -> Vc<CurrentChunkMethod> { pub fn current_chunk_method(&self) -> Vc<CurrentChunkMethod> {
self.current_chunk_method.cell() self.current_chunk_method.cell()
...@@ -699,7 +698,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -699,7 +698,7 @@ impl ChunkingContext for BrowserChunkingContext {
let mut assets = chunks let mut assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
...@@ -767,7 +766,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -767,7 +766,7 @@ impl ChunkingContext for BrowserChunkingContext {
let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
......
...@@ -139,7 +139,7 @@ use crate::{ ...@@ -139,7 +139,7 @@ use crate::{
exports_info::{ExportsInfoBinding, ExportsInfoRef}, exports_info::{ExportsInfoBinding, ExportsInfoRef},
ident::IdentReplacement, ident::IdentReplacement,
member::MemberReplacement, member::MemberReplacement,
node::{FilePathModuleReference, PackageJsonReference}, node::PackageJsonReference,
raw::{DirAssetReference, FileSourceReference}, raw::{DirAssetReference, FileSourceReference},
require_context::{RequireContextAssetReference, RequireContextMap}, require_context::{RequireContextAssetReference, RequireContextMap},
type_issue::SpecifiedModuleTypeIssue, type_issue::SpecifiedModuleTypeIssue,
...@@ -1935,7 +1935,7 @@ where ...@@ -1935,7 +1935,7 @@ where
if *compile_time_info.environment().rendering().await? == Rendering::Client { if *compile_time_info.environment().rendering().await? == Rendering::Client {
analysis.add_reference_code_gen( analysis.add_reference_code_gen(
WorkerAssetReference::new( WorkerAssetReference::new_web_worker(
origin, origin,
Request::parse(pat).to_resolved().await?, Request::parse(pat).to_resolved().await?,
issue_source(source, span), issue_source(source, span),
...@@ -1950,8 +1950,7 @@ where ...@@ -1950,8 +1950,7 @@ where
// Ignore (e.g. dynamic parameter or string literal), just as Webpack does // Ignore (e.g. dynamic parameter or string literal), just as Webpack does
return Ok(()); return Ok(());
} }
WellKnownFunctionKind::NodeWorkerConstructor if analysis.analyze_mode.is_tracing() => { WellKnownFunctionKind::NodeWorkerConstructor => {
// Only for tracing, not for bundling (yet?)
let args = linked_args().await?; let args = linked_args().await?;
if !args.is_empty() { if !args.is_empty() {
let pat = js_value_to_pattern(&args[0]); let pat = js_value_to_pattern(&args[0]);
...@@ -1968,17 +1967,20 @@ where ...@@ -1968,17 +1967,20 @@ where
return Ok(()); return Ok(());
} }
} }
analysis.add_reference(
FilePathModuleReference::new( analysis.add_reference_code_gen(
origin.asset_context(), WorkerAssetReference::new_node_worker_thread(
origin,
// WorkerThreads resolve filepaths relative to the process root
get_traced_project_dir().await?, get_traced_project_dir().await?,
Pattern::new(pat), Pattern::new(pat).to_resolved().await?,
collect_affecting_sources, collect_affecting_sources,
get_issue_source(), get_issue_source(),
) in_try,
.to_resolved() ),
.await?, ast_path.to_vec().into(),
); );
return Ok(()); return Ok(());
} }
let (args, hints) = explain_args(args); let (args, hints) = explain_args(args);
......
use anyhow::Result; use anyhow::Result;
use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, file_source::FileSource, raw_module::RawModule, reference::ModuleReference,
context::AssetContext, resolve::ModuleResolveResult,
file_source::FileSource,
issue::IssueSource,
raw_module::RawModule,
reference::ModuleReference,
reference_type::{ReferenceType, WorkerReferenceSubType},
resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
}; };
use crate::references::util::check_and_emit_too_many_matches_warning;
#[turbo_tasks::value] #[turbo_tasks::value]
#[derive(Hash, Clone, Debug)] #[derive(Hash, Clone, Debug)]
pub struct PackageJsonReference { pub struct PackageJsonReference {
...@@ -55,89 +46,3 @@ impl ValueToString for PackageJsonReference { ...@@ -55,89 +46,3 @@ impl ValueToString for PackageJsonReference {
)) ))
} }
} }
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct FilePathModuleReference {
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
}
#[turbo_tasks::value_impl]
impl FilePathModuleReference {
#[turbo_tasks::function]
pub fn new(
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
) -> Vc<Self> {
Self {
asset_context,
context_dir,
path,
collect_affecting_sources,
issue_source,
}
.cell()
}
}
// A reference to an module by absolute or cwd-relative file path (e.g. for the
// worker-threads `new Worker` which has the resolving behavior of `fs.readFile` but should treat
// the resolve result as an module instead of a raw source).
#[turbo_tasks::value_impl]
impl ModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let span = tracing::info_span!(
"trace module",
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_raw(
self.context_dir.clone(),
*self.path,
self.collect_affecting_sources,
/* force_in_lookup_dir */ false,
);
let result = self.asset_context.process_resolve_result(
result,
ReferenceType::Worker(WorkerReferenceSubType::NodeWorker),
);
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.path,
)
.await?;
Ok(result)
}
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for FilePathModuleReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("raw asset {}", self.path.to_string().await?,).into(),
))
}
}
use anyhow::Result; use anyhow::{Result, bail};
use indoc::formatdoc; use indoc::formatdoc;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc}; use turbo_tasks::{ResolvedVc, TryJoinIterExt, ValueDefault, Vc};
use turbopack_core::{ use turbopack_core::{
chunk::{ chunk::{
ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunkingContextExt, ChunksData, AsyncModuleInfo, ChunkData, ChunkItem, ChunkType, ChunkingContext, ChunkingContextExt,
availability_info::AvailabilityInfo, ChunksData, EvaluatableAsset, EvaluatableAssets, availability_info::AvailabilityInfo,
}, },
ident::AssetIdent, ident::AssetIdent,
module::Module, module::Module,
module_graph::{ModuleGraph, chunk_group_info::ChunkGroup}, module_graph::{ModuleGraph, chunk_group_info::ChunkGroup},
output::{OutputAssetsReference, OutputAssetsWithReferenced}, output::{OutputAsset, OutputAssets, OutputAssetsReference, OutputAssetsWithReferenced},
}; };
use super::module::WorkerLoaderModule; use super::{module::WorkerLoaderModule, worker_type::WorkerType};
use crate::{ use crate::{
chunk::{ chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkType,
...@@ -28,6 +28,7 @@ pub struct WorkerLoaderChunkItem { ...@@ -28,6 +28,7 @@ pub struct WorkerLoaderChunkItem {
pub module: ResolvedVc<WorkerLoaderModule>, pub module: ResolvedVc<WorkerLoaderModule>,
pub module_graph: ResolvedVc<ModuleGraph>, pub module_graph: ResolvedVc<ModuleGraph>,
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
pub worker_type: WorkerType,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -36,12 +37,55 @@ impl WorkerLoaderChunkItem { ...@@ -36,12 +37,55 @@ impl WorkerLoaderChunkItem {
async fn chunk_group(&self) -> Result<Vc<OutputAssetsWithReferenced>> { async fn chunk_group(&self) -> Result<Vc<OutputAssetsWithReferenced>> {
let module = self.module.await?; let module = self.module.await?;
Ok(self.chunking_context.evaluated_chunk_group_assets( Ok(match self.worker_type {
module.inner.ident().with_modifier(rcstr!("worker")), WorkerType::WebWorker => self.chunking_context.evaluated_chunk_group_assets(
ChunkGroup::Isolated(ResolvedVc::upcast(module.inner)), module
*self.module_graph, .inner
AvailabilityInfo::root(), .ident()
)) .with_modifier(self.worker_type.chunk_modifier_str()),
ChunkGroup::Isolated(ResolvedVc::upcast(module.inner)),
*self.module_graph,
AvailabilityInfo::root(),
),
// WorkerThreads are treated as an entry point, webworkers probably should to but
// currently it would lead to a cascade that we need to address.
WorkerType::NodeWorkerThread => {
let Some(evaluatable) =
ResolvedVc::try_sidecast::<Box<dyn EvaluatableAsset>>(module.inner)
else {
bail!("Worker module must be evaluatable");
};
let worker_path = self
.chunking_context
.chunk_path(
None,
module.inner.ident(),
Some(rcstr!("[worker thread]")),
rcstr!(".js"),
)
.owned()
.await?;
let entry_result = self
.chunking_context
.root_entry_chunk_group(
worker_path,
EvaluatableAssets::one(*evaluatable),
*self.module_graph,
OutputAssets::empty(),
OutputAssets::empty(),
)
.await?;
OutputAssetsWithReferenced {
assets: ResolvedVc::cell(vec![entry_result.asset]),
referenced_assets: ResolvedVc::cell(vec![]),
references: ResolvedVc::cell(vec![]),
}
.cell()
}
})
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -57,19 +101,74 @@ impl WorkerLoaderChunkItem { ...@@ -57,19 +101,74 @@ impl WorkerLoaderChunkItem {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl EcmascriptChunkItem for WorkerLoaderChunkItem { impl EcmascriptChunkItem for WorkerLoaderChunkItem {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn content(self: Vc<Self>) -> Result<Vc<EcmascriptChunkItemContent>> { fn content(self: Vc<Self>) -> Vc<EcmascriptChunkItemContent> {
let chunks_data = self.chunks_data().await?; panic!("should not be called");
let chunks_data = chunks_data.iter().try_join().await?; }
let chunks_data: Vec<_> = chunks_data
.iter() #[turbo_tasks::function]
.map(|chunk_data| EcmascriptChunkData::new(chunk_data)) async fn content_with_async_module_info(
.collect(); self: Vc<Self>,
_async_module_info: Option<Vc<AsyncModuleInfo>>,
let code = formatdoc! { estimated: bool,
r#" ) -> Result<Vc<EcmascriptChunkItemContent>> {
{TURBOPACK_EXPORT_VALUE}({TURBOPACK_WORKER_BLOB_URL}({chunks:#})); let this = self.await?;
"#,
chunks = StringifyJs(&chunks_data), let code = match this.worker_type {
WorkerType::WebWorker => {
// For web workers, generate code that creates a blob URL
let chunks_data = self.chunks_data().await?;
let chunks_data = chunks_data.iter().try_join().await?;
let chunks_data: Vec<_> = chunks_data
.iter()
.map(|chunk_data| EcmascriptChunkData::new(chunk_data))
.collect();
formatdoc! {
r#"
{TURBOPACK_EXPORT_VALUE}({TURBOPACK_WORKER_BLOB_URL}({chunks:#}));
"#,
chunks = StringifyJs(&chunks_data),
}
}
WorkerType::NodeWorkerThread => {
// For Node.js workers, export the path to the worker entry chunk
if estimated {
// In estimation mode we cannot call into chunking context APIs otherwise we
// will induce a turbo tasks cycle. But we only need an approximate solution.
formatdoc! {
r#"
{TURBOPACK_EXPORT_VALUE}(__dirname + "/" + {worker_path:#});
"#,
worker_path = StringifyJs(&"a_fake_path_for_size_estimation"),
}
} else {
let chunk_group = self.chunk_group().await?;
let assets = chunk_group.assets.await?;
// The last asset is the evaluate chunk (entry point) for the worker.
// The evaluated_chunk_group adds regular chunks first, then pushes the
// evaluate chunk last. The evaluate chunk contains the bootstrap code that
// loads the runtime and other chunks. For Node.js workers, we need a single
// file path (not a blob URL like browser workers), so we use the evaluate
// chunk which serves as the entry point.
let Some(entry_asset) = assets.last() else {
bail!("cannot find worker entry point asset");
};
let entry_path = entry_asset.path().await?;
// Get the filename of the worker entry chunk
// We use just the filename because both the loader module and the worker
// entry chunk are in the same directory (typically server/chunks/), so we
// don't need a relative path - __dirname will already point to the correct
// directory
formatdoc! {
r#"
{TURBOPACK_EXPORT_VALUE}(__dirname + "/" + {worker_path:#});
"#,
worker_path = StringifyJs(entry_path.file_name()),
}
}
}
}; };
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
...@@ -106,10 +205,8 @@ impl ChunkItem for WorkerLoaderChunkItem { ...@@ -106,10 +205,8 @@ impl ChunkItem for WorkerLoaderChunkItem {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
async fn ty(&self) -> Result<Vc<Box<dyn ChunkType>>> { fn ty(&self) -> Vc<Box<dyn ChunkType>> {
Ok(Vc::upcast( Vc::upcast(EcmascriptChunkType::value_default())
Vc::<EcmascriptChunkType>::default().resolve().await?,
))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
pub mod chunk_item; pub mod chunk_item;
pub mod module; pub mod module;
pub mod worker_type;
pub use worker_type::WorkerType;
use anyhow::Result; use anyhow::Result;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbopack_core::{ use turbopack_core::{
asset::{Asset, AssetContent}, asset::{Asset, AssetContent},
...@@ -14,25 +14,25 @@ use turbopack_core::{ ...@@ -14,25 +14,25 @@ use turbopack_core::{
resolve::ModuleResolveResult, resolve::ModuleResolveResult,
}; };
use super::chunk_item::WorkerLoaderChunkItem; use super::{chunk_item::WorkerLoaderChunkItem, worker_type::WorkerType};
/// The WorkerLoaderModule is a module that creates a separate root chunk group for the given module /// The WorkerLoaderModule is a module that creates a separate root chunk group for the given module
/// and exports a URL to pass to the worker constructor. /// and exports a URL (for web workers) or file path (for Node.js workers) to pass to the worker
/// constructor.
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct WorkerLoaderModule { pub struct WorkerLoaderModule {
pub inner: ResolvedVc<Box<dyn ChunkableModule>>, pub inner: ResolvedVc<Box<dyn ChunkableModule>>,
pub worker_type: WorkerType,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WorkerLoaderModule { impl WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn ChunkableModule>>) -> Vc<Self> { pub fn new(module: ResolvedVc<Box<dyn ChunkableModule>>, worker_type: WorkerType) -> Vc<Self> {
Self::cell(WorkerLoaderModule { inner: module }) Self::cell(WorkerLoaderModule {
} inner: module,
worker_type,
#[turbo_tasks::function] })
pub fn asset_ident_for(module: Vc<Box<dyn ChunkableModule>>) -> Vc<AssetIdent> {
module.ident().with_modifier(rcstr!("worker loader"))
} }
} }
...@@ -40,7 +40,9 @@ impl WorkerLoaderModule { ...@@ -40,7 +40,9 @@ impl WorkerLoaderModule {
impl Module for WorkerLoaderModule { impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> { fn ident(&self) -> Vc<AssetIdent> {
Self::asset_ident_for(*self.inner) self.inner
.ident()
.with_modifier(self.worker_type.modifier_str())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -50,8 +52,9 @@ impl Module for WorkerLoaderModule { ...@@ -50,8 +52,9 @@ impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> { async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
let this = self.await?;
Ok(Vc::cell(vec![ResolvedVc::upcast( Ok(Vc::cell(vec![ResolvedVc::upcast(
WorkerModuleReference::new(*ResolvedVc::upcast(self.await?.inner)) WorkerModuleReference::new(*ResolvedVc::upcast(this.inner), this.worker_type)
.to_resolved() .to_resolved()
.await?, .await?,
)])) )]))
...@@ -67,48 +70,61 @@ impl Module for WorkerLoaderModule { ...@@ -67,48 +70,61 @@ impl Module for WorkerLoaderModule {
impl Asset for WorkerLoaderModule { impl Asset for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
fn content(&self) -> Vc<AssetContent> { fn content(&self) -> Vc<AssetContent> {
panic!("content() should not be called"); // For content(), we delegate to the inner module to support some testing usecases that
// attempt to emit all assets. This follows the same pattern as other transform modules:
// - TracedAsset: delegates to inner module (explicit tracing wrapper)
// - EcmascriptModulePartAsset: delegates to full module (tree-shaking wrapper)
// - CachedExternalModule: returns NotFound (build-time only, no source to trace)
self.inner.content()
} }
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkableModule for WorkerLoaderModule { impl ChunkableModule for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
fn as_chunk_item( async fn as_chunk_item(
self: ResolvedVc<Self>, self: ResolvedVc<Self>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
) -> Vc<Box<dyn turbopack_core::chunk::ChunkItem>> { ) -> Result<Vc<Box<dyn turbopack_core::chunk::ChunkItem>>> {
Vc::upcast( Ok(Vc::upcast(
WorkerLoaderChunkItem { WorkerLoaderChunkItem {
module: self, module: self,
module_graph, module_graph,
chunking_context, chunking_context,
worker_type: self.await?.worker_type,
} }
.cell(), .cell(),
) ))
} }
} }
#[turbo_tasks::value] #[turbo_tasks::value]
struct WorkerModuleReference { struct WorkerModuleReference {
module: ResolvedVc<Box<dyn Module>>, module: ResolvedVc<Box<dyn Module>>,
worker_type: WorkerType,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WorkerModuleReference { impl WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> { pub fn new(module: ResolvedVc<Box<dyn Module>>, worker_type: WorkerType) -> Vc<Self> {
Self::cell(WorkerModuleReference { module }) Self::cell(WorkerModuleReference {
module,
worker_type,
})
} }
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkableModuleReference for WorkerModuleReference { impl ChunkableModuleReference for WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
fn chunking_type(self: Vc<Self>) -> Vc<ChunkingTypeOption> { fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Isolated { Vc::cell(Some(ChunkingType::Isolated {
_ty: ChunkGroupType::Evaluated, _ty: match self.worker_type {
WorkerType::WebWorker => ChunkGroupType::Evaluated,
WorkerType::NodeWorkerThread => ChunkGroupType::Entry,
},
merge_tag: None, merge_tag: None,
})) }))
} }
...@@ -126,6 +142,6 @@ impl ModuleReference for WorkerModuleReference { ...@@ -126,6 +142,6 @@ impl ModuleReference for WorkerModuleReference {
impl ValueToString for WorkerModuleReference { impl ValueToString for WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> { fn to_string(&self) -> Vc<RcStr> {
Vc::cell(rcstr!("worker module")) Vc::cell(self.worker_type.reference_str())
} }
} }
use bincode::{Decode, Encode};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs};
#[derive(
Debug, Clone, Copy, Hash, PartialEq, Eq, Encode, Decode, TraceRawVcs, NonLocalValue, TaskInput,
)]
pub enum WorkerType {
WebWorker,
NodeWorkerThread,
}
impl WorkerType {
pub fn modifier_str(&self) -> RcStr {
match self {
WorkerType::WebWorker => rcstr!("worker loader"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread loader"),
}
}
pub fn chunk_modifier_str(&self) -> RcStr {
match self {
WorkerType::WebWorker => rcstr!("worker"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread"),
}
}
pub fn reference_str(&self) -> RcStr {
match self {
WorkerType::WebWorker => rcstr!("worker module"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread module"),
}
}
}
...@@ -250,25 +250,6 @@ impl NodeJsChunkingContext { ...@@ -250,25 +250,6 @@ impl NodeJsChunkingContext {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl NodeJsChunkingContext { impl NodeJsChunkingContext {
#[turbo_tasks::function]
async fn generate_chunk(
self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<Vc<Box<dyn OutputAsset>>> {
Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{
Vc::upcast(EcmascriptBuildNodeChunk::new(self, *ecmascript_chunk))
} else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{
*output_asset
} else {
bail!("Unable to generate output asset for chunk");
},
)
}
/// Returns the kind of runtime to include in output chunks. /// Returns the kind of runtime to include in output chunks.
/// ///
/// This is defined directly on `NodeJsChunkingContext` so it is zero-cost /// This is defined directly on `NodeJsChunkingContext` so it is zero-cost
...@@ -290,6 +271,30 @@ impl NodeJsChunkingContext { ...@@ -290,6 +271,30 @@ impl NodeJsChunkingContext {
} }
} }
impl NodeJsChunkingContext {
async fn generate_chunk(
self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<ResolvedVc<Box<dyn OutputAsset>>> {
Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{
ResolvedVc::upcast(
EcmascriptBuildNodeChunk::new(self, *ecmascript_chunk)
.to_resolved()
.await?,
)
} else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{
output_asset
} else {
bail!("Unable to generate output asset for chunk");
},
)
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkingContext for NodeJsChunkingContext { impl ChunkingContext for NodeJsChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -476,7 +481,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -476,7 +481,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let assets = chunks let assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
...@@ -531,7 +536,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -531,7 +536,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let extra_chunks = extra_chunks.await?; let extra_chunks = extra_chunks.await?;
let mut other_chunks = chunks let mut other_chunks = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
other_chunks.extend(extra_chunks.iter().copied()); other_chunks.extend(extra_chunks.iter().copied());
......
...@@ -166,7 +166,7 @@ impl EcmascriptBuildNodeEntryChunk { ...@@ -166,7 +166,7 @@ impl EcmascriptBuildNodeEntryChunk {
impl ValueToString for EcmascriptBuildNodeEntryChunk { impl ValueToString for EcmascriptBuildNodeEntryChunk {
#[turbo_tasks::function] #[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> { fn to_string(&self) -> Vc<RcStr> {
Vc::cell(rcstr!("Ecmascript Build Node Evaluate Chunk")) Vc::cell(rcstr!("Ecmascript Build Node Entry Chunk"))
} }
} }
......
...@@ -437,6 +437,8 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V ...@@ -437,6 +437,8 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
ResolveOptionsContext { ResolveOptionsContext {
enable_typescript: true, enable_typescript: true,
enable_node_modules: Some(project_root.clone()), enable_node_modules: Some(project_root.clone()),
enable_node_native_modules: true,
enable_node_externals: true,
custom_conditions: vec![rcstr!("development")], custom_conditions: vec![rcstr!("development")],
rules: vec![( rules: vec![(
ContextCondition::InNodeModules, ContextCondition::InNodeModules,
......
const { Worker, isMainThread, parentPort } = require('node:worker_threads')
const path = require('path')
if (isMainThread) {
it('should run a worker thread with separate file', async () => {
const worker = new Worker(path.join(__dirname, 'worker.js'))
const message = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
expect(message).toBe('pong')
await worker.terminate()
})
it('should handle self-referencing worker (__filename)', async () => {
const worker = new Worker(__filename)
const message = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('self-ref-ping')
})
expect(message).toBe('self-ref-pong')
await worker.terminate()
})
async function runDynamicWorker(workerType, args) {
const worker = new Worker(`./${workerType}-worker.js`)
try {
return await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage(args)
})
} finally {
await worker.terminate()
}
}
it('should handle dynamic worker selection with pattern', async () => {
const response = await runDynamicWorker('math', { a: 5, b: 3 })
expect(response).toEqual({ type: 'math', result: 8 })
})
it('should handle another dynamic worker type', async () => {
const response = await runDynamicWorker('string', { text: 'hello' })
expect(response).toEqual({ type: 'string', result: 'HELLO' })
})
} else {
// Worker thread - handle self-referencing worker messages
parentPort.on('message', (msg) => {
if (msg === 'self-ref-ping') {
parentPort.postMessage('self-ref-pong')
}
})
}
const { parentPort } = require('node:worker_threads')
if (parentPort) {
parentPort.on('message', (data) => {
const result = data.a + data.b
parentPort.postMessage({ type: 'math', result })
})
}
const { parentPort } = require('node:worker_threads')
if (parentPort) {
parentPort.on('message', (data) => {
const result = data.text.toUpperCase()
parentPort.postMessage({ type: 'string', result })
})
}
const { parentPort } = require('node:worker_threads')
// Worker thread code
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort.postMessage('pong')
} else if (msg.type === 'compute') {
const result = msg.a + msg.b
parentPort.postMessage({ type: 'result', value: result })
}
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment