Unverified Commit b690af09 authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Reapply "[turbopack] Add bundling support for worker_threads" (#88725) (#88967)

## What?

This PR reapplies #87746 which adds bundling support for Node.js `worker_threads` in Turbopack.

## Why?

The original PR (#87746) was reverted in #88725 because it broke builds that use packages like `pino` with transports. These packages use dynamic patterns like `join(__dirname, 'lib', 'worker.js')` to resolve worker entry points, which can match non-evaluatable files like `package.json` or `tsconfig.json`.

## How?

This PR reapplies the original changes with the following fixes:

1. **Downgrade errors to warnings in tracing contexts**: When `loose_errors` is enabled (tracing mode) or `in_try` is true, worker entry point validation errors are now emitted as warnings instead of errors. This follows the established Turbopack pattern used in `handle_resolve_error` and other resolve error handling.

2. **Improved error messages**: Error messages for non-chunkable and non-evaluatable worker entry point modules now include:
   - The module identifier (so you can see which file caused the issue)
   - The issue source location (pointing to the `new Worker()` call in source code)

3. **Added regression test**: A new pino-based test case exercises the `thread-stream` worker pattern that caused the original failure. This test runs in `CodeGenerationAndTracing` mode via `test-start-turbo`.

## Changes from original PR

- Added `source` field to `CodeGenerationIssue` to support showing issue source locations
- Added `get_issue_severity()` helper that checks `loose_errors` and `in_try` to determine severity
- Fixed typo: `"SharedbWorker"` → `"SharedWorker"` in `to_string` implementation
- Added pino regression test in `test/e2e/app-dir/node-worker-threads/`

## Testing

- ✅ `pnpm test-dev-turbo test/e2e/app-dir/node-worker-threads/` - all tests pass
- ✅ `pnpm test-start-turbo test/e2e/app-dir/node-worker-threads/` - all tests pass (exercises bundling mode)
- ✅ Verified test fails without the `loose_errors` fix
- ✅ Verified vercel-docs build succeeds with warnings instead of errors
parent bb88ea52
edition = "2024"
max_width = 100 max_width = 100
comment_width = 100 comment_width = 100
......
...@@ -14,10 +14,7 @@ use turbo_tasks::{Completion, ResolvedVc, Vc}; ...@@ -14,10 +14,7 @@ use turbo_tasks::{Completion, ResolvedVc, Vc};
use turbo_tasks_fs::{self, File, FileContent, FileSystemPath}; use turbo_tasks_fs::{self, File, FileContent, FileSystemPath};
use turbopack_core::{ use turbopack_core::{
asset::AssetContent, asset::AssetContent,
chunk::{ chunk::{ChunkingContextExt, EntryChunkGroupResult, availability_info::AvailabilityInfo},
ChunkingContext, ChunkingContextExt, EntryChunkGroupResult,
availability_info::AvailabilityInfo,
},
context::AssetContext, context::AssetContext,
module::Module, module::Module,
module_graph::{ module_graph::{
...@@ -138,7 +135,7 @@ impl MiddlewareEndpoint { ...@@ -138,7 +135,7 @@ impl MiddlewareEndpoint {
}; };
let EntryChunkGroupResult { asset: chunk, .. } = *chunking_context let EntryChunkGroupResult { asset: chunk, .. } = *chunking_context
.entry_chunk_group( .root_entry_chunk_group(
this.project this.project
.node_root() .node_root()
.await? .await?
...@@ -147,7 +144,6 @@ impl MiddlewareEndpoint { ...@@ -147,7 +144,6 @@ impl MiddlewareEndpoint {
module_graph, module_graph,
OutputAssets::empty(), OutputAssets::empty(),
OutputAssets::empty(), OutputAssets::empty(),
AvailabilityInfo::root(),
) )
.await?; .await?;
Ok(*chunk) Ok(*chunk)
......
...@@ -76,9 +76,6 @@ impl OutputAsset for NftJsonAsset { ...@@ -76,9 +76,6 @@ impl OutputAsset for NftJsonAsset {
} }
} }
#[turbo_tasks::value(transparent)]
pub struct OutputSpecifier(Option<RcStr>);
fn get_output_specifier( fn get_output_specifier(
path_ref: &FileSystemPath, path_ref: &FileSystemPath,
ident_folder: &FileSystemPath, ident_folder: &FileSystemPath,
...@@ -97,9 +94,8 @@ fn get_output_specifier( ...@@ -97,9 +94,8 @@ fn get_output_specifier(
.get_relative_path_to(path_ref) .get_relative_path_to(path_ref)
.unwrap()); .unwrap());
} }
// This should effectively be unreachable // This should effectively be unreachable
bail!("NftJsonAsset: cannot handle filepath {path_ref}"); bail!("NftJsonAsset: cannot handle filepath '{path_ref}'");
} }
/// Apply outputFileTracingIncludes patterns to find additional files /// Apply outputFileTracingIncludes patterns to find additional files
...@@ -285,13 +281,23 @@ impl Asset for NftJsonAsset { ...@@ -285,13 +281,23 @@ impl Asset for NftJsonAsset {
} }
} }
let specifier = get_output_specifier( let specifier = match get_output_specifier(
&referenced_chunk_path, &referenced_chunk_path,
&ident_folder, &ident_folder,
&ident_folder_in_project_fs, &ident_folder_in_project_fs,
&output_root_ref, &output_root_ref,
&project_root_ref, &project_root_ref,
)?; ) {
Ok(specifier) => specifier,
Err(err) => {
return Err(err.context(format!(
"NftJsonAsset: cannot handle filepath '{chunk_path}' for \
{referenced_chunk:?} it is not under the output_root: \
'{output_root_ref}' or the project_root: '{project_root_ref}'",
chunk_path = referenced_chunk_path.value_to_string().await?
)));
}
};
result.insert(specifier); result.insert(specifier);
} }
......
import { NextResponse } from 'next/server'
import pino from 'pino'
export async function GET() {
try {
// Create a pino logger with a transport
// This internally uses thread-stream which creates worker_threads
const logger = pino({
transport: {
target: 'pino/file',
options: { destination: 1 }, // stdout
},
})
// Log a test message - this triggers the transport worker
logger.info('pino test message')
// Flush the logger to ensure the message is sent
// Use nextTick to allow async initialization of the transport
await new Promise((resolve) => process.nextTick(resolve))
return NextResponse.json({
success: true,
message: 'pino logger with transport initialized successfully',
})
} catch (error) {
return NextResponse.json(
{ success: false, error: String(error) },
{ status: 500 }
)
}
}
import { Worker } from 'node:worker_threads'
import { NextResponse } from 'next/server'
export async function GET() {
try {
// Test with a relative path instead of __filename
const worker = new Worker('./app/worker-dir/simple-worker.ts')
const message = await new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker timeout'))
}, 5000)
worker.on('message', (msg) => {
clearTimeout(timeout)
resolve(msg)
})
worker.on('error', (err) => {
clearTimeout(timeout)
reject(err)
})
worker.on('exit', (code) => {
if (code !== 0) {
clearTimeout(timeout)
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
await worker.terminate()
return NextResponse.json({ success: true, message })
} catch (error) {
return NextResponse.json(
{ success: false, error: String(error) },
{ status: 500 }
)
}
}
import { Worker, isMainThread, parentPort } from 'node:worker_threads'
import { NextResponse } from 'next/server'
// This file tests self-referencing worker threads using __filename
// This pattern is used by libraries like @duckdb/duckdb-wasm
if (!isMainThread && parentPort) {
// Worker thread - handle messages
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort!.postMessage('pong')
}
})
}
export async function GET() {
if (!isMainThread) {
// If we're in a worker, don't try to create another worker
return NextResponse.json({ error: 'Already in worker' }, { status: 500 })
}
// Log __filename for debugging
console.log('__filename:', __filename)
console.log('typeof __filename:', typeof __filename)
try {
// Create a worker using __filename - this is the pattern that triggers the cycle bug
const worker = new Worker(__filename)
const message = await new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker timeout'))
}, 5000)
worker.on('message', (msg) => {
clearTimeout(timeout)
resolve(msg)
})
worker.on('error', (err) => {
clearTimeout(timeout)
reject(err)
})
worker.on('exit', (code) => {
if (code !== 0) {
clearTimeout(timeout)
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
await worker.terminate()
return NextResponse.json({ success: true, message })
} catch (error) {
return NextResponse.json(
{ success: false, error: String(error) },
{ status: 500 }
)
}
}
export default function RootLayout({
children,
}: {
children: React.ReactNode
}) {
return (
<html>
<body>{children}</body>
</html>
)
}
import { parentPort } from 'node:worker_threads'
if (parentPort) {
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort!.postMessage('pong from simple worker')
}
})
}
import { nextTestSetup } from 'e2e-utils'
describe('node-worker-threads', () => {
const { next, skipped, isTurbopack } = nextTestSetup({
files: __dirname,
skipDeployment: true,
dependencies: {
pino: '9.6.0',
},
})
if (skipped) {
return
}
// These tests are Turbopack-specific since they rely on Turbopack's worker bundling
if (!isTurbopack) {
it.skip('webpack doesnt support bundling worker-threads', () => {})
return
}
it('should handle simple worker with relative path', async () => {
const res = await next.fetch('/api/simple-worker-test')
const data = await res.json()
console.log('Simple worker response:', data)
expect(res.status).toBe(200)
expect(data.success).toBe(true)
expect(data.message).toBe('pong from simple worker')
})
it('should handle self-referencing worker with __filename', async () => {
const res = await next.fetch('/api/worker-test')
const data = await res.json()
console.log('Self-ref worker response:', data)
expect(res.status).toBe(200)
expect(data.success).toBe(true)
expect(data.message).toBe('pong')
})
it('should handle pino logger with transport (thread-stream)', async () => {
// Pino with transports uses thread-stream internally, which creates worker_threads
// with a broad pattern like join(__dirname, 'lib', 'worker.js') that can match
// non-evaluatable files like package.json. This tests that we properly downgrade
// those errors to warnings via loose_errors.
const res = await next.fetch('/api/pino-test')
const data = await res.json()
console.log('Pino test response:', data)
expect(res.status).toBe(200)
expect(data.success).toBe(true)
})
})
...@@ -372,10 +372,7 @@ impl BrowserChunkingContext { ...@@ -372,10 +372,7 @@ impl BrowserChunkingContext {
} }
} }
} }
#[turbo_tasks::value_impl]
impl BrowserChunkingContext { impl BrowserChunkingContext {
#[turbo_tasks::function]
fn generate_evaluate_chunk( fn generate_evaluate_chunk(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
...@@ -392,8 +389,6 @@ impl BrowserChunkingContext { ...@@ -392,8 +389,6 @@ impl BrowserChunkingContext {
module_graph, module_graph,
)) ))
} }
#[turbo_tasks::function]
fn generate_chunk_list_register_chunk( fn generate_chunk_list_register_chunk(
self: Vc<Self>, self: Vc<Self>,
ident: Vc<AssetIdent>, ident: Vc<AssetIdent>,
...@@ -409,26 +404,30 @@ impl BrowserChunkingContext { ...@@ -409,26 +404,30 @@ impl BrowserChunkingContext {
source, source,
)) ))
} }
#[turbo_tasks::function]
async fn generate_chunk( async fn generate_chunk(
self: Vc<Self>, self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>, chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<Vc<Box<dyn OutputAsset>>> { ) -> Result<ResolvedVc<Box<dyn OutputAsset>>> {
Ok( Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk) if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{ {
Vc::upcast(EcmascriptBrowserChunk::new(self, *ecmascript_chunk)) ResolvedVc::upcast(
EcmascriptBrowserChunk::new(self, *ecmascript_chunk)
.to_resolved()
.await?,
)
} else if let Some(output_asset) = } else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk) ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{ {
*output_asset output_asset
} else { } else {
bail!("Unable to generate output asset for chunk"); bail!("Unable to generate output asset for chunk");
}, },
) )
} }
}
#[turbo_tasks::value_impl]
impl BrowserChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn current_chunk_method(&self) -> Vc<CurrentChunkMethod> { pub fn current_chunk_method(&self) -> Vc<CurrentChunkMethod> {
self.current_chunk_method.cell() self.current_chunk_method.cell()
...@@ -724,7 +723,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -724,7 +723,7 @@ impl ChunkingContext for BrowserChunkingContext {
let mut assets = chunks let mut assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
...@@ -787,7 +786,7 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -787,7 +786,7 @@ impl ChunkingContext for BrowserChunkingContext {
let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks let mut assets: Vec<ResolvedVc<Box<dyn OutputAsset>>> = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
......
...@@ -286,7 +286,7 @@ async fn build_internal( ...@@ -286,7 +286,7 @@ async fn build_internal(
let ty = ReferenceType::Entry(EntryReferenceSubType::Undefined); let ty = ReferenceType::Entry(EntryReferenceSubType::Undefined);
let request = request_vc.await?; let request = request_vc.await?;
origin origin
.resolve_asset(request_vc, origin.resolve_options(ty.clone()), ty) .resolve_asset(request_vc, origin.resolve_options(), ty)
.await? .await?
.first_module() .first_module()
.await? .await?
......
...@@ -139,7 +139,7 @@ pub async fn create_web_entry_source( ...@@ -139,7 +139,7 @@ pub async fn create_web_entry_source(
.map(|request| async move { .map(|request| async move {
let ty = ReferenceType::Entry(EntryReferenceSubType::Web); let ty = ReferenceType::Entry(EntryReferenceSubType::Web);
Ok(origin Ok(origin
.resolve_asset(request, origin.resolve_options(ty.clone()), ty) .resolve_asset(request, origin.resolve_options(), ty)
.await? .await?
.resolve() .resolve()
.await? .await?
......
...@@ -69,11 +69,7 @@ pub trait AssetContext { ...@@ -69,11 +69,7 @@ pub trait AssetContext {
/// Gets the resolve options for a given path. /// Gets the resolve options for a given path.
#[turbo_tasks::function] #[turbo_tasks::function]
fn resolve_options( fn resolve_options(self: Vc<Self>, origin_path: FileSystemPath) -> Vc<ResolveOptions>;
self: Vc<Self>,
origin_path: FileSystemPath,
reference_type: ReferenceType,
) -> Vc<ResolveOptions>;
/// Resolves an request to an [ModuleResolveResult]. /// Resolves an request to an [ModuleResolveResult].
#[turbo_tasks::function] #[turbo_tasks::function]
......
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use super::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString}; use super::{
Issue, IssueSeverity, IssueSource, IssueStage, OptionIssueSource, OptionStyledString,
StyledString,
};
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct CodeGenerationIssue { pub struct CodeGenerationIssue {
...@@ -9,6 +12,8 @@ pub struct CodeGenerationIssue { ...@@ -9,6 +12,8 @@ pub struct CodeGenerationIssue {
pub path: FileSystemPath, pub path: FileSystemPath,
pub title: ResolvedVc<StyledString>, pub title: ResolvedVc<StyledString>,
pub message: ResolvedVc<StyledString>, pub message: ResolvedVc<StyledString>,
/// Optional source location that points to where the issue originates
pub source: Option<IssueSource>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -36,4 +41,9 @@ impl Issue for CodeGenerationIssue { ...@@ -36,4 +41,9 @@ impl Issue for CodeGenerationIssue {
fn description(&self) -> Vc<OptionStyledString> { fn description(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(self.message)) Vc::cell(Some(self.message))
} }
#[turbo_tasks::function]
fn source(&self) -> Vc<OptionIssueSource> {
Vc::cell(self.source)
}
} }
...@@ -1654,7 +1654,7 @@ pub async fn url_resolve( ...@@ -1654,7 +1654,7 @@ pub async fn url_resolve(
issue_source: Option<IssueSource>, issue_source: Option<IssueSource>,
is_optional: bool, is_optional: bool,
) -> Result<Vc<ModuleResolveResult>> { ) -> Result<Vc<ModuleResolveResult>> {
let resolve_options = origin.resolve_options(reference_type.clone()); let resolve_options = origin.resolve_options();
let rel_request = request.as_relative(); let rel_request = request.as_relative();
let origin_path_parent = origin.origin_path().await?.parent(); let origin_path_parent = origin.origin_path().await?.parent();
let rel_result = resolve( let rel_result = resolve(
......
...@@ -34,13 +34,10 @@ pub trait ResolveOrigin { ...@@ -34,13 +34,10 @@ pub trait ResolveOrigin {
/// Get the resolve options that apply for this origin. /// Get the resolve options that apply for this origin.
#[turbo_tasks::function] #[turbo_tasks::function]
async fn resolve_options( async fn resolve_options(self: Vc<Self>) -> Result<Vc<ResolveOptions>> {
self: Vc<Self>,
reference_type: ReferenceType,
) -> Result<Vc<ResolveOptions>> {
Ok(self Ok(self
.asset_context() .asset_context()
.resolve_options(self.origin_path().owned().await?, reference_type)) .resolve_options(self.origin_path().owned().await?))
} }
} }
......
...@@ -284,6 +284,7 @@ async fn module_factory_with_code_generation_issue( ...@@ -284,6 +284,7 @@ async fn module_factory_with_code_generation_issue(
title: StyledString::Text(rcstr!("Code generation for chunk item errored")) title: StyledString::Text(rcstr!("Code generation for chunk item errored"))
.resolved_cell(), .resolved_cell(),
message: StyledString::Text(error_message).resolved_cell(), message: StyledString::Text(error_message).resolved_cell(),
source: None,
} }
.resolved_cell() .resolved_cell()
.emit(); .emit();
......
...@@ -197,14 +197,15 @@ pub enum AnalyzeMode { ...@@ -197,14 +197,15 @@ pub enum AnalyzeMode {
/// For bundling only, no tracing of referenced files. /// For bundling only, no tracing of referenced files.
#[default] #[default]
CodeGeneration, CodeGeneration,
/// For bundling and tracing of referenced files. /// For bundling and finding references to external referenced files
CodeGenerationAndTracing, CodeGenerationAndTracing,
/// For tracing of referenced files only, no bundling (i.e. no codegen). /// For tracing transitive external references (i.e. no codegen).
Tracing, Tracing,
} }
impl AnalyzeMode { impl AnalyzeMode {
pub fn is_tracing(self) -> bool { /// Are we currently collecting references to external assets. e.g. filesystem dependencies
pub fn is_tracing_assets(self) -> bool {
match self { match self {
AnalyzeMode::Tracing | AnalyzeMode::CodeGenerationAndTracing => true, AnalyzeMode::Tracing | AnalyzeMode::CodeGenerationAndTracing => true,
AnalyzeMode::CodeGeneration => false, AnalyzeMode::CodeGeneration => false,
......
...@@ -311,7 +311,7 @@ impl Module for CachedExternalModule { ...@@ -311,7 +311,7 @@ impl Module for CachedExternalModule {
origin origin
.resolve_asset( .resolve_asset(
Request::parse_string(self.request.clone()), Request::parse_string(self.request.clone()),
origin.resolve_options(ReferenceType::Undefined), origin.resolve_options(),
ReferenceType::Undefined, ReferenceType::Undefined,
) )
.await? .await?
......
use anyhow::Result; use anyhow::Result;
use tracing::Instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbo_tasks::{ResolvedVc, ValueToString, Vc};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, file_source::FileSource, raw_module::RawModule, reference::ModuleReference,
context::AssetContext, resolve::ModuleResolveResult,
file_source::FileSource,
issue::IssueSource,
raw_module::RawModule,
reference::ModuleReference,
reference_type::{ReferenceType, WorkerReferenceSubType},
resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
}; };
use crate::references::util::check_and_emit_too_many_matches_warning;
#[turbo_tasks::value] #[turbo_tasks::value]
#[derive(Hash, Clone, Debug)] #[derive(Hash, Clone, Debug)]
pub struct PackageJsonReference { pub struct PackageJsonReference {
...@@ -55,89 +46,3 @@ impl ValueToString for PackageJsonReference { ...@@ -55,89 +46,3 @@ impl ValueToString for PackageJsonReference {
)) ))
} }
} }
#[turbo_tasks::value]
#[derive(Hash, Debug)]
pub struct FilePathModuleReference {
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
}
#[turbo_tasks::value_impl]
impl FilePathModuleReference {
#[turbo_tasks::function]
pub fn new(
asset_context: ResolvedVc<Box<dyn AssetContext>>,
context_dir: FileSystemPath,
path: ResolvedVc<Pattern>,
collect_affecting_sources: bool,
issue_source: IssueSource,
) -> Vc<Self> {
Self {
asset_context,
context_dir,
path,
collect_affecting_sources,
issue_source,
}
.cell()
}
}
// A reference to an module by absolute or cwd-relative file path (e.g. for the
// worker-threads `new Worker` which has the resolving behavior of `fs.readFile` but should treat
// the resolve result as an module instead of a raw source).
#[turbo_tasks::value_impl]
impl ModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let span = tracing::info_span!(
"trace module",
pattern = display(self.path.to_string().await?)
);
async {
let result = resolve_raw(
self.context_dir.clone(),
*self.path,
self.collect_affecting_sources,
/* force_in_lookup_dir */ false,
);
let result = self.asset_context.process_resolve_result(
result,
ReferenceType::Worker(WorkerReferenceSubType::NodeWorker),
);
check_and_emit_too_many_matches_warning(
result,
self.issue_source,
self.context_dir.clone(),
self.path,
)
.await?;
Ok(result)
}
.instrument(span)
.await
}
}
#[turbo_tasks::value_impl]
impl ChunkableModuleReference for FilePathModuleReference {
#[turbo_tasks::function]
fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Traced))
}
}
#[turbo_tasks::value_impl]
impl ValueToString for FilePathModuleReference {
#[turbo_tasks::function]
async fn to_string(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
format!("raw asset {}", self.path.to_string().await?,).into(),
))
}
}
...@@ -345,6 +345,7 @@ async fn to_single_pattern_mapping( ...@@ -345,6 +345,7 @@ async fn to_single_pattern_mapping(
) )
.resolved_cell(), .resolved_cell(),
path: origin.origin_path().owned().await?, path: origin.origin_path().owned().await?,
source: None,
} }
.resolved_cell() .resolved_cell()
.emit(); .emit();
...@@ -376,6 +377,7 @@ async fn to_single_pattern_mapping( ...@@ -376,6 +377,7 @@ async fn to_single_pattern_mapping(
)) ))
.resolved_cell(), .resolved_cell(),
path: origin.origin_path().owned().await?, path: origin.origin_path().owned().await?,
source: None,
} }
.resolved_cell() .resolved_cell()
.emit(); .emit();
......
...@@ -9,7 +9,7 @@ use turbopack_core::{ ...@@ -9,7 +9,7 @@ use turbopack_core::{
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
raw_module::RawModule, raw_module::RawModule,
reference::{ModuleReference, ModuleReferences}, reference::{ModuleReference, ModuleReferences},
reference_type::{CommonJsReferenceSubType, ReferenceType}, reference_type::CommonJsReferenceSubType,
resolve::{ModuleResolveResult, origin::ResolveOrigin, parse::Request}, resolve::{ModuleResolveResult, origin::ResolveOrigin, parse::Request},
source::Source, source::Source,
}; };
...@@ -55,10 +55,7 @@ impl Module for TsConfigModuleAsset { ...@@ -55,10 +55,7 @@ impl Module for TsConfigModuleAsset {
let configs = read_tsconfigs( let configs = read_tsconfigs(
self.source.content().file_content(), self.source.content().file_content(),
self.source, self.source,
apply_cjs_specific_options( apply_cjs_specific_options(self.origin.resolve_options()),
self.origin
.resolve_options(ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined)),
),
) )
.await?; .await?;
references.extend( references.extend(
......
...@@ -159,8 +159,7 @@ pub struct WebpackRuntimeAssetReference { ...@@ -159,8 +159,7 @@ pub struct WebpackRuntimeAssetReference {
impl ModuleReference for WebpackRuntimeAssetReference { impl ModuleReference for WebpackRuntimeAssetReference {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> { async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
let ty = ReferenceType::CommonJs(CommonJsReferenceSubType::Undefined); let options = self.origin.resolve_options();
let options = self.origin.resolve_options(ty.clone());
let options = apply_cjs_specific_options(options); let options = apply_cjs_specific_options(options);
......
pub mod chunk_item; pub mod chunk_item;
pub mod module; pub mod module;
pub mod worker_type;
pub use worker_type::WorkerType;
...@@ -11,18 +11,18 @@ use turbopack_core::{ ...@@ -11,18 +11,18 @@ use turbopack_core::{
module::{Module, ModuleSideEffects}, module::{Module, ModuleSideEffects},
module_graph::ModuleGraph, module_graph::ModuleGraph,
reference::{ModuleReference, ModuleReferences}, reference::{ModuleReference, ModuleReferences},
reference_type::WorkerReferenceSubType,
resolve::ModuleResolveResult, resolve::ModuleResolveResult,
}; };
use super::chunk_item::WorkerLoaderChunkItem; use super::{chunk_item::WorkerLoaderChunkItem, worker_type::WorkerType};
/// The WorkerLoaderModule is a module that creates a separate root chunk group for the given module /// The WorkerLoaderModule is a module that creates a separate root chunk group for the given module
/// and exports a URL to pass to the worker constructor. /// and exports a URL (for web workers) or file path (for Node.js workers) to pass to the worker
/// constructor.
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct WorkerLoaderModule { pub struct WorkerLoaderModule {
pub inner: ResolvedVc<Box<dyn ChunkableModule>>, pub inner: ResolvedVc<Box<dyn ChunkableModule>>,
pub worker_type: WorkerReferenceSubType, pub worker_type: WorkerType,
pub asset_context: ResolvedVc<Box<dyn AssetContext>>, pub asset_context: ResolvedVc<Box<dyn AssetContext>>,
} }
...@@ -31,7 +31,7 @@ impl WorkerLoaderModule { ...@@ -31,7 +31,7 @@ impl WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new( pub fn new(
module: ResolvedVc<Box<dyn ChunkableModule>>, module: ResolvedVc<Box<dyn ChunkableModule>>,
worker_type: WorkerReferenceSubType, worker_type: WorkerType,
asset_context: ResolvedVc<Box<dyn AssetContext>>, asset_context: ResolvedVc<Box<dyn AssetContext>>,
) -> Vc<Self> { ) -> Vc<Self> {
Self::cell(WorkerLoaderModule { Self::cell(WorkerLoaderModule {
...@@ -40,18 +40,15 @@ impl WorkerLoaderModule { ...@@ -40,18 +40,15 @@ impl WorkerLoaderModule {
asset_context, asset_context,
}) })
} }
#[turbo_tasks::function]
pub fn asset_ident_for(module: Vc<Box<dyn ChunkableModule>>) -> Vc<AssetIdent> {
module.ident().with_modifier(rcstr!("worker loader"))
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Module for WorkerLoaderModule { impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> { fn ident(&self) -> Vc<AssetIdent> {
Self::asset_ident_for(*self.inner) self.inner
.ident()
.with_modifier(self.worker_type.modifier_str())
} }
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -61,8 +58,9 @@ impl Module for WorkerLoaderModule { ...@@ -61,8 +58,9 @@ impl Module for WorkerLoaderModule {
#[turbo_tasks::function] #[turbo_tasks::function]
async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> { async fn references(self: Vc<Self>) -> Result<Vc<ModuleReferences>> {
let this = self.await?;
Ok(Vc::cell(vec![ResolvedVc::upcast( Ok(Vc::cell(vec![ResolvedVc::upcast(
WorkerModuleReference::new(*ResolvedVc::upcast(self.await?.inner)) WorkerModuleReference::new(*ResolvedVc::upcast(this.inner), this.worker_type)
.to_resolved() .to_resolved()
.await?, .await?,
)])) )]))
...@@ -99,22 +97,29 @@ impl ChunkableModule for WorkerLoaderModule { ...@@ -99,22 +97,29 @@ impl ChunkableModule for WorkerLoaderModule {
#[turbo_tasks::value] #[turbo_tasks::value]
struct WorkerModuleReference { struct WorkerModuleReference {
module: ResolvedVc<Box<dyn Module>>, module: ResolvedVc<Box<dyn Module>>,
worker_type: WorkerType,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WorkerModuleReference { impl WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(module: ResolvedVc<Box<dyn Module>>) -> Vc<Self> { pub fn new(module: ResolvedVc<Box<dyn Module>>, worker_type: WorkerType) -> Vc<Self> {
Self::cell(WorkerModuleReference { module }) Self::cell(WorkerModuleReference {
module,
worker_type,
})
} }
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkableModuleReference for WorkerModuleReference { impl ChunkableModuleReference for WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
fn chunking_type(self: Vc<Self>) -> Vc<ChunkingTypeOption> { fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
Vc::cell(Some(ChunkingType::Isolated { Vc::cell(Some(ChunkingType::Isolated {
_ty: ChunkGroupType::Evaluated, _ty: match self.worker_type {
WorkerType::SharedWebWorker | WorkerType::WebWorker => ChunkGroupType::Evaluated,
WorkerType::NodeWorkerThread => ChunkGroupType::Entry,
},
merge_tag: None, merge_tag: None,
})) }))
} }
...@@ -132,6 +137,10 @@ impl ModuleReference for WorkerModuleReference { ...@@ -132,6 +137,10 @@ impl ModuleReference for WorkerModuleReference {
impl ValueToString for WorkerModuleReference { impl ValueToString for WorkerModuleReference {
#[turbo_tasks::function] #[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> { fn to_string(&self) -> Vc<RcStr> {
Vc::cell(rcstr!("worker module")) Vc::cell(match self.worker_type {
WorkerType::WebWorker => rcstr!("web worker module"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread module"),
WorkerType::SharedWebWorker => rcstr!("shared web worker module"),
})
} }
} }
use bincode::{Decode, Encode};
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs};
use turbopack_core::reference_type::{ReferenceType, WorkerReferenceSubType};
#[derive(
Debug, Clone, Copy, Hash, PartialEq, Eq, Encode, Decode, TraceRawVcs, NonLocalValue, TaskInput,
)]
pub enum WorkerType {
WebWorker,
SharedWebWorker,
NodeWorkerThread,
}
impl WorkerType {
pub fn modifier_str(&self) -> RcStr {
match self {
WorkerType::SharedWebWorker | WorkerType::WebWorker => rcstr!("worker loader"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread loader"),
}
}
pub fn chunk_modifier_str(&self) -> RcStr {
match self {
WorkerType::SharedWebWorker | WorkerType::WebWorker => rcstr!("worker"),
WorkerType::NodeWorkerThread => rcstr!("node worker thread"),
}
}
pub fn reference_type(&self) -> ReferenceType {
ReferenceType::Worker(match self {
WorkerType::WebWorker => WorkerReferenceSubType::WebWorker,
WorkerType::SharedWebWorker => WorkerReferenceSubType::SharedWorker,
WorkerType::NodeWorkerThread => WorkerReferenceSubType::NodeWorker,
})
}
}
...@@ -266,25 +266,6 @@ impl NodeJsChunkingContext { ...@@ -266,25 +266,6 @@ impl NodeJsChunkingContext {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl NodeJsChunkingContext { impl NodeJsChunkingContext {
#[turbo_tasks::function]
async fn generate_chunk(
self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<Vc<Box<dyn OutputAsset>>> {
Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{
Vc::upcast(EcmascriptBuildNodeChunk::new(self, *ecmascript_chunk))
} else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{
*output_asset
} else {
bail!("Unable to generate output asset for chunk");
},
)
}
/// Returns the kind of runtime to include in output chunks. /// Returns the kind of runtime to include in output chunks.
/// ///
/// This is defined directly on `NodeJsChunkingContext` so it is zero-cost /// This is defined directly on `NodeJsChunkingContext` so it is zero-cost
...@@ -306,6 +287,30 @@ impl NodeJsChunkingContext { ...@@ -306,6 +287,30 @@ impl NodeJsChunkingContext {
} }
} }
impl NodeJsChunkingContext {
async fn generate_chunk(
self: Vc<Self>,
chunk: ResolvedVc<Box<dyn Chunk>>,
) -> Result<ResolvedVc<Box<dyn OutputAsset>>> {
Ok(
if let Some(ecmascript_chunk) = ResolvedVc::try_downcast_type::<EcmascriptChunk>(chunk)
{
ResolvedVc::upcast(
EcmascriptBuildNodeChunk::new(self, *ecmascript_chunk)
.to_resolved()
.await?,
)
} else if let Some(output_asset) =
ResolvedVc::try_sidecast::<Box<dyn OutputAsset>>(chunk)
{
output_asset
} else {
bail!("Unable to generate output asset for chunk");
},
)
}
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl ChunkingContext for NodeJsChunkingContext { impl ChunkingContext for NodeJsChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -499,7 +504,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -499,7 +504,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let assets = chunks let assets = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
...@@ -551,7 +556,7 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -551,7 +556,7 @@ impl ChunkingContext for NodeJsChunkingContext {
let extra_chunks = extra_chunks.await?; let extra_chunks = extra_chunks.await?;
let mut other_chunks = chunks let mut other_chunks = chunks
.iter() .iter()
.map(|chunk| self.generate_chunk(**chunk).to_resolved()) .map(|chunk| self.generate_chunk(*chunk))
.try_join() .try_join()
.await?; .await?;
other_chunks.extend(extra_chunks.iter().copied()); other_chunks.extend(extra_chunks.iter().copied());
......
...@@ -166,7 +166,7 @@ impl EcmascriptBuildNodeEntryChunk { ...@@ -166,7 +166,7 @@ impl EcmascriptBuildNodeEntryChunk {
impl ValueToString for EcmascriptBuildNodeEntryChunk { impl ValueToString for EcmascriptBuildNodeEntryChunk {
#[turbo_tasks::function] #[turbo_tasks::function]
fn to_string(&self) -> Vc<RcStr> { fn to_string(&self) -> Vc<RcStr> {
Vc::cell(rcstr!("Ecmascript Build Node Evaluate Chunk")) Vc::cell(rcstr!("Ecmascript Build Node Entry Chunk"))
} }
} }
......
...@@ -94,7 +94,7 @@ pub async fn esm_resolve( ...@@ -94,7 +94,7 @@ pub async fn esm_resolve(
issue_source: Option<IssueSource>, issue_source: Option<IssueSource>,
) -> Result<Vc<ModuleResolveResult>> { ) -> Result<Vc<ModuleResolveResult>> {
let ty = ReferenceType::EcmaScriptModules(ty); let ty = ReferenceType::EcmaScriptModules(ty);
let options = apply_esm_specific_options(origin.resolve_options(ty.clone()), &ty) let options = apply_esm_specific_options(origin.resolve_options(), &ty)
.resolve() .resolve()
.await?; .await?;
specific_resolve(origin, request, options, ty, is_optional, issue_source).await specific_resolve(origin, request, options, ty, is_optional, issue_source).await
...@@ -109,7 +109,7 @@ pub async fn cjs_resolve( ...@@ -109,7 +109,7 @@ pub async fn cjs_resolve(
is_optional: bool, is_optional: bool,
) -> Result<Vc<ModuleResolveResult>> { ) -> Result<Vc<ModuleResolveResult>> {
let ty = ReferenceType::CommonJs(ty); let ty = ReferenceType::CommonJs(ty);
let options = apply_cjs_specific_options(origin.resolve_options(ty.clone())) let options = apply_cjs_specific_options(origin.resolve_options())
.resolve() .resolve()
.await?; .await?;
specific_resolve(origin, request, options, ty, is_optional, issue_source).await specific_resolve(origin, request, options, ty, is_optional, issue_source).await
...@@ -124,7 +124,7 @@ pub async fn cjs_resolve_source( ...@@ -124,7 +124,7 @@ pub async fn cjs_resolve_source(
is_optional: bool, is_optional: bool,
) -> Result<Vc<ResolveResult>> { ) -> Result<Vc<ResolveResult>> {
let ty = ReferenceType::CommonJs(ty); let ty = ReferenceType::CommonJs(ty);
let options = apply_cjs_specific_options(origin.resolve_options(ty.clone())) let options = apply_cjs_specific_options(origin.resolve_options())
.resolve() .resolve()
.await?; .await?;
let result = resolve( let result = resolve(
......
...@@ -417,7 +417,7 @@ pub async fn type_resolve( ...@@ -417,7 +417,7 @@ pub async fn type_resolve(
) -> Result<Vc<ModuleResolveResult>> { ) -> Result<Vc<ModuleResolveResult>> {
let ty = ReferenceType::TypeScript(TypeScriptReferenceSubType::Undefined); let ty = ReferenceType::TypeScript(TypeScriptReferenceSubType::Undefined);
let context_path = origin.origin_path().await?.parent(); let context_path = origin.origin_path().await?.parent();
let options = origin.resolve_options(ty.clone()); let options = origin.resolve_options();
let options = apply_typescript_types_options(options); let options = apply_typescript_types_options(options);
let types_request = if let Request::Module { let types_request = if let Request::Module {
module: m, module: m,
......
...@@ -32,7 +32,6 @@ impl AssetContext for NoopAssetContext { ...@@ -32,7 +32,6 @@ impl AssetContext for NoopAssetContext {
async fn resolve_options( async fn resolve_options(
self: Vc<Self>, self: Vc<Self>,
_origin_path: FileSystemPath, _origin_path: FileSystemPath,
_reference_type: ReferenceType,
) -> Result<Vc<ResolveOptions>> { ) -> Result<Vc<ResolveOptions>> {
Ok(ResolveOptions::default().cell()) Ok(ResolveOptions::default().cell())
} }
......
...@@ -437,6 +437,8 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V ...@@ -437,6 +437,8 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
ResolveOptionsContext { ResolveOptionsContext {
enable_typescript: true, enable_typescript: true,
enable_node_modules: Some(project_root.clone()), enable_node_modules: Some(project_root.clone()),
enable_node_native_modules: true,
enable_node_externals: true,
custom_conditions: vec![rcstr!("development")], custom_conditions: vec![rcstr!("development")],
rules: vec![( rules: vec![(
ContextCondition::InNodeModules, ContextCondition::InNodeModules,
......
const { Worker, isMainThread, parentPort } = require('node:worker_threads')
const path = require('path')
if (isMainThread) {
it('should run a worker thread with separate file', async () => {
const worker = new Worker(path.join(__dirname, 'worker.js'))
const message = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('ping')
})
expect(message).toBe('pong')
await worker.terminate()
})
it('should handle self-referencing worker (__filename)', async () => {
const worker = new Worker(__filename)
const message = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage('self-ref-ping')
})
expect(message).toBe('self-ref-pong')
await worker.terminate()
})
async function runDynamicWorker(workerType, args) {
const worker = new Worker(`./${workerType}-worker.js`)
try {
return await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
worker.postMessage(args)
})
} finally {
await worker.terminate()
}
}
it('should handle dynamic worker selection with pattern', async () => {
const response = await runDynamicWorker('math', { a: 5, b: 3 })
expect(response).toEqual({ type: 'math', result: 8 })
})
it('should handle another dynamic worker type', async () => {
const response = await runDynamicWorker('string', { text: 'hello' })
expect(response).toEqual({ type: 'string', result: 'HELLO' })
})
} else {
// Worker thread - handle self-referencing worker messages
parentPort.on('message', (msg) => {
if (msg === 'self-ref-ping') {
parentPort.postMessage('self-ref-pong')
}
})
}
const { parentPort } = require('node:worker_threads')
if (parentPort) {
parentPort.on('message', (data) => {
const result = data.a + data.b
parentPort.postMessage({ type: 'math', result })
})
}
const { parentPort } = require('node:worker_threads')
if (parentPort) {
parentPort.on('message', (data) => {
const result = data.text.toUpperCase()
parentPort.postMessage({ type: 'string', result })
})
}
const { parentPort } = require('node:worker_threads')
// Worker thread code
parentPort.on('message', (msg) => {
if (msg === 'ping') {
parentPort.postMessage('pong')
} else if (msg.type === 'compute') {
const result = msg.a + msg.b
parentPort.postMessage({ type: 'result', value: result })
}
})
...@@ -797,7 +797,6 @@ impl AssetContext for ModuleAssetContext { ...@@ -797,7 +797,6 @@ impl AssetContext for ModuleAssetContext {
async fn resolve_options( async fn resolve_options(
self: Vc<Self>, self: Vc<Self>,
origin_path: FileSystemPath, origin_path: FileSystemPath,
_reference_type: ReferenceType,
) -> Result<Vc<ResolveOptions>> { ) -> Result<Vc<ResolveOptions>> {
let this = self.await?; let this = self.await?;
let module_asset_context = if let Some(transition) = this.transition { let module_asset_context = if let Some(transition) = this.transition {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment