Unverified Commit c5b2cf1e authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Allow turbopack-node backend to be swapped at runtime using an…

Turbopack: Allow turbopack-node backend to be swapped at runtime using an experimental config option (#90671)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->

Follow-up to https://github.com/vercel/next.js/pull/86266  
This was mostly LLM-generated, but it's just moving things around and threading through a config option.  
  
We'd like to be able to turn this on/off with an experimental config option:  
\- Fixes issues where RA thinks a bunch of code is dead.  
\- Lets us isolate potential issues to the worker thread implementation.  
\- Provides an escape hatch for users in case there are issues.  
\- Better fits the cargo "additive feature" model.
parent 28df39ba
...@@ -9,6 +9,11 @@ autobenches = false ...@@ -9,6 +9,11 @@ autobenches = false
[lib] [lib]
bench = false bench = false
[features]
default = ["process_pool"]
process_pool = ["next-core/process_pool", "turbopack-node/process_pool"]
worker_pool = ["next-core/worker_pool", "turbopack-node/worker_pool"]
[lints] [lints]
workspace = true workspace = true
......
...@@ -14,7 +14,9 @@ use next_core::{ ...@@ -14,7 +14,9 @@ use next_core::{
next_client::{ next_client::{
ClientChunkingContextOptions, get_client_chunking_context, get_client_compile_time_info, ClientChunkingContextOptions, get_client_chunking_context, get_client_compile_time_info,
}, },
next_config::{ModuleIds as ModuleIdStrategyConfig, NextConfig}, next_config::{
ModuleIds as ModuleIdStrategyConfig, NextConfig, TurbopackPluginRuntimeStrategy,
},
next_edge::context::EdgeChunkingContextOptions, next_edge::context::EdgeChunkingContextOptions,
next_server::{ next_server::{
ServerChunkingContextOptions, ServerContextType, get_server_chunking_context, ServerChunkingContextOptions, ServerContextType, get_server_chunking_context,
...@@ -79,7 +81,11 @@ use turbopack_core::{ ...@@ -79,7 +81,11 @@ use turbopack_core::{
NotFoundVersion, OptionVersionedContent, Update, Version, VersionState, VersionedContent, NotFoundVersion, OptionVersionedContent, Update, Version, VersionState, VersionedContent,
}, },
}; };
#[cfg(feature = "process_pool")]
use turbopack_node::child_process_backend;
use turbopack_node::execution_context::ExecutionContext; use turbopack_node::execution_context::ExecutionContext;
#[cfg(feature = "worker_pool")]
use turbopack_node::worker_threads_backend;
use turbopack_nodejs::NodeJsChunkingContext; use turbopack_nodejs::NodeJsChunkingContext;
use crate::{ use crate::{
...@@ -1217,6 +1223,16 @@ impl Project { ...@@ -1217,6 +1223,16 @@ impl Project {
pub(super) async fn execution_context(self: Vc<Self>) -> Result<Vc<ExecutionContext>> { pub(super) async fn execution_context(self: Vc<Self>) -> Result<Vc<ExecutionContext>> {
let node_root = self.node_root().owned().await?; let node_root = self.node_root().owned().await?;
let next_mode = self.next_mode().await?; let next_mode = self.next_mode().await?;
let strategy = *self
.next_config()
.turbopack_plugin_runtime_strategy()
.await?;
let node_backend = match strategy {
#[cfg(feature = "worker_pool")]
TurbopackPluginRuntimeStrategy::WorkerThreads => worker_threads_backend(),
#[cfg(feature = "process_pool")]
TurbopackPluginRuntimeStrategy::ChildProcesses => child_process_backend(),
};
let node_execution_chunking_context = Vc::upcast( let node_execution_chunking_context = Vc::upcast(
NodeJsChunkingContext::builder( NodeJsChunkingContext::builder(
...@@ -1237,6 +1253,7 @@ impl Project { ...@@ -1237,6 +1253,7 @@ impl Project {
self.project_path().owned().await?, self.project_path().owned().await?,
node_execution_chunking_context, node_execution_chunking_context,
self.env(), self.env(),
node_backend,
)) ))
} }
...@@ -1475,10 +1492,12 @@ impl Project { ...@@ -1475,10 +1492,12 @@ impl Project {
// At this point all modules have been computed and we can get rid of the node.js // At this point all modules have been computed and we can get rid of the node.js
// process pools // process pools
let execution_context = self.execution_context().await?;
let node_backend = execution_context.node_backend.into_trait_ref().await?;
if *self.is_watch_enabled().await? { if *self.is_watch_enabled().await? {
turbopack_node::evaluate::scale_down(); node_backend.scale_down()?;
} else { } else {
turbopack_node::evaluate::scale_zero(); node_backend.scale_zero()?;
} }
Ok(module_graphs_vc) Ok(module_graphs_vc)
......
...@@ -87,6 +87,9 @@ turbopack-static = { workspace = true } ...@@ -87,6 +87,9 @@ turbopack-static = { workspace = true }
turbopack-trace-utils = { workspace = true } turbopack-trace-utils = { workspace = true }
[features] [features]
default = ["process_pool"]
process_pool = ["turbopack-node/process_pool"]
worker_pool = ["turbopack-node/worker_pool"]
next-font-local = [] next-font-local = []
plugin = [ plugin = [
"swc_core/plugin_transform_host_native", "swc_core/plugin_transform_host_native",
......
...@@ -920,6 +920,16 @@ pub enum ModuleIds { ...@@ -920,6 +920,16 @@ pub enum ModuleIds {
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct OptionModuleIds(pub Option<ModuleIds>); pub struct OptionModuleIds(pub Option<ModuleIds>);
#[turbo_tasks::value(operation)]
#[derive(Copy, Clone, Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum TurbopackPluginRuntimeStrategy {
#[cfg(feature = "worker_pool")]
WorkerThreads,
#[cfg(feature = "process_pool")]
ChildProcesses,
}
#[derive( #[derive(
Clone, Debug, PartialEq, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Encode, Decode, Clone, Debug, PartialEq, Deserialize, TraceRawVcs, NonLocalValue, OperationValue, Encode, Decode,
)] )]
...@@ -1150,6 +1160,7 @@ pub struct ExperimentalConfig { ...@@ -1150,6 +1160,7 @@ pub struct ExperimentalConfig {
turbopack_minify: Option<bool>, turbopack_minify: Option<bool>,
turbopack_module_ids: Option<ModuleIds>, turbopack_module_ids: Option<ModuleIds>,
turbopack_plugin_runtime_strategy: Option<TurbopackPluginRuntimeStrategy>,
turbopack_source_maps: Option<bool>, turbopack_source_maps: Option<bool>,
turbopack_input_source_maps: Option<bool>, turbopack_input_source_maps: Option<bool>,
turbopack_tree_shaking: Option<bool>, turbopack_tree_shaking: Option<bool>,
...@@ -2046,6 +2057,19 @@ impl NextConfig { ...@@ -2046,6 +2057,19 @@ impl NextConfig {
) )
} }
#[turbo_tasks::function]
pub fn turbopack_plugin_runtime_strategy(&self) -> Vc<TurbopackPluginRuntimeStrategy> {
#[cfg(feature = "worker_pool")]
let default = TurbopackPluginRuntimeStrategy::WorkerThreads;
#[cfg(all(not(feature = "worker_pool"), feature = "process_pool"))]
let default = TurbopackPluginRuntimeStrategy::ChildProcesses;
self.experimental
.turbopack_plugin_runtime_strategy
.unwrap_or(default)
.cell()
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn module_ids(&self, mode: Vc<NextMode>) -> Result<Vc<ModuleIds>> { pub async fn module_ids(&self, mode: Vc<NextMode>) -> Result<Vc<ModuleIds>> {
Ok(match *mode.await? { Ok(match *mode.await? {
......
...@@ -741,6 +741,7 @@ async fn get_mock_stylesheet( ...@@ -741,6 +741,7 @@ async fn get_mock_stylesheet(
env, env,
project_path: _, project_path: _,
chunking_context, chunking_context,
node_backend,
} = *execution_context.await?; } = *execution_context.await?;
let asset_context = node_evaluate_asset_context( let asset_context = node_evaluate_asset_context(
execution_context, execution_context,
...@@ -770,7 +771,7 @@ async fn get_mock_stylesheet( ...@@ -770,7 +771,7 @@ async fn get_mock_stylesheet(
) )
.module(); .module();
let entries = get_evaluate_entries(mocked_response_asset, asset_context, None); let entries = get_evaluate_entries(mocked_response_asset, asset_context, *node_backend, None);
let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries(
entries.graph_entries().to_resolved().await?, entries.graph_entries().to_resolved().await?,
false, false,
...@@ -783,6 +784,7 @@ async fn get_mock_stylesheet( ...@@ -783,6 +784,7 @@ async fn get_mock_stylesheet(
entries, entries,
root, root,
*env, *env,
*node_backend,
loader_source, loader_source,
*chunking_context, *chunking_context,
module_graph, module_graph,
......
...@@ -110,7 +110,7 @@ turbo-tasks = { workspace = true } ...@@ -110,7 +110,7 @@ turbo-tasks = { workspace = true }
turbo-tasks-backend = { workspace = true } turbo-tasks-backend = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-unix-path = { workspace = true } turbo-unix-path = { workspace = true }
next-api = { workspace = true } next-api = { workspace = true, features = ["worker_pool"] }
next-build = { workspace = true } next-build = { workspace = true }
next-core = { workspace = true } next-core = { workspace = true }
......
...@@ -308,6 +308,7 @@ export default defineConfig([ ...@@ -308,6 +308,7 @@ export default defineConfig([
'itCI', 'itCI',
'itHeaded', 'itHeaded',
'itTurbopackDev', 'itTurbopackDev',
'itOnlyTurbopack',
], ],
}, },
], ],
......
...@@ -324,6 +324,9 @@ export const experimentalSchema = { ...@@ -324,6 +324,9 @@ export const experimentalSchema = {
webpackBuildWorker: z.boolean().optional(), webpackBuildWorker: z.boolean().optional(),
webpackMemoryOptimizations: z.boolean().optional(), webpackMemoryOptimizations: z.boolean().optional(),
turbopackMemoryLimit: z.number().optional(), turbopackMemoryLimit: z.number().optional(),
turbopackPluginRuntimeStrategy: z
.enum(['workerThreads', 'childProcesses'])
.optional(),
turbopackMinify: z.boolean().optional(), turbopackMinify: z.boolean().optional(),
turbopackFileSystemCacheForDev: z.boolean().optional(), turbopackFileSystemCacheForDev: z.boolean().optional(),
turbopackFileSystemCacheForBuild: z.boolean().optional(), turbopackFileSystemCacheForBuild: z.boolean().optional(),
......
...@@ -497,6 +497,11 @@ export interface ExperimentalConfig { ...@@ -497,6 +497,11 @@ export interface ExperimentalConfig {
*/ */
turbopackMemoryLimit?: number turbopackMemoryLimit?: number
/**
* Selects the runtime backend used by Turbopack for Node.js evaluation.
*/
turbopackPluginRuntimeStrategy?: 'workerThreads' | 'childProcesses'
/** /**
* Enable minification. Defaults to true in build mode and false in dev mode. * Enable minification. Defaults to true in build mode and false in dev mode.
*/ */
...@@ -1717,6 +1722,7 @@ export const defaultConfig = Object.freeze({ ...@@ -1717,6 +1722,7 @@ export const defaultConfig = Object.freeze({
turbopackFileSystemCacheForDev: true, turbopackFileSystemCacheForDev: true,
turbopackFileSystemCacheForBuild: false, turbopackFileSystemCacheForBuild: false,
turbopackInferModuleSideEffects: true, turbopackInferModuleSideEffects: true,
turbopackPluginRuntimeStrategy: 'workerThreads',
devCacheControlNoCache: false, devCacheControlNoCache: false,
}, },
htmlLimitedBots: undefined, htmlLimitedBots: undefined,
......
const runtimeStrategy =
process.env.TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY || 'workerThreads'
module.exports = {
experimental: {
turbopackPluginRuntimeStrategy: runtimeStrategy,
},
compiler: {
defineServer: {
'process.env.__TEST_BUILD_PID': String(process.pid),
},
},
turbopack: {
rules: {
'*.pid': {
as: '*.js',
loaders: ['./pid-loader.js'],
},
},
},
}
const loaderData = require('../../input.pid')
export default function handler(_req, res) {
res.status(200).json({
loaderPid: String(loaderData.loaderPid),
buildPid: String(process.env.__TEST_BUILD_PID),
})
}
export default function Page() {
return 'ok'
}
module.exports = function loader() {
return `module.exports = ${JSON.stringify({ loaderPid: String(process.pid) })}`
}
import { nextTestSetup } from 'e2e-utils'
describe.each([
['workerThreads', true],
['childProcesses', false],
] as const)(
'turbopack-node-backend (%s)',
(turbopackPluginRuntimeStrategy, expectSamePid) => {
const { next, isTurbopack } = nextTestSetup({
files: __dirname,
env: {
TEST_TURBOPACK_PLUGIN_RUNTIME_STRATEGY: turbopackPluginRuntimeStrategy,
},
})
const itOnlyTurbopack = isTurbopack ? it : it.skip
itOnlyTurbopack('should match expected loader pid behavior', async () => {
const response = await next.fetch('/api/pid')
expect(response.status).toBe(200)
const data = await response.json()
expect(data.buildPid).toBeDefined()
expect(data.loaderPid).toBeDefined()
if (expectSamePid) {
expect(data.loaderPid).toBe(data.buildPid)
} else {
expect(data.loaderPid).not.toBe(data.buildPid)
}
})
}
)
...@@ -44,7 +44,7 @@ use turbopack_css::chunk::CssChunkType; ...@@ -44,7 +44,7 @@ use turbopack_css::chunk::CssChunkType;
use turbopack_ecmascript::chunk::EcmascriptChunkType; use turbopack_ecmascript::chunk::EcmascriptChunkType;
use turbopack_ecmascript_runtime::RuntimeType; use turbopack_ecmascript_runtime::RuntimeType;
use turbopack_env::dotenv::load_env; use turbopack_env::dotenv::load_env;
use turbopack_node::execution_context::ExecutionContext; use turbopack_node::{child_process_backend, execution_context::ExecutionContext};
use turbopack_nodejs::NodeJsChunkingContext; use turbopack_nodejs::NodeJsChunkingContext;
use crate::{ use crate::{
...@@ -225,6 +225,7 @@ async fn build_internal( ...@@ -225,6 +225,7 @@ async fn build_internal(
}; };
let compile_time_info = get_client_compile_time_info(browserslist_query.clone(), node_env); let compile_time_info = get_client_compile_time_info(browserslist_query.clone(), node_env);
let node_backend = child_process_backend();
let execution_context = ExecutionContext::new( let execution_context = ExecutionContext::new(
root_path.clone(), root_path.clone(),
Vc::upcast( Vc::upcast(
...@@ -245,6 +246,7 @@ async fn build_internal( ...@@ -245,6 +246,7 @@ async fn build_internal(
.build(), .build(),
), ),
load_env(root_path.clone()), load_env(root_path.clone()),
node_backend,
); );
let asset_context = get_client_asset_context( let asset_context = get_client_asset_context(
......
...@@ -40,7 +40,7 @@ use turbopack_dev_server::{ ...@@ -40,7 +40,7 @@ use turbopack_dev_server::{
}; };
use turbopack_ecmascript_runtime::RuntimeType; use turbopack_ecmascript_runtime::RuntimeType;
use turbopack_env::dotenv::load_env; use turbopack_env::dotenv::load_env;
use turbopack_node::execution_context::ExecutionContext; use turbopack_node::{child_process_backend, execution_context::ExecutionContext};
use turbopack_nodejs::NodeJsChunkingContext; use turbopack_nodejs::NodeJsChunkingContext;
use self::web_entry_source::create_web_entry_source; use self::web_entry_source::create_web_entry_source;
...@@ -298,8 +298,13 @@ async fn source( ...@@ -298,8 +298,13 @@ async fn source(
) )
.build(); .build();
let execution_context = let node_backend = child_process_backend();
ExecutionContext::new(root_path.clone(), Vc::upcast(build_chunking_context), env); let execution_context = ExecutionContext::new(
root_path.clone(),
Vc::upcast(build_chunking_context),
env,
node_backend,
);
let server_fs = Vc::upcast::<Box<dyn FileSystem>>(ServerFileSystem::new()); let server_fs = Vc::upcast::<Box<dyn FileSystem>>(ServerFileSystem::new());
let server_root = server_fs.root().owned().await?; let server_root = server_fs.root().owned().await?;
......
use std::{future::Future, path::PathBuf, pin::Pin};
use anyhow::Result;
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
use turbo_tasks::ResolvedVc;
use turbo_tasks_fs::FileSystemPath;
use crate::{AssetsForSourceMapping, evaluate::EvaluatePool};
pub struct CreatePoolOptions {
pub cwd: PathBuf,
pub entrypoint: PathBuf,
pub env: FxHashMap<RcStr, RcStr>,
pub assets_for_source_mapping: ResolvedVc<AssetsForSourceMapping>,
pub assets_root: FileSystemPath,
pub project_dir: FileSystemPath,
pub concurrency: usize,
pub debug: bool,
}
pub type CreatePoolFuture = Pin<Box<dyn Future<Output = Result<EvaluatePool>> + Send + 'static>>;
mod sealed {
#[turbo_tasks::value_trait]
pub(crate) trait Sealed {}
}
#[cfg(feature = "worker_pool")]
#[turbo_tasks::value_impl]
impl sealed::Sealed for crate::worker_pool::WorkerThreadsBackend {}
#[cfg(feature = "process_pool")]
#[turbo_tasks::value_impl]
impl sealed::Sealed for crate::process_pool::ChildProcessesBackend {}
#[turbo_tasks::value_trait]
pub trait NodeBackend: sealed::Sealed {
fn runtime_module_path(&self) -> RcStr;
fn globals_module_path(&self) -> RcStr;
fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture;
fn scale_down(&self) -> Result<()>;
fn scale_zero(&self) -> Result<()>;
}
...@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; ...@@ -10,8 +10,8 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::Value as JsonValue; use serde_json::Value as JsonValue;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ReadRef, Completion, Effects, FxIndexMap, IntoTraitRef, NonLocalValue, OperationVc, PrettyPrintError,
ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects, ReadRef, ResolvedVc, TaskInput, TryJoinIterExt, Vc, duration_span, fxindexmap, get_effects,
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
...@@ -35,13 +35,14 @@ use turbopack_core::{ ...@@ -35,13 +35,14 @@ use turbopack_core::{
virtual_source::VirtualSource, virtual_source::VirtualSource,
}; };
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
use crate::process_pool::ChildProcessPool;
#[cfg(feature = "worker_pool")]
use crate::worker_pool::WorkerThreadPool;
use crate::{ use crate::{
AssetsForSourceMapping, embed_js::embed_file_path, emit, emit_package_json, AssetsForSourceMapping,
format::FormattingMode, internal_assets_for_source_mapping, pool_stats::PoolStatsSnapshot, backend::{CreatePoolOptions, NodeBackend},
embed_js::embed_file_path,
emit, emit_package_json,
format::FormattingMode,
internal_assets_for_source_mapping,
pool_stats::PoolStatsSnapshot,
source_map::StructuredError, source_map::StructuredError,
}; };
...@@ -98,7 +99,6 @@ impl EvaluatePool { ...@@ -98,7 +99,6 @@ impl EvaluatePool {
self.pool.stats() self.pool.stats()
} }
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
pub fn pre_warm(&self) { pub fn pre_warm(&self) {
self.pool.pre_warm() self.pool.pre_warm()
} }
...@@ -108,7 +108,11 @@ impl EvaluatePool { ...@@ -108,7 +108,11 @@ impl EvaluatePool {
pub trait EvaluateOperation: Send + Sync { pub trait EvaluateOperation: Send + Sync {
async fn operation(&self) -> Result<Box<dyn Operation>>; async fn operation(&self) -> Result<Box<dyn Operation>>;
fn stats(&self) -> PoolStatsSnapshot; fn stats(&self) -> PoolStatsSnapshot;
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] /// Eagerly spawn a Node.js worker so it's ready when the first [`Self::operation`] is called.
/// The worker should go into the idle queue.
///
/// If a worker request comes in while this is still initializing, it should wait on the bootup
/// semaphore and will resume when the worker is ready.
fn pre_warm(&self); fn pre_warm(&self);
} }
...@@ -209,6 +213,7 @@ pub async fn get_evaluate_pool( ...@@ -209,6 +213,7 @@ pub async fn get_evaluate_pool(
entries: ResolvedVc<EvaluateEntries>, entries: ResolvedVc<EvaluateEntries>,
cwd: FileSystemPath, cwd: FileSystemPath,
env: ResolvedVc<Box<dyn ProcessEnv>>, env: ResolvedVc<Box<dyn ProcessEnv>>,
node_backend: ResolvedVc<Box<dyn NodeBackend>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
additional_invalidation: ResolvedVc<Completion>, additional_invalidation: ResolvedVc<Completion>,
...@@ -253,31 +258,19 @@ pub async fn get_evaluate_pool( ...@@ -253,31 +258,19 @@ pub async fn get_evaluate_pool(
} }
}; };
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))] let node_backend = node_backend.into_trait_ref().await?;
#[allow(unused_variables)] let pool = node_backend
let pool = ChildProcessPool::create( .create_pool(CreatePoolOptions {
cwd.clone(), cwd,
entrypoint.clone(), entrypoint,
env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), env: env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
assets_for_source_mapping, assets_for_source_mapping,
output_root.clone(), assets_root: output_root.clone(),
chunking_context.root_path().owned().await?, project_dir: chunking_context.root_path().owned().await?,
available_parallelism().map_or(1, |v| v.get()), concurrency: available_parallelism().map_or(1, |v| v.get()),
debug, debug,
); })
#[cfg(feature = "worker_pool")] .await?;
let pool = WorkerThreadPool::create(
cwd,
entrypoint,
env.iter().map(|(k, v)| (k.clone(), v.clone())).collect(),
assets_for_source_mapping,
output_root.clone(),
chunking_context.root_path().owned().await?,
available_parallelism().map_or(1, |v| v.get()),
debug,
)
.await;
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
pool.pre_warm(); pool.pre_warm();
additional_invalidation.await?; additional_invalidation.await?;
Ok(pool.cell()) Ok(pool.cell())
...@@ -430,13 +423,11 @@ impl EvaluateEntries { ...@@ -430,13 +423,11 @@ impl EvaluateEntries {
pub async fn get_evaluate_entries( pub async fn get_evaluate_entries(
module_asset: ResolvedVc<Box<dyn Module>>, module_asset: ResolvedVc<Box<dyn Module>>,
asset_context: ResolvedVc<Box<dyn AssetContext>>, asset_context: ResolvedVc<Box<dyn AssetContext>>,
node_backend: ResolvedVc<Box<dyn NodeBackend>>,
runtime_entries: Option<ResolvedVc<EvaluatableAssets>>, runtime_entries: Option<ResolvedVc<EvaluatableAssets>>,
) -> Result<Vc<EvaluateEntries>> { ) -> Result<Vc<EvaluateEntries>> {
let runtime_module_path = if cfg!(all(feature = "process_pool", not(feature = "worker_pool"))) { let node_backend = node_backend.into_trait_ref().await?;
rcstr!("child_process/evaluate.ts") let runtime_module_path = node_backend.runtime_module_path();
} else {
rcstr!("worker_thread/evaluate.ts")
};
let runtime_asset = asset_context let runtime_asset = asset_context
.process( .process(
...@@ -471,12 +462,7 @@ pub async fn get_evaluate_entries( ...@@ -471,12 +462,7 @@ pub async fn get_evaluate_entries(
let runtime_entries = { let runtime_entries = {
let mut entries = vec![]; let mut entries = vec![];
let global_module_path = let global_module_path = node_backend.globals_module_path();
if cfg!(all(feature = "process_pool", not(feature = "worker_pool"))) {
rcstr!("child_process/globals.ts")
} else {
rcstr!("worker_thread/globals.ts")
};
let globals_module = asset_context let globals_module = asset_context
.process( .process(
...@@ -521,6 +507,7 @@ pub async fn evaluate( ...@@ -521,6 +507,7 @@ pub async fn evaluate(
entries: ResolvedVc<EvaluateEntries>, entries: ResolvedVc<EvaluateEntries>,
cwd: FileSystemPath, cwd: FileSystemPath,
env: ResolvedVc<Box<dyn ProcessEnv>>, env: ResolvedVc<Box<dyn ProcessEnv>>,
node_backend: ResolvedVc<Box<dyn NodeBackend>>,
context_source_for_issue: ResolvedVc<Box<dyn Source>>, context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
...@@ -532,6 +519,7 @@ pub async fn evaluate( ...@@ -532,6 +519,7 @@ pub async fn evaluate(
entries, entries,
cwd, cwd,
env, env,
node_backend,
context_source_for_issue, context_source_for_issue,
chunking_context, chunking_context,
module_graph, module_graph,
...@@ -606,6 +594,7 @@ struct BasicEvaluateContext { ...@@ -606,6 +594,7 @@ struct BasicEvaluateContext {
entries: ResolvedVc<EvaluateEntries>, entries: ResolvedVc<EvaluateEntries>,
cwd: FileSystemPath, cwd: FileSystemPath,
env: ResolvedVc<Box<dyn ProcessEnv>>, env: ResolvedVc<Box<dyn ProcessEnv>>,
node_backend: ResolvedVc<Box<dyn NodeBackend>>,
context_source_for_issue: ResolvedVc<Box<dyn Source>>, context_source_for_issue: ResolvedVc<Box<dyn Source>>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
module_graph: ResolvedVc<ModuleGraph>, module_graph: ResolvedVc<ModuleGraph>,
...@@ -625,6 +614,7 @@ impl EvaluateContext for BasicEvaluateContext { ...@@ -625,6 +614,7 @@ impl EvaluateContext for BasicEvaluateContext {
self.entries, self.entries,
self.cwd.clone(), self.cwd.clone(),
self.env, self.env,
self.node_backend,
self.chunking_context, self.chunking_context,
self.module_graph, self.module_graph,
self.additional_invalidation, self.additional_invalidation,
...@@ -731,25 +721,3 @@ impl Issue for EvaluationIssue { ...@@ -731,25 +721,3 @@ impl Issue for EvaluationIssue {
Vc::cell(Some(self.source)) Vc::cell(Some(self.source))
} }
} }
pub fn scale_down() {
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
{
ChildProcessPool::scale_down();
}
#[cfg(feature = "worker_pool")]
{
WorkerThreadPool::scale_down();
}
}
pub fn scale_zero() {
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
{
ChildProcessPool::scale_zero();
}
#[cfg(feature = "worker_pool")]
{
WorkerThreadPool::scale_zero();
}
}
...@@ -3,11 +3,14 @@ use turbo_tasks_env::ProcessEnv; ...@@ -3,11 +3,14 @@ use turbo_tasks_env::ProcessEnv;
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::chunk::ChunkingContext; use turbopack_core::chunk::ChunkingContext;
use crate::backend::NodeBackend;
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct ExecutionContext { pub struct ExecutionContext {
pub project_path: FileSystemPath, pub project_path: FileSystemPath,
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
pub env: ResolvedVc<Box<dyn ProcessEnv>>, pub env: ResolvedVc<Box<dyn ProcessEnv>>,
pub node_backend: ResolvedVc<Box<dyn NodeBackend>>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -17,11 +20,13 @@ impl ExecutionContext { ...@@ -17,11 +20,13 @@ impl ExecutionContext {
project_path: FileSystemPath, project_path: FileSystemPath,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
env: ResolvedVc<Box<dyn ProcessEnv>>, env: ResolvedVc<Box<dyn ProcessEnv>>,
node_backend: ResolvedVc<Box<dyn NodeBackend>>,
) -> Vc<Self> { ) -> Vc<Self> {
ExecutionContext { ExecutionContext {
project_path, project_path,
chunking_context, chunking_context,
env, env,
node_backend,
} }
.cell() .cell()
} }
...@@ -40,4 +45,9 @@ impl ExecutionContext { ...@@ -40,4 +45,9 @@ impl ExecutionContext {
pub fn env(&self) -> Vc<Box<dyn ProcessEnv>> { pub fn env(&self) -> Vc<Box<dyn ProcessEnv>> {
*self.env *self.env
} }
#[turbo_tasks::function]
pub fn node_backend(&self) -> Vc<Box<dyn NodeBackend>> {
*self.node_backend
}
} }
...@@ -13,20 +13,30 @@ use turbopack_core::{ ...@@ -13,20 +13,30 @@ use turbopack_core::{
virtual_output::VirtualOutputAsset, virtual_output::VirtualOutputAsset,
}; };
mod backend;
pub mod debug; pub mod debug;
pub mod embed_js; pub mod embed_js;
pub mod evaluate; pub mod evaluate;
pub mod execution_context; pub mod execution_context;
mod format; mod format;
mod pool_stats; mod pool_stats;
// When both features are enabled, worker_pool takes priority #[cfg(feature = "process_pool")]
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
pub mod process_pool; pub mod process_pool;
pub mod source_map; pub mod source_map;
pub mod transforms; pub mod transforms;
#[cfg(feature = "worker_pool")] #[cfg(feature = "worker_pool")]
pub mod worker_pool; pub mod worker_pool;
pub use backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend};
#[cfg(feature = "process_pool")]
pub fn child_process_backend() -> Vc<Box<dyn NodeBackend>> {
Vc::upcast(process_pool::ChildProcessesBackend.cell())
}
#[cfg(feature = "worker_pool")]
pub fn worker_threads_backend() -> Vc<Box<dyn NodeBackend>> {
Vc::upcast(worker_pool::WorkerThreadsBackend.cell())
}
#[turbo_tasks::function] #[turbo_tasks::function]
async fn emit( async fn emit(
intermediate_asset: Vc<Box<dyn OutputAsset>>, intermediate_asset: Vc<Box<dyn OutputAsset>>,
......
...@@ -24,13 +24,14 @@ use tokio::{ ...@@ -24,13 +24,14 @@ use tokio::{
sync::Semaphore, sync::Semaphore,
time::{sleep, timeout}, time::{sleep, timeout},
}; };
use turbo_rcstr::RcStr; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span}; use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack_ecmascript::magic_identifier::unmangle_identifiers; use turbopack_ecmascript::magic_identifier::unmangle_identifiers;
use crate::{ use crate::{
AssetsForSourceMapping, AssetsForSourceMapping,
backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend},
evaluate::{EvaluateOperation, EvaluatePool, Operation}, evaluate::{EvaluateOperation, EvaluatePool, Operation},
format::FormattingMode, format::FormattingMode,
pool_stats::{AcquiredPermits, NodeJsPoolStats, PoolStatsSnapshot}, pool_stats::{AcquiredPermits, NodeJsPoolStats, PoolStatsSnapshot},
...@@ -597,6 +598,56 @@ impl ChildProcessPool { ...@@ -597,6 +598,56 @@ impl ChildProcessPool {
} }
} }
#[turbo_tasks::value(shared)]
pub(crate) struct ChildProcessesBackend;
#[turbo_tasks::value_impl]
impl NodeBackend for ChildProcessesBackend {
fn runtime_module_path(&self) -> RcStr {
rcstr!("child_process/evaluate.ts")
}
fn globals_module_path(&self) -> RcStr {
rcstr!("child_process/globals.ts")
}
fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture {
Box::pin(async move {
let CreatePoolOptions {
cwd,
entrypoint,
env,
assets_for_source_mapping,
assets_root,
project_dir,
concurrency,
debug,
} = options;
Ok(ChildProcessPool::create(
cwd,
entrypoint,
env,
assets_for_source_mapping,
assets_root,
project_dir,
concurrency,
debug,
))
})
}
fn scale_down(&self) -> Result<()> {
ChildProcessPool::scale_down();
Ok(())
}
fn scale_zero(&self) -> Result<()> {
ChildProcessPool::scale_zero();
Ok(())
}
}
#[async_trait::async_trait] #[async_trait::async_trait]
impl EvaluateOperation for ChildProcessPool { impl EvaluateOperation for ChildProcessPool {
async fn operation(&self) -> Result<Box<dyn Operation>> { async fn operation(&self) -> Result<Box<dyn Operation>> {
......
...@@ -461,6 +461,7 @@ impl PostCssTransformedAsset { ...@@ -461,6 +461,7 @@ impl PostCssTransformedAsset {
project_path, project_path,
chunking_context, chunking_context,
env, env,
node_backend,
} = &*self.execution_context.await?; } = &*self.execution_context.await?;
// For this postcss transform, there is no guarantee that looking up for the // For this postcss transform, there is no guarantee that looking up for the
...@@ -508,9 +509,10 @@ impl PostCssTransformedAsset { ...@@ -508,9 +509,10 @@ impl PostCssTransformedAsset {
let postcss_executor = let postcss_executor =
postcss_executor(*evaluate_context, project_path.clone(), config_path).module(); postcss_executor(*evaluate_context, project_path.clone(), config_path).module();
let entries = get_evaluate_entries(postcss_executor, *evaluate_context, None) let entries =
.to_resolved() get_evaluate_entries(postcss_executor, *evaluate_context, **node_backend, None)
.await?; .to_resolved()
.await?;
let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries(
entries.graph_entries().to_resolved().await?, entries.graph_entries().to_resolved().await?,
...@@ -537,6 +539,7 @@ impl PostCssTransformedAsset { ...@@ -537,6 +539,7 @@ impl PostCssTransformedAsset {
entries, entries,
cwd: project_path.clone(), cwd: project_path.clone(),
env: *env, env: *env,
node_backend: *node_backend,
context_source_for_issue: self.source, context_source_for_issue: self.source,
chunking_context: *chunking_context, chunking_context: *chunking_context,
module_graph, module_graph,
......
...@@ -51,6 +51,7 @@ use turbopack_resolve::{ ...@@ -51,6 +51,7 @@ use turbopack_resolve::{
use crate::{ use crate::{
AssetsForSourceMapping, AssetsForSourceMapping,
backend::NodeBackend,
debug::should_debug, debug::should_debug,
embed_js::embed_file_path, embed_js::embed_file_path,
evaluate::{ evaluate::{
...@@ -209,6 +210,7 @@ impl WebpackLoadersProcessedAsset { ...@@ -209,6 +210,7 @@ impl WebpackLoadersProcessedAsset {
project_path, project_path,
chunking_context, chunking_context,
env, env,
node_backend,
} = &*transform.execution_context.await?; } = &*transform.execution_context.await?;
let source_content = self.source.content(); let source_content = self.source.content();
let AssetContent::File(file) = *source_content.await? else { let AssetContent::File(file) = *source_content.await? else {
...@@ -240,9 +242,14 @@ impl WebpackLoadersProcessedAsset { ...@@ -240,9 +242,14 @@ impl WebpackLoadersProcessedAsset {
let webpack_loaders_executor = webpack_loaders_executor(*evaluate_context).module(); let webpack_loaders_executor = webpack_loaders_executor(*evaluate_context).module();
let entries = get_evaluate_entries(webpack_loaders_executor, *evaluate_context, None) let entries = get_evaluate_entries(
.to_resolved() webpack_loaders_executor,
.await?; *evaluate_context,
**node_backend,
None,
)
.to_resolved()
.await?;
let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries( let module_graph = ModuleGraph::from_single_graph(SingleModuleGraph::new_with_entries(
entries.graph_entries().to_resolved().await?, entries.graph_entries().to_resolved().await?,
...@@ -264,6 +271,7 @@ impl WebpackLoadersProcessedAsset { ...@@ -264,6 +271,7 @@ impl WebpackLoadersProcessedAsset {
entries, entries,
cwd: project_path.clone(), cwd: project_path.clone(),
env: *env, env: *env,
node_backend: *node_backend,
context_source_for_issue: self.source, context_source_for_issue: self.source,
chunking_context: *chunking_context, chunking_context: *chunking_context,
module_graph, module_graph,
...@@ -434,6 +442,7 @@ pub struct WebpackLoaderContext { ...@@ -434,6 +442,7 @@ pub struct WebpackLoaderContext {
pub entries: ResolvedVc<EvaluateEntries>, pub entries: ResolvedVc<EvaluateEntries>,
pub cwd: FileSystemPath, pub cwd: FileSystemPath,
pub env: ResolvedVc<Box<dyn ProcessEnv>>, pub env: ResolvedVc<Box<dyn ProcessEnv>>,
pub node_backend: ResolvedVc<Box<dyn NodeBackend>>,
pub context_source_for_issue: ResolvedVc<Box<dyn Source>>, pub context_source_for_issue: ResolvedVc<Box<dyn Source>>,
pub module_graph: ResolvedVc<ModuleGraph>, pub module_graph: ResolvedVc<ModuleGraph>,
pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>, pub chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
...@@ -453,6 +462,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -453,6 +462,7 @@ impl EvaluateContext for WebpackLoaderContext {
self.entries, self.entries,
self.cwd.clone(), self.cwd.clone(),
self.env, self.env,
self.node_backend,
self.chunking_context, self.chunking_context,
self.module_graph, self.module_graph,
self.additional_invalidation, self.additional_invalidation,
......
...@@ -13,12 +13,13 @@ use tokio::{ ...@@ -13,12 +13,13 @@ use tokio::{
sync::{Semaphore, oneshot}, sync::{Semaphore, oneshot},
time::sleep, time::sleep,
}; };
use turbo_rcstr::RcStr; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, duration_span}; use turbo_tasks::{ResolvedVc, duration_span};
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use crate::{ use crate::{
AssetsForSourceMapping, AssetsForSourceMapping,
backend::{CreatePoolFuture, CreatePoolOptions, NodeBackend},
evaluate::{EvaluateOperation, EvaluatePool, Operation}, evaluate::{EvaluateOperation, EvaluatePool, Operation},
pool_stats::{AcquiredPermits, PoolStatsSnapshot}, pool_stats::{AcquiredPermits, PoolStatsSnapshot},
worker_pool::{ worker_pool::{
...@@ -148,6 +149,57 @@ impl WorkerThreadPool { ...@@ -148,6 +149,57 @@ impl WorkerThreadPool {
} }
} }
#[turbo_tasks::value(shared)]
pub(crate) struct WorkerThreadsBackend;
#[turbo_tasks::value_impl]
impl NodeBackend for WorkerThreadsBackend {
fn runtime_module_path(&self) -> RcStr {
rcstr!("worker_thread/evaluate.ts")
}
fn globals_module_path(&self) -> RcStr {
rcstr!("worker_thread/globals.ts")
}
fn create_pool(&self, options: CreatePoolOptions) -> CreatePoolFuture {
Box::pin(async move {
let CreatePoolOptions {
cwd,
entrypoint,
env,
assets_for_source_mapping,
assets_root,
project_dir,
concurrency,
debug,
} = options;
Ok(WorkerThreadPool::create(
cwd,
entrypoint,
env,
assets_for_source_mapping,
assets_root,
project_dir,
concurrency,
debug,
)
.await)
})
}
fn scale_down(&self) -> Result<()> {
WorkerThreadPool::scale_down();
Ok(())
}
fn scale_zero(&self) -> Result<()> {
WorkerThreadPool::scale_zero();
Ok(())
}
}
impl WorkerThreadPool { impl WorkerThreadPool {
pub fn scale_down() { pub fn scale_down() {
let _ = WORKER_POOL_OPERATION.scale_down(); let _ = WORKER_POOL_OPERATION.scale_down();
...@@ -207,4 +259,8 @@ impl EvaluateOperation for WorkerThreadPool { ...@@ -207,4 +259,8 @@ impl EvaluateOperation for WorkerThreadPool {
fn stats(&self) -> PoolStatsSnapshot { fn stats(&self) -> PoolStatsSnapshot {
self.state.stats.lock().snapshot() self.state.stats.lock().snapshot()
} }
fn pre_warm(&self) {
// TODO: This is a no-op for worker_pool right now, only process_pool implements it
}
} }
...@@ -51,6 +51,7 @@ use turbopack_css::chunk::CssChunkType; ...@@ -51,6 +51,7 @@ use turbopack_css::chunk::CssChunkType;
use turbopack_ecmascript::{TreeShakingMode, chunk::EcmascriptChunkType}; use turbopack_ecmascript::{TreeShakingMode, chunk::EcmascriptChunkType};
use turbopack_ecmascript_runtime::RuntimeType; use turbopack_ecmascript_runtime::RuntimeType;
use turbopack_node::{ use turbopack_node::{
child_process_backend,
debug::should_debug, debug::should_debug,
evaluate::{evaluate, get_evaluate_entries}, evaluate::{evaluate, get_evaluate_entries},
}; };
...@@ -482,7 +483,9 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V ...@@ -482,7 +483,9 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
) )
.module(); .module();
let entries = get_evaluate_entries(jest_entry_asset, asset_context, None); let node_backend = child_process_backend();
let entries = get_evaluate_entries(jest_entry_asset, asset_context, node_backend, None);
let single_graph = SingleModuleGraph::new_with_entries( let single_graph = SingleModuleGraph::new_with_entries(
entries.graph_entries().to_resolved().await?, entries.graph_entries().to_resolved().await?,
...@@ -569,6 +572,7 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V ...@@ -569,6 +572,7 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V
entries, entries,
path.clone(), path.clone(),
Vc::upcast(CommandLineProcessEnv::new()), Vc::upcast(CommandLineProcessEnv::new()),
node_backend,
Vc::upcast(test_source), Vc::upcast(test_source),
Vc::upcast(chunking_context), Vc::upcast(chunking_context),
module_graph, module_graph,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment