Unverified Commit 2ba4d6ee authored by xusd320's avatar xusd320 Committed by GitHub

refactor(turbopack-node): support execution by napi and worker_threads (#86266)

1. refactor `turbopack-node` to run webpack loaders via nodejs
`worker_threads`
2. the `child_proccess` loader runner is keep in the crate feature to
run unit tests in `turbopack-tests` or run `turbopack-cli`
3. if wanting to switch back to `child_proccess` loader runner, change
features of `turbopack_node/worker_pool` to
`turbopack_node/process_pool` in crate `next-swc-napi`
4. after switched to `worker_threads` loader runner, debugging with
`CDP` (spawn child_process via `--inspect-brk`) doesn't work anymore,
but you can use breakpoint debugging in IDE (like vscode) directly

---------
Co-authored-by: 's avatarNiklas Mischkulnig <4586894+mischnic@users.noreply.github.com>
Co-authored-by: 's avatarBenjamin Woodruff <benjamin.woodruff@vercel.com>
parent 350ce78c
......@@ -4465,6 +4465,7 @@ dependencies = [
"turbo-tasks",
"turbo-tasks-backend",
"turbo-tasks-malloc",
"turbopack-node",
"turbopack-trace-utils",
]
......@@ -4607,6 +4608,7 @@ dependencies = [
"turbopack-core",
"turbopack-ecmascript-hmr-protocol",
"turbopack-ecmascript-plugins",
"turbopack-node",
"turbopack-trace-server",
"turbopack-trace-utils",
"url",
......@@ -10177,10 +10179,13 @@ dependencies = [
"base64 0.21.4",
"bincode 2.0.1",
"const_format",
"dashmap 6.1.0",
"either",
"futures",
"futures-retry",
"indoc",
"napi",
"napi-derive",
"once_cell",
"owo-colors",
"parking_lot",
......
......@@ -325,7 +325,7 @@ turbopack-ecmascript-hmr-protocol = { path = "turbopack/crates/turbopack-ecmascr
turbopack-env = { path = "turbopack/crates/turbopack-env" }
turbopack-image = { path = "turbopack/crates/turbopack-image" }
turbopack-mdx = { path = "turbopack/crates/turbopack-mdx" }
turbopack-node = { path = "turbopack/crates/turbopack-node" }
turbopack-node = { path = "turbopack/crates/turbopack-node", default-features = false }
turbopack-resolve = { path = "turbopack/crates/turbopack-resolve" }
turbopack-static = { path = "turbopack/crates/turbopack-static" }
turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" }
......@@ -419,6 +419,8 @@ napi = { version = "2", default-features = false, features = [
"napi5",
"compat-mode",
] }
napi-derive = "2"
napi-build = "2"
nohash-hasher = "0.2.0"
notify = "8.1.0"
once_cell = "1.17.1"
......
......@@ -40,7 +40,7 @@ turbopack-browser = { workspace = true }
turbopack-core = { workspace = true }
turbopack-css = { workspace = true }
turbopack-ecmascript = { workspace = true }
turbopack-node = { workspace = true }
turbopack-node = { workspace = true, default-features = false }
turbopack-nodejs = { workspace = true }
turbopack-resolve = { workspace = true }
turbopack-wasm = { workspace = true }
......
......@@ -27,4 +27,4 @@ turbo-tasks = { workspace = true }
turbo-tasks-backend = { workspace = true }
turbo-tasks-malloc = { workspace = true }
turbopack-trace-utils = { workspace = true }
turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]}
......@@ -80,7 +80,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [
] }
turbopack-ecmascript-runtime = { workspace = true }
turbopack-image = { workspace = true }
turbopack-node = { workspace = true }
turbopack-node = { workspace = true, default-features = false }
turbopack-nodejs = { workspace = true }
turbopack-resolve = { workspace = true }
turbopack-static = { workspace = true }
......
......@@ -137,8 +137,11 @@ impl VisitMut for TransformVisitor {
Expr::Ident(ident) if is_error_class_name(ident.sym.as_str()) => {
if let Some(args) = &new_expr.args {
// AggregateError(errors, message) has the message as the second arg
let message_arg_index =
if ident.sym.as_str() == "AggregateError" { 1 } else { 0 };
let message_arg_index = if ident.sym.as_str() == "AggregateError" {
1
} else {
0
};
if let Some(message_arg) = args.get(message_arg_index) {
new_error_expr = Some(new_expr.clone());
error_message = Some(stringify_new_error_arg(
......@@ -154,8 +157,11 @@ impl VisitMut for TransformVisitor {
Callee::Expr(expr) => match &**expr {
Expr::Ident(ident) if is_error_class_name(ident.sym.as_str()) => {
// AggregateError(errors, message) has the message as the second arg
let message_arg_index =
if ident.sym.as_str() == "AggregateError" { 1 } else { 0 };
let message_arg_index = if ident.sym.as_str() == "AggregateError" {
1
} else {
0
};
if let Some(message_arg) = call_expr.args.get(message_arg_index) {
error_message = Some(stringify_new_error_arg(
&message_arg.expr,
......
......@@ -46,12 +46,12 @@ workspace = true
[package.metadata.cargo-shear]
ignored = [
# we need to set features on these packages when building for WASM, but we don't directly use them
"getrandom",
"iana-time-zone",
# the plugins feature needs to set a feature on this transitively depended-on package, we never
# directly import it
"turbopack-ecmascript-plugins",
# we need to set features on these packages when building for WASM, but we don't directly use them
"getrandom",
"iana-time-zone",
# the plugins feature needs to set a feature on this transitively depended-on package, we never
# directly import it
"turbopack-ecmascript-plugins",
]
[dependencies]
......@@ -64,7 +64,7 @@ flate2 = { workspace = true }
futures-util = { workspace = true }
owo-colors = { workspace = true }
napi = { workspace = true }
napi-derive = "2"
napi-derive = { workspace = true }
next-custom-transforms = { workspace = true }
next-taskless = { workspace = true }
rand = { workspace = true }
......@@ -117,7 +117,7 @@ next-core = { workspace = true }
mdxjs = { workspace = true, features = ["serializable"] }
turbo-tasks-malloc = { workspace = true, default-features = false, features = [
"custom_allocator"
"custom_allocator",
] }
turbopack-core = { workspace = true }
......@@ -125,6 +125,7 @@ turbopack-ecmascript-hmr-protocol = { workspace = true }
turbopack-trace-utils = { workspace = true }
turbopack-trace-server = { workspace = true }
turbopack-ecmascript-plugins = { workspace = true, optional = true }
turbopack-node = { workspace = true, default-features = false, features = ["worker_pool"] }
[target.'cfg(windows)'.dependencies]
windows-sys = "0.60"
......@@ -145,8 +146,7 @@ tokio = { workspace = true, features = ["full"] }
[build-dependencies]
anyhow = { workspace = true }
napi-build = "2"
napi-build = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
vergen-gitcl = { workspace = true }
......@@ -34,6 +34,30 @@ export declare class ExternalObject<T> {
[K: symbol]: T
}
}
export declare function registerWorkerScheduler(
creator: (arg: NapiWorkerCreation) => any,
terminator: (arg: NapiWorkerTermination) => any
): void
export declare function workerCreated(workerId: number): void
export interface NapiWorkerCreation {
options: NapiWorkerOptions
}
export interface NapiWorkerOptions {
filename: RcStr
cwd: RcStr
}
export interface NapiWorkerTermination {
options: NapiWorkerOptions
workerId: number
}
export interface NapiTaskMessage {
taskId: number
data: Buffer
}
export declare function recvTaskMessageInWorker(
workerId: number
): Promise<NapiTaskMessage>
export declare function sendTaskMessage(message: NapiTaskMessage): Promise<void>
export declare function lockfileTryAcquireSync(
path: string,
content?: string | undefined | null
......
......@@ -43,6 +43,7 @@ import type {
WrittenEndpoint,
} from './types'
import { throwTurbopackInternalError } from '../../shared/lib/turbopack/internal-error'
import { runLoaderWorkerPool } from './loaderWorkerPool'
export enum HmrTarget {
Client = 'client',
......@@ -500,6 +501,7 @@ const normalizePathOnWindows = (p: string) =>
// TODO(sokra) Support wasm option.
function bindingToApi(
binding: RawBindings,
bindingPath: string,
_wasm: boolean
): Binding['turbo']['createProject'] {
type NativeFunction<T> = (
......@@ -680,6 +682,10 @@ function bindingToApi(
constructor(nativeProject: { __napiType: 'Project' }) {
this._nativeProject = nativeProject
if (typeof binding.registerWorkerScheduler === 'function') {
runLoaderWorkerPool(binding, bindingPath)
}
}
async update(options: PartialProjectOptions) {
......@@ -1459,21 +1465,24 @@ function loadNative(importPath?: string) {
throw new Error('cannot run loadNative when `NEXT_TEST_WASM` is set')
}
const customBindings: RawBindings = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
? require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS)
const customBindingsPath = !!__INTERNAL_CUSTOM_TURBOPACK_BINDINGS
? require.resolve(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS)
: null
const customBindings: RawBindings =
customBindingsPath != null ? require(customBindingsPath!) : null
let bindings: RawBindings = customBindings
let bindingsPath = customBindingsPath
// callers expect that if loadNative throws, it's an array of strings.
let attempts: string[] = []
let attempts: any[] = []
const NEXT_TEST_NATIVE_DIR = process.env.NEXT_TEST_NATIVE_DIR
for (const triple of triples) {
if (NEXT_TEST_NATIVE_DIR) {
try {
const bindingForTest = `${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node`
// Use the binary directly to skip `pnpm pack` for testing as it's slow because of the large native binary.
bindings = require(
`${NEXT_TEST_NATIVE_DIR}/next-swc.${triple.platformArchABI}.node`
)
bindingsPath = require.resolve(bindingForTest)
bindings = require(bindingsPath)
infoLog(
'next-swc build: local built @next/swc from NEXT_TEST_NATIVE_DIR'
)
......@@ -1485,9 +1494,9 @@ function loadNative(importPath?: string) {
}
} else if (process.env.NEXT_TEST_NATIVE_IGNORE_LOCAL_INSTALL !== 'true') {
try {
bindings = require(
`@next/swc/native/next-swc.${triple.platformArchABI}.node`
)
const normalBinding = `@next/swc/native/next-swc.${triple.platformArchABI}.node`
bindings = require(normalBinding)
bindingsPath = require.resolve(normalBinding)
infoLog('next-swc build: local built @next/swc')
break
} catch (e) {}
......@@ -1509,6 +1518,7 @@ function loadNative(importPath?: string) {
: `@next/swc-${triple.platformArchABI}`
try {
bindings = require(pkg)
bindingsPath = require.resolve(pkg)
if (!importPath) {
checkVersionMismatch(require(`${pkg}/package.json`))
}
......@@ -1587,7 +1597,11 @@ function loadNative(importPath?: string) {
initCustomTraceSubscriber: bindings.initCustomTraceSubscriber,
teardownTraceSubscriber: bindings.teardownTraceSubscriber,
turbo: {
createProject: bindingToApi(customBindings ?? bindings, false),
createProject: bindingToApi(
customBindings ?? bindings,
customBindingsPath ?? bindingsPath!,
false
),
startTurbopackTraceServer(traceFilePath, port) {
Log.warn(
`Turbopack trace server started. View trace at https://trace.nextjs.org${port != null ? `?port=${port}` : ''}`
......
import { Worker } from 'worker_threads'
const loaderWorkers: Record<string, Map<number, Worker>> = {}
function getPoolId(cwd: string, filename: string) {
return `${cwd}:${filename}`
}
export async function runLoaderWorkerPool(
bindings: typeof import('./generated-native'),
bindingPath: string
) {
bindings.registerWorkerScheduler(
(creation) => {
const {
options: { filename, cwd },
} = creation
const poolId = getPoolId(cwd, filename)
const worker = new Worker(/* turbopackIgnore: true*/ filename, {
workerData: {
bindingPath,
cwd,
},
})
// This will cause handing when run in jest worker, but not as a first level thread of nodejs thread
// worker.unref()
const workers =
loaderWorkers[poolId] || (loaderWorkers[poolId] = new Map())
workers.set(worker.threadId, worker)
},
(termination) => {
const {
options: { filename, cwd },
workerId,
} = termination
const poolId = getPoolId(cwd, filename)
const workers = loaderWorkers[poolId]
workers.get(workerId)?.terminate()
workers.delete(workerId)
}
)
}
......@@ -148,8 +148,13 @@ async function readNormalizedNFT(next, name) {
"/node_modules/next/dist/build/static-paths/app/extract-pathname-route-param-segments-from-loader-tree.js",
"/node_modules/next/dist/build/static-paths/pages.js",
"/node_modules/next/dist/build/static-paths/utils.js",
"/node_modules/next/dist/build/swc/helpers.js",
"/node_modules/next/dist/build/swc/index.js",
"/node_modules/next/dist/build/swc/install-bindings.js",
"/node_modules/next/dist/build/swc/jest-transformer.js",
"/node_modules/next/dist/build/swc/loaderWorkerPool.js",
"/node_modules/next/dist/build/swc/options.js",
"/node_modules/next/dist/build/swc/types.js",
"/node_modules/next/dist/build/utils.js",
"/node_modules/next/dist/cli/next-test.js",
"/node_modules/next/dist/client/*",
......
......@@ -69,7 +69,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [
] }
turbopack-ecmascript-runtime = { workspace = true }
turbopack-env = { workspace = true }
turbopack-node = { workspace = true }
turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]}
turbopack-nodejs = { workspace = true }
turbopack-resolve = { workspace = true }
turbopack-trace-utils = { workspace = true }
......
......@@ -10,8 +10,11 @@ autobenches = false
bench = false
[features]
default = ["process_pool"]
# enable "HMR" for embedded assets
dynamic_embed_contents = ["turbo-tasks-fs/dynamic_embed_contents"]
process_pool = ["tokio/full"]
worker_pool = ["tokio/sync", "napi", "napi-derive", "turbo-rcstr/napi"]
[lints]
workspace = true
......@@ -20,6 +23,7 @@ workspace = true
anyhow = { workspace = true }
async-stream = "0.3.4"
async-trait = { workspace = true }
dashmap = { workspace = true }
base64 = "0.21.0"
bincode = { workspace = true }
const_format = { workspace = true }
......@@ -35,7 +39,7 @@ rustc-hash = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_with = { workspace = true, features = ["base64"] }
tokio = { workspace = true, features = ["full"] }
tokio = { workspace = true, optional = true }
tracing = { workspace = true }
turbo-bincode = { workspace = true }
turbo-rcstr = { workspace = true }
......@@ -47,9 +51,10 @@ turbopack-cli-utils = { workspace = true }
turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true }
turbopack-resolve = { workspace = true }
napi = { workspace = true, features = ["error_anyhow"], optional = true }
napi-derive = { workspace = true, optional = true }
[dev-dependencies]
rustc-hash = { workspace = true }
turbo-tasks-backend = { workspace = true }
turbo-tasks-testing = { workspace = true }
import { IPC } from './index'
import type { Ipc as GenericIpc } from './index'
import type { Channel as Ipc } from '../types'
type IpcIncomingMessage =
| {
type: 'evaluate'
......@@ -29,11 +31,6 @@ type IpcOutgoingMessage =
data: any
}
export type Ipc<IM, RM> = {
sendInfo(message: IM): Promise<void>
sendRequest(message: RM): Promise<unknown>
sendError(error: Error): Promise<never>
}
const ipc = IPC as GenericIpc<IpcIncomingMessage, IpcOutgoingMessage>
const queue: string[][] = []
......
import { createConnection } from 'node:net'
import { Writable } from 'node:stream'
import type { StackFrame } from '../compiled/stacktrace-parser'
import { parse as parseStackTrace } from '../compiled/stacktrace-parser'
import { getProperError } from './error'
export type StructuredError = {
name: string
message: string
stack: StackFrame[]
cause: StructuredError | undefined
}
export function structuredError(e: unknown): StructuredError {
e = getProperError(e)
return {
name: e.name,
message: e.message,
stack: typeof e.stack === 'string' ? parseStackTrace(e.stack) : [],
cause: e.cause ? structuredError(getProperError(e.cause)) : undefined,
}
}
import { structuredError } from '../error'
type State =
| {
......
import type { StackFrame } from './compiled/stacktrace-parser'
import { parse as parseStackTrace } from './compiled/stacktrace-parser'
// merged from next.js
// https://github.com/vercel/next.js/blob/e657741b9908cf0044aaef959c0c4defb19ed6d8/packages/next/src/lib/is-error.ts
// https://github.com/vercel/next.js/blob/e657741b9908cf0044aaef959c0c4defb19ed6d8/packages/next/src/shared/lib/is-plain-object.ts
......@@ -50,3 +53,21 @@ function isPlainObject(value: any): boolean {
*/
return prototype === null || prototype.hasOwnProperty('isPrototypeOf')
}
export type StructuredError = {
name: string
message: string
stack: StackFrame[]
cause: StructuredError | undefined
}
export function structuredError(e: Error | string): StructuredError {
e = getProperError(e)
return {
name: e.name,
message: e.message,
stack: typeof e.stack === 'string' ? parseStackTrace(e.stack) : [],
cause: e.cause ? structuredError(getProperError(e.cause)) : undefined,
}
}
......@@ -2,9 +2,9 @@
* Shared utilities for our 2 transform implementations.
*/
import type { Ipc } from '../ipc/evaluate'
import type { Channel as Ipc } from '../types'
import { relative, isAbsolute, join, sep } from 'path'
import { type StructuredError } from '../ipc'
import { type StructuredError } from '../error'
import { type StackFrame } from '../compiled/stacktrace-parser'
export type IpcInfoMessage =
......
declare const __turbopack_external_require__: {
resolve: (name: string, opt: { paths: string[] }) => string
resolve: (name: string, opt?: { paths: string[] }) => string
} & ((id: string, thunk: () => any, esm?: boolean) => any)
import type { Ipc } from '../ipc/evaluate'
import type { Channel as Ipc } from '../types'
import { dirname, resolve as pathResolve, relative } from 'path'
import {
StackFrame,
parse as parseStackTrace,
} from '../compiled/stacktrace-parser'
import { structuredError, type StructuredError } from '../ipc'
import { structuredError, type StructuredError } from '../error'
import {
fromPath,
getReadEnvVariables,
......
export type Channel<IM, RM> = {
sendInfo(message: IM): Promise<void>
sendRequest(message: RM): Promise<unknown>
sendError(error: Error): Promise<void>
}
import { threadId as workerId, workerData } from 'worker_threads'
import { structuredError } from '../error'
import type { Channel } from '../types'
import { Binding, TaskChannel, TEXT_ENCODER, TEXT_DECODER } from './taskChannel'
if (!workerData.hasOwnProperty('bindingPath')) {
throw new Error('bindingPath not set in loader worker thread')
}
const binding: Binding = require(
/* turbopackIgnore: true */ workerData.bindingPath
)
binding.workerCreated(workerId)
export const run = async (
moduleFactory: () => Promise<{
init?: () => Promise<void>
default: (channel: Channel<any, any>, ...deserializedArgs: any[]) => any
}>
) => {
let getValue: (channel: Channel<any, any>, ...deserializedArgs: any[]) => any
let isRunning = false
const queue: Array<{ taskId: number; args: string[] }> = []
const run = async (taskId: number, args: string[]) => {
try {
if (typeof getValue !== 'function') {
const module = await moduleFactory()
if (typeof module.init === 'function') {
await module.init()
}
getValue = module.default
}
const value = await getValue(new TaskChannel(binding, taskId), ...args)
await binding.sendTaskMessage({
taskId,
data: TEXT_ENCODER.encode(
JSON.stringify({
type: 'end',
data: value === undefined ? undefined : JSON.stringify(value),
duration: 0,
})
),
})
} catch (err) {
await binding.sendTaskMessage({
taskId,
data: TEXT_ENCODER.encode(
JSON.stringify({
type: 'error',
...structuredError(err as Error),
})
),
})
}
if (queue.length > 0) {
const next = queue.shift()!
run(next.taskId, next.args)
} else {
isRunning = false
}
}
while (true) {
const { taskId, data: msg_data } =
await binding.recvTaskMessageInWorker(workerId)
const msg_str = TEXT_DECODER.decode(msg_data)
const msg = JSON.parse(msg_str) as
| {
type: 'evaluate'
args: string[]
}
| {
type: 'result'
id: number
error?: string
data?: any
}
switch (msg.type) {
case 'evaluate': {
if (!isRunning) {
isRunning = true
run(taskId, msg.args)
} else {
queue.push({ taskId, args: msg.args })
}
break
}
case 'result': {
const request = TaskChannel.requests.get(msg.id)
if (request) {
TaskChannel.requests.delete(msg.id)
if (msg.error) {
// Need to reject at next macro task queue, because some rejection callbacks is not registered when executing to here,
// that will cause the error be propergated to schedule thread, then causing panic.
// The situation always happen when using sass-loader, it will try to resolve many posible dependencies,
// some of them may fail with error.
setTimeout(() => request.reject(new Error(msg.error)), 0)
} else {
request.resolve(msg.data)
}
}
break
}
default: {
console.error('unexpected message type', (msg as any).type)
}
}
}
}
import { workerData } from 'worker_threads'
// We override the `cwd` in workers. Every worker thread gets an isolated
// `process` object, so this mutation is safe.
if (!workerData.hasOwnProperty('cwd')) {
throw new Error('cwd not set in loader worker thread')
}
process.cwd = () => workerData.cwd
// @ts-ignore
process.turbopack = {}
import { structuredError } from '../error'
export const TEXT_ENCODER = new TextEncoder()
export const TEXT_DECODER = new TextDecoder()
export interface TaskMessage {
taskId: number
data: Uint8Array
}
export interface Binding {
recvTaskMessageInWorker(workerId: number): Promise<TaskMessage>
sendTaskMessage(msg: TaskMessage): Promise<void>
workerCreated(workerId: number): void
}
// Export this, maybe in the future, we can add an implementation via web worker on browser
export class TaskChannel {
static nextId = 1
static requests = new Map()
constructor(
private binding: Binding,
private taskId: number
) {}
async sendInfo(message: any) {
return await this.binding.sendTaskMessage({
taskId: this.taskId,
data: TEXT_ENCODER.encode(
JSON.stringify({
type: 'info',
data: message,
})
),
})
}
async sendRequest(message: any) {
const id = TaskChannel.nextId++
let resolve, reject
const promise = new Promise((res, rej) => {
resolve = res
reject = rej
})
TaskChannel.requests.set(id, { resolve, reject })
return await this.binding
.sendTaskMessage({
taskId: this.taskId,
data: TEXT_ENCODER.encode(
JSON.stringify({ type: 'request', id, data: message })
),
})
.then(() => promise)
}
async sendError(error: Error) {
try {
await this.binding.sendTaskMessage({
taskId: this.taskId,
data: TEXT_ENCODER.encode(
JSON.stringify({
type: 'error',
...structuredError(error),
})
),
})
} catch (err) {
// There's nothing we can do about errors that happen after this point, we can't tell anyone
// about them.
console.error('failed to send error back to rust:', err)
}
}
}
use std::fmt::Display;
use owo_colors::{OwoColorize, Style};
#[derive(Clone, Copy)]
pub enum FormattingMode {
/// No formatting, just print the output
Plain,
/// Use ansi colors to format the output
AnsiColors,
}
impl FormattingMode {
pub fn magic_identifier<'a>(&self, content: impl Display + 'a) -> impl Display + 'a {
match self {
FormattingMode::Plain => format!("{{{content}}}"),
FormattingMode::AnsiColors => format!("{{{content}}}").italic().to_string(),
}
}
pub fn lowlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a {
match self {
FormattingMode::Plain => Style::new(),
FormattingMode::AnsiColors => Style::new().dimmed(),
}
.style(content)
}
pub fn highlight<'a>(&self, content: impl Display + 'a) -> impl Display + 'a {
match self {
FormattingMode::Plain => Style::new(),
FormattingMode::AnsiColors => Style::new().bold().underline(),
}
.style(content)
}
}
......@@ -17,10 +17,15 @@ pub mod debug;
pub mod embed_js;
pub mod evaluate;
pub mod execution_context;
mod heap_queue;
pub mod pool;
mod format;
mod pool_stats;
// When both features are enabled, worker_pool takes priority
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
pub mod process_pool;
pub mod source_map;
pub mod transforms;
#[cfg(feature = "worker_pool")]
pub mod worker_pool;
#[turbo_tasks::function]
async fn emit(
......
use std::{cmp::max, fmt::Debug, time::Duration};
use tokio::sync::OwnedSemaphorePermit;
pub enum AcquiredPermits {
Idle {
// This is used for drop
_concurrency_permit: OwnedSemaphorePermit,
},
Fresh {
// This is used for drop
_concurrency_permit: OwnedSemaphorePermit,
// This is used for drop
_bootup_permit: OwnedSemaphorePermit,
},
}
#[derive(Default)]
pub struct NodeJsPoolStats {
pub total_bootup_time: Duration,
pub bootup_count: u32,
pub total_cold_process_time: Duration,
pub cold_process_count: u32,
pub total_warm_process_time: Duration,
pub warm_process_count: u32,
pub workers: u32,
pub booting_workers: u32,
pub queued_tasks: u32,
}
impl NodeJsPoolStats {
#[allow(unused)]
pub fn add_bootup_time(&mut self, time: Duration) {
self.total_bootup_time += time;
self.bootup_count += 1;
}
pub fn add_booting_worker(&mut self) {
self.booting_workers += 1;
self.workers += 1;
}
pub fn finished_booting_worker(&mut self) {
self.booting_workers = self.booting_workers.saturating_sub(1);
}
pub fn remove_worker(&mut self) {
self.workers = self.workers.saturating_sub(1);
}
#[allow(unused)]
pub fn add_queued_task(&mut self) {
self.queued_tasks += 1;
}
#[allow(unused)]
pub fn add_cold_process_time(&mut self, time: Duration) {
self.total_cold_process_time += time;
self.cold_process_count += 1;
self.queued_tasks = self.queued_tasks.saturating_sub(1);
}
#[allow(unused)]
pub fn add_warm_process_time(&mut self, time: Duration) {
self.total_warm_process_time += time;
self.warm_process_count += 1;
self.queued_tasks = self.queued_tasks.saturating_sub(1);
}
pub fn estimated_bootup_time(&self) -> Duration {
if self.bootup_count == 0 {
Duration::from_millis(200)
} else {
self.total_bootup_time / self.bootup_count
}
}
pub fn estimated_warm_process_time(&self) -> Duration {
if self.warm_process_count == 0 {
self.estimated_cold_process_time()
} else {
self.total_warm_process_time / self.warm_process_count
}
}
pub fn estimated_cold_process_time(&self) -> Duration {
if self.cold_process_count == 0 {
// We assume cold processing is half of bootup time
self.estimated_bootup_time() / 2
} else {
self.total_cold_process_time / self.cold_process_count
}
}
#[allow(unused)]
pub fn wait_time_before_bootup(&self) -> Duration {
if self.workers == 0 {
return Duration::ZERO;
}
let booting_workers = self.booting_workers;
let workers = self.workers;
let warm_process_time = self.estimated_warm_process_time();
let expected_completion = self.expected_completion(workers, booting_workers);
let new_process_duration =
self.estimated_bootup_time() + self.estimated_cold_process_time();
if expected_completion + warm_process_time < new_process_duration {
// Running the task with the existing warm pool is faster
return (expected_completion + warm_process_time + new_process_duration) / 2;
}
let expected_completion_with_additional_worker = max(
new_process_duration,
self.expected_completion(workers + 1, booting_workers + 1),
);
if expected_completion > expected_completion_with_additional_worker {
// Scaling up the pool would help to complete work faster
return Duration::ZERO;
}
// It's expected to be faster if we queue the task
(expected_completion + expected_completion_with_additional_worker) / 2
}
fn expected_completion(&self, workers: u32, booting_workers: u32) -> Duration {
if workers == 0 {
return Duration::MAX;
}
let bootup_time = self.estimated_bootup_time();
let cold_process_time = self.estimated_cold_process_time();
let warm_process_time = self.estimated_warm_process_time();
let expected_full_workers_in = booting_workers * (bootup_time / 2 + cold_process_time);
let expected_completed_task_until_full_workers = {
let millis = max(1, warm_process_time.as_millis());
let ready_workers = workers - booting_workers;
(expected_full_workers_in.as_millis() / millis) as u32 * ready_workers
};
let remaining_tasks = self
.queued_tasks
.saturating_sub(expected_completed_task_until_full_workers);
if remaining_tasks > 0 {
expected_full_workers_in + warm_process_time * remaining_tasks / workers
} else {
warm_process_time * self.queued_tasks / workers
}
}
pub fn snapshot(&self) -> PoolStatsSnapshot {
PoolStatsSnapshot {
bootup_count: self.bootup_count,
warm_operation_count: self.warm_process_count,
cold_operation_count: self.cold_process_count,
workers: self.workers,
booting_workers: self.booting_workers,
}
}
}
impl Debug for NodeJsPoolStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NodeJsPoolStats")
.field("queued_tasks", &self.queued_tasks)
.field("workers", &self.workers)
.field("booting_workers", &self.booting_workers)
.field(
"expected_completion",
&self.expected_completion(self.workers, self.booting_workers),
)
.field("bootup_time", &self.estimated_bootup_time())
.field("cold_process_time", &self.estimated_cold_process_time())
.field("warm_process_time", &self.estimated_warm_process_time())
.field("bootup_count", &self.bootup_count)
.field("cold_process_count", &self.cold_process_count)
.field("warm_process_count", &self.warm_process_count)
.finish()
}
}
/// A snapshot of pool statistics, useful for testing and diagnostics.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PoolStatsSnapshot {
/// Total number of processes ever successfully booted.
pub bootup_count: u32,
/// Number of completed operations that reused an idle process.
pub warm_operation_count: u32,
/// Number of completed operations that spawned a fresh process.
pub cold_operation_count: u32,
/// Current number of tracked workers (booting + idle + in-use).
pub workers: u32,
/// Current number of workers still booting.
pub booting_workers: u32,
}
......@@ -21,7 +21,7 @@ use turbopack_core::{
use turbopack_ecmascript::magic_identifier::unmangle_identifiers;
pub use crate::source_map::trace::{StackFrame, TraceResult, trace_source_map};
use crate::{AssetsForSourceMapping, pool::FormattingMode};
use crate::{AssetsForSourceMapping, format::FormattingMode};
pub mod trace;
......
......@@ -54,11 +54,11 @@ use crate::{
debug::should_debug,
embed_js::embed_file_path,
evaluate::{
EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluationIssue, custom_evaluate,
get_evaluate_entries, get_evaluate_pool,
EnvVarTracking, EvaluateContext, EvaluateEntries, EvaluatePool, EvaluationIssue,
custom_evaluate, get_evaluate_entries, get_evaluate_pool,
},
execution_context::ExecutionContext,
pool::{FormattingMode, NodeJsPool},
format::FormattingMode,
source_map::{StackFrame, StructuredError},
transforms::util::{EmittedAsset, emitted_assets_to_virtual_sources},
};
......@@ -448,7 +448,7 @@ impl EvaluateContext for WebpackLoaderContext {
type ResponseMessage = ResponseMessage;
type State = Vec<LogInfo>;
fn pool(&self) -> OperationVc<crate::pool::NodeJsPool> {
fn pool(&self) -> OperationVc<EvaluatePool> {
get_evaluate_pool(
self.entries,
self.cwd.clone(),
......@@ -476,7 +476,7 @@ impl EvaluateContext for WebpackLoaderContext {
true
}
async fn emit_error(&self, error: StructuredError, pool: &NodeJsPool) -> Result<()> {
async fn emit_error(&self, error: StructuredError, pool: &EvaluatePool) -> Result<()> {
EvaluationIssue {
error,
source: IssueSource::from_source_only(self.context_source_for_issue),
......@@ -493,7 +493,7 @@ impl EvaluateContext for WebpackLoaderContext {
&self,
state: &mut Self::State,
data: Self::InfoMessage,
pool: &NodeJsPool,
pool: &EvaluatePool,
) -> Result<()> {
match data {
InfoMessage::Dependencies {
......@@ -569,7 +569,7 @@ impl EvaluateContext for WebpackLoaderContext {
&self,
_state: &mut Self::State,
data: Self::RequestMessage,
_pool: &NodeJsPool,
_pool: &EvaluatePool,
) -> Result<Self::ResponseMessage> {
match data {
RequestMessage::Resolve {
......@@ -623,7 +623,7 @@ impl EvaluateContext for WebpackLoaderContext {
}
}
async fn finish(&self, state: Self::State, pool: &NodeJsPool) -> Result<()> {
async fn finish(&self, state: Self::State, pool: &EvaluatePool) -> Result<()> {
let has_errors = state.iter().any(|log| log.log_type == LogType::Error);
let has_warnings = state.iter().any(|log| log.log_type == LogType::Warn);
if has_errors || has_warnings {
......
use std::{
path::PathBuf,
sync::{
Arc,
atomic::{AtomicU32, Ordering},
},
};
use anyhow::{Context, Result};
use rustc_hash::FxHashMap;
use tokio::{
select,
sync::{Semaphore, oneshot},
time::sleep,
};
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, duration_span};
use turbo_tasks_fs::FileSystemPath;
use crate::{
AssetsForSourceMapping,
evaluate::{EvaluateOperation, EvaluatePool, Operation},
pool_stats::{AcquiredPermits, PoolStatsSnapshot},
worker_pool::{
operation::{
PoolState, TaskChannels, WORKER_POOL_OPERATION, WorkerOperation, WorkerOptions,
get_pool_state,
},
worker_thread::create_worker,
},
};
mod operation;
mod worker_thread;
static OPERATION_TASK_ID: AtomicU32 = AtomicU32::new(1);
#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)]
pub(crate) struct WorkerThreadPool {
worker_options: Arc<WorkerOptions>,
concurrency: usize,
pub(crate) assets_for_source_mapping: ResolvedVc<AssetsForSourceMapping>,
pub(crate) assets_root: FileSystemPath,
pub(crate) project_dir: FileSystemPath,
#[turbo_tasks(trace_ignore, debug_ignore)]
state: Arc<PoolState>,
#[turbo_tasks(trace_ignore, debug_ignore)]
concurrency_semaphore: Arc<Semaphore>,
#[turbo_tasks(trace_ignore, debug_ignore)]
bootup_semaphore: Arc<Semaphore>,
}
impl WorkerThreadPool {
pub(crate) async fn create(
cwd: PathBuf,
entrypoint: PathBuf,
// The worker thread will inherit env from parent process, so it's not needed
_env: FxHashMap<RcStr, RcStr>,
assets_for_source_mapping: ResolvedVc<AssetsForSourceMapping>,
assets_root: FileSystemPath,
project_dir: FileSystemPath,
concurrency: usize,
debug: bool,
) -> EvaluatePool {
let cwd: RcStr = cwd.to_string_lossy().into();
let filename: RcStr = entrypoint.to_string_lossy().into();
let worker_options = Arc::new(WorkerOptions { cwd, filename });
let state = get_pool_state(worker_options.clone()).await;
EvaluatePool::new(
Box::new(Self {
worker_options,
concurrency: (if debug { 1 } else { concurrency }),
assets_for_source_mapping,
assets_root: assets_root.clone(),
project_dir: project_dir.clone(),
state,
concurrency_semaphore: Arc::new(Semaphore::new(if debug {
1
} else {
concurrency
})),
bootup_semaphore: Arc::new(Semaphore::new(1)),
}) as Box<dyn EvaluateOperation>,
assets_for_source_mapping,
assets_root,
project_dir,
)
}
async fn acquire_worker(&self) -> Result<(u32, AcquiredPermits)> {
let concurrency_permit = self.concurrency_semaphore.clone().acquire_owned().await?;
{
let mut idle = self.state.idle_workers.lock();
if let Some(worker_id) = idle.pop() {
return Ok((
worker_id,
AcquiredPermits::Idle {
_concurrency_permit: concurrency_permit,
},
));
}
}
let (tx, rx) = oneshot::channel();
{
let mut waiters = self.state.waiters.lock();
let mut idle = self.state.idle_workers.lock();
if let Some(worker_id) = idle.pop() {
return Ok((
worker_id,
AcquiredPermits::Idle {
_concurrency_permit: concurrency_permit,
},
));
}
waiters.push(tx);
}
let bootup = async {
let permit = self.bootup_semaphore.clone().acquire_owned().await;
let wait_time = self.state.stats.lock().wait_time_before_bootup();
sleep(wait_time).await;
permit
};
select! {
worker_id = rx => {
let worker_id = worker_id?;
Ok((worker_id, AcquiredPermits::Idle { _concurrency_permit: concurrency_permit }))
}
bootup_permit = bootup => {
let bootup_permit = bootup_permit.context("acquiring bootup permit")?;
{
self.state.stats.lock().add_booting_worker();
}
let worker_id = create_worker(self.worker_options.clone()).await?;
{
let mut stats = self.state.stats.lock();
stats.finished_booting_worker();
}
self.bootup_semaphore.add_permits(1);
Ok((worker_id, AcquiredPermits::Fresh { _concurrency_permit: concurrency_permit, _bootup_permit: bootup_permit }))
}
}
}
}
impl WorkerThreadPool {
pub fn scale_down() {
let _ = WORKER_POOL_OPERATION.scale_down();
}
pub fn scale_zero() {
let _ = WORKER_POOL_OPERATION.scale_zero();
}
}
#[async_trait::async_trait]
impl EvaluateOperation for WorkerThreadPool {
async fn operation(&self) -> Result<Box<dyn Operation>> {
let operation = {
let _guard = duration_span!("Node.js operation");
let worker_options = self.worker_options.clone();
let task_id = OPERATION_TASK_ID.fetch_add(1, Ordering::Release);
if task_id == 0 {
panic!("Node.js operation task id overflow")
}
let (worker_id, permits) = self.acquire_worker().await?;
let state = self.state.clone();
// Pre-allocate channels for this task to avoid HashMap lookups during communication
let channels = TaskChannels::new(task_id, worker_id);
WorkerOperation {
worker_options,
worker_id,
state: state.clone(),
on_drop: Some(Box::new(move |worker_id| {
let mut waiters = state.waiters.lock();
loop {
if let Some(tx) = waiters.pop() {
if tx.send(worker_id).is_ok() {
break;
}
} else {
state.idle_workers.lock().push(worker_id);
break;
}
}
})),
_permits: permits,
channels,
}
};
Ok(Box::new(operation))
}
/// Returns a snapshot of the pool's internal statistics.
fn stats(&self) -> PoolStatsSnapshot {
self.state.stats.lock().snapshot()
}
}
use std::{
process::ExitStatus,
sync::{Arc, LazyLock},
};
use anyhow::{Context, Result};
use parking_lot::Mutex;
use rustc_hash::FxHashMap;
use tokio::sync::{
Mutex as AsyncMutex,
mpsc::{self, UnboundedReceiver, UnboundedSender},
oneshot,
};
use turbo_rcstr::RcStr;
use crate::{
evaluate::Operation,
pool_stats::{AcquiredPermits, NodeJsPoolStats},
worker_pool::worker_thread,
};
/// A bidirectional message channel using unbounded mpsc.
#[derive(Clone)]
pub(crate) struct MessageChannel<T: Send + Sync + 'static> {
sender: UnboundedSender<T>,
receiver: Arc<AsyncMutex<UnboundedReceiver<T>>>,
}
impl<T: Send + Sync + 'static> MessageChannel<T> {
pub(super) fn unbounded() -> Self {
let (sender, receiver) = mpsc::unbounded_channel();
Self {
sender,
receiver: Arc::new(AsyncMutex::new(receiver)),
}
}
pub(crate) async fn send(&self, message: T) -> Result<()> {
self.sender
.send(message)
.map_err(|_| anyhow::anyhow!("failed to send message"))
}
pub(crate) async fn recv(&self) -> Result<T> {
let mut rx = self.receiver.lock().await;
rx.recv()
.await
.ok_or_else(|| anyhow::anyhow!("failed to recv message"))
}
}
#[derive(Default)]
pub(crate) struct PoolState {
pub(crate) idle_workers: Mutex<Vec<u32>>,
pub(crate) stats: Arc<Mutex<NodeJsPoolStats>>,
pub(crate) waiters: Mutex<Vec<oneshot::Sender<u32>>>,
}
#[turbo_tasks::value(cell = "new", serialization = "none", eq = "manual", shared)]
#[derive(Clone, PartialEq, Eq, Hash)]
pub(super) struct WorkerOptions {
pub(super) filename: RcStr,
pub(super) cwd: RcStr,
}
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
pub(super) struct TaskMessage {
pub task_id: u32,
pub data: Vec<u8>,
}
#[derive(Default)]
pub(crate) struct WorkerPoolOperation {
#[allow(clippy::type_complexity)]
worker_routed_channel: Mutex<FxHashMap<u32, Arc<MessageChannel<(u32, Vec<u8>)>>>>,
#[allow(clippy::type_complexity)]
task_routed_channel: Mutex<FxHashMap<u32, Arc<MessageChannel<Vec<u8>>>>>,
pub(crate) pools: Mutex<FxHashMap<Arc<WorkerOptions>, Arc<PoolState>>>,
}
impl WorkerPoolOperation {
pub(crate) async fn get_pool_state(
&self,
worker_options: Arc<WorkerOptions>,
) -> Arc<PoolState> {
self.pools.lock().entry(worker_options).or_default().clone()
}
pub(crate) fn scale_down(&self) -> Result<()> {
let mut to_terminate = Vec::new();
{
let pools = self.pools.lock();
for (worker_options, state) in pools.iter() {
let mut idle = state.idle_workers.lock();
if idle.len() > 1 {
let workers = idle.split_off(1);
let mut stats = state.stats.lock();
for worker_id in workers {
stats.remove_worker();
to_terminate.push((worker_options.clone(), worker_id));
}
}
}
}
to_terminate
.into_iter()
.map(|(worker_options, worker_id)| self.terminate_worker(worker_options, worker_id))
.collect::<Result<Vec<_>>>()?;
Ok(())
}
pub(crate) fn scale_zero(&self) -> Result<()> {
let mut to_terminate = Vec::new();
{
let pools = self.pools.lock();
for (worker_options, state) in pools.iter() {
let mut idle = state.idle_workers.lock();
let workers = std::mem::take(&mut *idle);
let mut stats = state.stats.lock();
for worker_id in workers {
stats.remove_worker();
to_terminate.push((worker_options.clone(), worker_id));
}
}
}
to_terminate
.into_iter()
.map(|(worker_options, worker_id)| self.terminate_worker(worker_options, worker_id))
.collect::<Result<Vec<_>>>()?;
Ok(())
}
pub(crate) fn terminate_worker(
&self,
worker_options: Arc<WorkerOptions>,
worker_id: u32,
) -> Result<()> {
self.remove_worker_channel(worker_id);
worker_thread::terminate_worker(worker_options, worker_id);
Ok(())
}
fn remove_worker_channel(&self, worker_id: u32) {
self.worker_routed_channel.lock().remove(&worker_id);
}
pub(crate) async fn recv_task_message_in_worker(
&self,
worker_id: u32,
) -> Result<(u32, Vec<u8>)> {
let channel = {
let mut map = self.worker_routed_channel.lock();
map.entry(worker_id)
.or_insert_with(|| Arc::new(MessageChannel::unbounded()))
.clone()
};
channel
.recv()
.await
.with_context(|| format!("failed to recv message in worker {worker_id}"))
}
pub(crate) async fn send_task_message(&self, message: TaskMessage) -> Result<()> {
let channel = {
let mut map = self.task_routed_channel.lock();
map.entry(message.task_id)
.or_insert_with(|| Arc::new(MessageChannel::unbounded()))
.clone()
};
channel
.send(message.data)
.await
.with_context(|| format!("failed to send response for task {}", message.task_id))
}
}
pub(crate) static WORKER_POOL_OPERATION: LazyLock<WorkerPoolOperation> =
LazyLock::new(WorkerPoolOperation::default);
pub(crate) fn terminate_worker(worker_options: Arc<WorkerOptions>, worker_id: u32) -> Result<()> {
WORKER_POOL_OPERATION.terminate_worker(worker_options, worker_id)
}
pub(crate) async fn get_pool_state(worker_options: Arc<WorkerOptions>) -> Arc<PoolState> {
WORKER_POOL_OPERATION.get_pool_state(worker_options).await
}
/// Pre-allocated channels for a single task's communication.
/// Holds Arc references to avoid HashMap lookups during send/recv.
pub(crate) struct TaskChannels {
/// Channel for Rust -> Worker communication (task_id, data)
worker_channel: Arc<MessageChannel<(u32, Vec<u8>)>>,
/// Channel for Worker -> Rust communication (data)
task_channel: Arc<MessageChannel<Vec<u8>>>,
task_id: u32,
}
impl TaskChannels {
/// Create and register channels for a new task.
/// Channels are inserted into the global maps so JS workers can find them.
pub(crate) fn new(task_id: u32, worker_id: u32) -> Self {
let worker_channel = {
let mut map = WORKER_POOL_OPERATION.worker_routed_channel.lock();
map.entry(worker_id)
.or_insert_with(|| Arc::new(MessageChannel::unbounded()))
.clone()
};
let task_channel = {
let mut map = WORKER_POOL_OPERATION.task_routed_channel.lock();
map.entry(task_id)
.or_insert_with(|| Arc::new(MessageChannel::unbounded()))
.clone()
};
Self {
worker_channel,
task_channel,
task_id,
}
}
/// Send message to worker (Rust -> JS Worker)
pub(crate) async fn send_to_worker(&self, message: Vec<u8>) -> Result<()> {
self.worker_channel
.send((self.task_id, message))
.await
.context("failed to send message to worker")
}
/// Receive message from worker (JS Worker -> Rust)
pub(crate) async fn recv_from_worker(&self) -> Result<Vec<u8>> {
self.task_channel
.recv()
.await
.context("failed to recv task message")
}
}
impl Drop for TaskChannels {
fn drop(&mut self) {
// Only remove task channel, worker channel is shared across tasks
WORKER_POOL_OPERATION
.task_routed_channel
.lock()
.remove(&self.task_id);
}
}
pub(crate) struct WorkerOperation {
pub(crate) worker_options: Arc<WorkerOptions>,
pub(crate) worker_id: u32,
pub(crate) state: Arc<PoolState>,
pub(crate) on_drop: Option<Box<dyn FnOnce(u32) + Send + Sync>>,
pub(crate) _permits: AcquiredPermits,
/// Pre-allocated channels for this task
pub(crate) channels: TaskChannels,
}
impl Drop for WorkerOperation {
fn drop(&mut self) {
if let Some(on_drop) = self.on_drop.take() {
on_drop(self.worker_id);
}
// TaskChannels handles its own cleanup in its Drop impl
}
}
#[async_trait::async_trait]
impl Operation for WorkerOperation {
async fn recv(&mut self) -> Result<Vec<u8>> {
self.channels.recv_from_worker().await
}
async fn send(&mut self, message: Vec<u8>) -> Result<()> {
self.channels.send_to_worker(message).await
}
async fn wait_or_kill(&mut self) -> Result<ExitStatus> {
if self.on_drop.is_some() {
self.state.stats.lock().remove_worker();
self.on_drop = None;
}
terminate_worker(self.worker_options.clone(), self.worker_id)?;
Ok(ExitStatus::default())
}
fn disallow_reuse(&mut self) {
if self.on_drop.is_some() {
self.state.stats.lock().remove_worker();
self.on_drop = None;
}
}
}
use std::{collections::VecDeque, sync::Arc};
use napi::{
Env,
threadsafe_function::{ErrorStrategy, ThreadsafeFunction, ThreadsafeFunctionCallMode},
};
use napi_derive::napi;
use once_cell::sync::OnceCell;
use parking_lot::Mutex;
use tokio::sync::oneshot;
use turbo_rcstr::RcStr;
use crate::worker_pool::{
WorkerOptions,
operation::{TaskMessage, WORKER_POOL_OPERATION},
};
static WORKER_CREATOR: OnceCell<ThreadsafeFunction<NapiWorkerCreation, ErrorStrategy::Fatal>> =
OnceCell::new();
static WORKER_TERMINATOR: OnceCell<
ThreadsafeFunction<NapiWorkerTermination, ErrorStrategy::Fatal>,
> = OnceCell::new();
static PENDING_CREATIONS: OnceCell<Mutex<VecDeque<oneshot::Sender<u32>>>> = OnceCell::new();
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
#[napi]
pub fn register_worker_scheduler(
env: Env,
creator: ThreadsafeFunction<NapiWorkerCreation, ErrorStrategy::Fatal>,
terminator: ThreadsafeFunction<NapiWorkerTermination, ErrorStrategy::Fatal>,
) -> napi::Result<()> {
// Unref ThreadsafeFunction so it doesn't keep the Node.js event loop alive.
// Call unref on the functions before storing them globally.
let creator_unrefed = {
let mut c = creator;
// Safe to call unref; if the napi crate provides this method it will drop the ref
// preventing the ThreadsafeFunction from keeping the loop alive.
let _ = c.unref(&env);
c
};
let terminator_unrefed = {
let mut t = terminator;
let _ = t.unref(&env);
t
};
WORKER_CREATOR
.set(creator_unrefed)
.map_err(|_| napi::Error::from_reason("Worker creator already registered"))?;
WORKER_TERMINATOR
.set(terminator_unrefed)
.map_err(|_| napi::Error::from_reason("Worker terminator already registered"))
}
pub async fn create_worker(options: Arc<WorkerOptions>) -> anyhow::Result<u32> {
let (tx, rx) = oneshot::channel();
let napi_options = (&options).into();
{
let pending = PENDING_CREATIONS.get_or_init(|| Mutex::new(VecDeque::new()));
// ensure pool entry exists for these options so scale ops can observe it
WORKER_POOL_OPERATION
.pools
.lock()
.entry(options.clone())
.or_default();
pending.lock().push_back(tx);
}
if let Some(creator) = WORKER_CREATOR.get() {
creator.call(
NapiWorkerCreation {
options: napi_options,
},
ThreadsafeFunctionCallMode::NonBlocking,
);
} else {
return Err(anyhow::anyhow!("Worker creator not registered"));
}
let worker_id = rx.await?;
Ok(worker_id)
}
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
#[napi]
pub fn worker_created(worker_id: u32) {
if let Some(pending) = PENDING_CREATIONS.get()
&& let Some(tx) = pending.lock().pop_front()
{
let _ = tx.send(worker_id);
}
}
pub fn terminate_worker(options: Arc<WorkerOptions>, worker_id: u32) {
if let Some(terminator) = WORKER_TERMINATOR.get() {
terminator.call(
NapiWorkerTermination {
options: options.into(),
worker_id,
},
ThreadsafeFunctionCallMode::NonBlocking,
);
}
}
#[napi(object)]
pub struct NapiWorkerCreation {
pub options: NapiWorkerOptions,
}
#[napi(object)]
pub struct NapiWorkerOptions {
pub filename: RcStr,
pub cwd: RcStr,
}
impl<T> From<T> for NapiWorkerOptions
where
T: AsRef<WorkerOptions>,
{
fn from(pool_options: T) -> Self {
let WorkerOptions { filename, cwd } = pool_options.as_ref();
NapiWorkerOptions {
filename: filename.clone(),
cwd: cwd.clone(),
}
}
}
#[napi(object)]
pub struct NapiWorkerTermination {
pub options: NapiWorkerOptions,
pub worker_id: u32,
}
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
#[napi(object)]
pub struct NapiTaskMessage {
pub task_id: u32,
pub data: napi::bindgen_prelude::Buffer,
}
impl From<NapiTaskMessage> for TaskMessage {
fn from(message: NapiTaskMessage) -> Self {
let NapiTaskMessage { task_id, data } = message;
TaskMessage {
task_id,
data: data.into(),
}
}
}
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
#[napi]
pub async fn recv_task_message_in_worker(worker_id: u32) -> napi::Result<NapiTaskMessage> {
let (task_id, message) = WORKER_POOL_OPERATION
.recv_task_message_in_worker(worker_id)
.await?;
Ok(NapiTaskMessage {
task_id,
data: message.into(),
})
}
// Allow dead_code for test builds where napi exports are not entry points
#[allow(dead_code)]
#[napi]
pub async fn send_task_message(message: NapiTaskMessage) -> napi::Result<()> {
Ok(WORKER_POOL_OPERATION
.send_task_message(message.into())
.await?)
}
#![allow(unused_imports)]
#![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)]
......@@ -10,41 +11,55 @@ use turbo_rcstr::RcStr;
use turbo_tasks::Vc;
use turbo_tasks_fs::{FileSystemPath, VirtualFileSystem};
use turbo_tasks_testing::{Registration, register, run_once_without_cache_check};
use turbopack_node::{AssetsForSourceMapping, pool::NodeJsPool};
use turbopack_node::{
AssetsForSourceMapping,
evaluate::{EvaluatePool, Operation},
};
static REGISTRATION: Registration = register!();
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
fn test_worker(name: &str) -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(format!("tests/{name}"))
}
/// Create a pool backed by the given test worker JS file.
async fn create_pool(worker: &str, concurrency: usize) -> Result<NodeJsPool> {
let vfs = VirtualFileSystem::new();
let fs: Vc<Box<dyn turbo_tasks_fs::FileSystem>> = Vc::upcast(vfs);
let fs = fs.to_resolved().await?;
let root_path = FileSystemPath {
fs,
path: RcStr::default(),
};
let assets: Vc<AssetsForSourceMapping> = Vc::cell(Default::default());
let assets = assets.to_resolved().await?;
Ok(NodeJsPool::new(
std::env::current_dir()?,
test_worker(worker),
FxHashMap::default(),
assets,
root_path.clone(),
root_path,
concurrency,
false,
))
async fn create_pool(worker: &str, concurrency: usize) -> Result<EvaluatePool> {
#[cfg(feature = "worker_pool")]
{
let _ = (worker, concurrency);
unreachable!("Tests only work with process_pool feature");
}
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
{
use turbopack_node::process_pool::ChildProcessPool;
let vfs = VirtualFileSystem::new();
let fs: Vc<Box<dyn turbo_tasks_fs::FileSystem>> = Vc::upcast(vfs);
let fs = fs.to_resolved().await?;
let root_path = FileSystemPath {
fs,
path: RcStr::default(),
};
let assets: Vc<AssetsForSourceMapping> = Vc::cell(Default::default());
let assets = assets.to_resolved().await?;
Ok(ChildProcessPool::create(
std::env::current_dir()?,
test_worker(worker),
FxHashMap::default(),
assets,
root_path.clone(),
root_path,
concurrency,
false,
))
}
}
/// Shorthand: create a pool with the echo worker.
async fn create_test_pool(concurrency: usize) -> Result<NodeJsPool> {
async fn create_test_pool(concurrency: usize) -> Result<EvaluatePool> {
create_pool("pool_test_worker.js", concurrency).await
}
......@@ -57,9 +72,9 @@ struct Echo {
}
/// Helper: send a message and return the echo response.
async fn send_recv(op: &mut turbopack_node::pool::NodeJsOperation, msg: serde_json::Value) -> Echo {
op.send(msg).await.unwrap();
op.recv().await.unwrap()
async fn send_recv(op: &mut Box<dyn Operation>, msg: serde_json::Value) -> Echo {
op.send(serde_json::to_vec(&msg).unwrap()).await.unwrap();
serde_json::from_slice(&op.recv().await.unwrap()).unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
......@@ -130,6 +145,7 @@ async fn test_pool_process_reuse() {
.await;
}
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_pool_pre_warm() {
run_once_without_cache_check(&REGISTRATION, async {
......@@ -243,6 +259,7 @@ async fn test_pool_concurrent_operations() {
.await;
}
#[cfg(all(feature = "process_pool", not(feature = "worker_pool")))]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_pool_pre_warm_failure() {
run_once_without_cache_check(&REGISTRATION, async {
......
......@@ -14,6 +14,7 @@ workspace = true
[dependencies]
turbopack = { workspace = true }
turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]}
[dev-dependencies]
anyhow = { workspace = true }
......@@ -42,7 +43,7 @@ turbopack-ecmascript-plugins = { workspace = true, features = [
] }
turbopack-ecmascript-runtime = { workspace = true }
turbopack-env = { workspace = true }
turbopack-node = { workspace = true }
turbopack-node = { workspace = true, default-features = false, features = ["process_pool"]}
turbopack-nodejs = { workspace = true, features = ["test"] }
turbopack-resolve = { workspace = true }
turbopack-test-utils = { workspace = true }
......
......@@ -35,7 +35,7 @@ turbopack-css = { workspace = true }
turbopack-ecmascript = { workspace = true }
turbopack-env = { workspace = true }
turbopack-mdx = { workspace = true }
turbopack-node = { workspace = true }
turbopack-node = { workspace = true, default-features = false }
turbopack-resolve = { workspace = true }
turbopack-static = { workspace = true }
turbopack-wasm = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment