Unverified Commit 36458f90 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: refactor idle process queue and fix bugs and race conditions (#82630)

### What?

Fixes a bug that breaks the pool after scaling down

Closes PACK-5249
parent df9f3ba4
use std::{collections::BinaryHeap, sync::Arc};
use parking_lot::Mutex;
use tokio::sync::{AcquireError, Semaphore};
pub struct HeapQueue<T> {
heap: Mutex<BinaryHeap<T>>,
semaphore: Semaphore,
}
impl<T: Ord> HeapQueue<T> {
pub fn new() -> Self {
Self {
heap: Mutex::new(BinaryHeap::new()),
semaphore: Semaphore::new(0),
}
}
pub fn push(self: &Arc<Self>, item: T, active_queues: &Mutex<Vec<Arc<Self>>>) {
{
let mut heap = self.heap.lock();
if heap.is_empty() {
// If the heap was empty, add this queue to the active queues
let mut queues = active_queues.lock();
queues.push(self.clone());
}
heap.push(item);
}
self.semaphore.add_permits(1);
}
pub async fn pop(
self: &Arc<Self>,
active_queues: &Mutex<Vec<Arc<Self>>>,
) -> Result<T, AcquireError> {
self.semaphore.acquire().await?.forget();
let mut heap = self.heap.lock();
let item = heap.pop().unwrap();
if heap.is_empty() {
// If the heap is empty, remove this queue from the active queues
let mut queues = active_queues.lock();
if let Some(pos) = queues.iter().position(|q| Arc::ptr_eq(q, self)) {
queues.remove(pos);
}
}
Ok(item)
}
pub fn reduce_to_one(&self) {
// Drain the semaphore permits
let mut n = self.semaphore.forget_permits(usize::MAX);
if n <= 1 {
self.semaphore.add_permits(n);
return;
}
let mut heap: parking_lot::lock_api::MutexGuard<'_, parking_lot::RawMutex, BinaryHeap<T>> =
self.heap.lock();
// We must only pop n items even if there are more since we only have n permits
let top = heap.pop().unwrap();
n -= 1;
for _ in 0..n {
heap.pop();
}
heap.push(top);
self.semaphore.add_permits(1);
}
pub fn reduce_to_zero(self: &Arc<Self>, active_queues: &Mutex<Vec<Arc<Self>>>) {
// Drain the semaphore permits
let n = self.semaphore.forget_permits(usize::MAX);
if n == 0 {
return;
}
let mut heap = self.heap.lock();
// We must only pop n items even if there are more since we only have n permits
for _ in 0..n {
heap.pop();
}
if heap.is_empty() {
// If the heap is empty, remove this queue from the active queues
let mut queues = active_queues.lock();
if let Some(pos) = queues.iter().position(|q| Arc::ptr_eq(q, self)) {
queues.remove(pos);
}
}
}
}
...@@ -31,6 +31,7 @@ pub mod debug; ...@@ -31,6 +31,7 @@ pub mod debug;
pub mod embed_js; pub mod embed_js;
pub mod evaluate; pub mod evaluate;
pub mod execution_context; pub mod execution_context;
mod heap_queue;
mod node_entry; mod node_entry;
mod pool; mod pool;
pub mod render; pub mod render;
......
use std::{ use std::{
borrow::Cow, borrow::Cow,
cmp::max, cmp::max,
collections::BinaryHeap,
fmt::{Debug, Display}, fmt::{Debug, Display},
future::Future, future::Future,
mem::take, mem::take,
...@@ -34,7 +33,7 @@ use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span}; ...@@ -34,7 +33,7 @@ use turbo_tasks::{FxIndexSet, ResolvedVc, Vc, duration_span};
use turbo_tasks_fs::{FileSystemPath, json::parse_json_with_source_context}; use turbo_tasks_fs::{FileSystemPath, json::parse_json_with_source_context};
use turbopack_ecmascript::magic_identifier::unmangle_identifiers; use turbopack_ecmascript::magic_identifier::unmangle_identifiers;
use crate::{AssetsForSourceMapping, source_map::apply_source_mapping}; use crate::{AssetsForSourceMapping, heap_queue::HeapQueue, source_map::apply_source_mapping};
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
pub enum FormattingMode { pub enum FormattingMode {
...@@ -706,11 +705,11 @@ enum AcquiredPermits { ...@@ -706,11 +705,11 @@ enum AcquiredPermits {
}, },
} }
type IdleProcessesList = Arc<Mutex<BinaryHeap<NodeJsPoolProcess>>>; type IdleProcessQueues = Mutex<Vec<Arc<HeapQueue<NodeJsPoolProcess>>>>;
/// All non-empty `IdleProcessesList`s of the whole application. /// All non-empty `IdleProcessQueues`s of the whole application.
/// This is used to scale down processes globally. /// This is used to scale down processes globally.
static ACTIVE_POOLS: Lazy<Mutex<Vec<IdleProcessesList>>> = Lazy::new(Default::default); static ACTIVE_POOLS: Lazy<IdleProcessQueues> = Lazy::new(Default::default);
/// A pool of Node.js workers operating on [entrypoint] with specific [cwd] and /// A pool of Node.js workers operating on [entrypoint] with specific [cwd] and
/// [env]. /// [env].
...@@ -730,7 +729,7 @@ pub struct NodeJsPool { ...@@ -730,7 +729,7 @@ pub struct NodeJsPool {
pub assets_root: FileSystemPath, pub assets_root: FileSystemPath,
pub project_dir: FileSystemPath, pub project_dir: FileSystemPath,
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(trace_ignore, debug_ignore)]
processes: Arc<Mutex<BinaryHeap<NodeJsPoolProcess>>>, idle_processes: Arc<HeapQueue<NodeJsPoolProcess>>,
/// Semaphore to limit the number of concurrent operations in general /// Semaphore to limit the number of concurrent operations in general
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(trace_ignore, debug_ignore)]
concurrency_semaphore: Arc<Semaphore>, concurrency_semaphore: Arc<Semaphore>,
...@@ -738,9 +737,6 @@ pub struct NodeJsPool { ...@@ -738,9 +737,6 @@ pub struct NodeJsPool {
/// (excludes one-off processes) /// (excludes one-off processes)
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(trace_ignore, debug_ignore)]
bootup_semaphore: Arc<Semaphore>, bootup_semaphore: Arc<Semaphore>,
/// Semaphore to wait for an idle process to become available
#[turbo_tasks(trace_ignore, debug_ignore)]
idle_process_semaphore: Arc<Semaphore>,
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(trace_ignore, debug_ignore)]
shared_stdout: SharedOutputSet, shared_stdout: SharedOutputSet,
#[turbo_tasks(trace_ignore, debug_ignore)] #[turbo_tasks(trace_ignore, debug_ignore)]
...@@ -770,10 +766,9 @@ impl NodeJsPool { ...@@ -770,10 +766,9 @@ impl NodeJsPool {
assets_for_source_mapping, assets_for_source_mapping,
assets_root, assets_root,
project_dir, project_dir,
processes: Arc::new(Mutex::new(BinaryHeap::new())),
concurrency_semaphore: Arc::new(Semaphore::new(if debug { 1 } else { concurrency })), concurrency_semaphore: Arc::new(Semaphore::new(if debug { 1 } else { concurrency })),
bootup_semaphore: Arc::new(Semaphore::new(1)), bootup_semaphore: Arc::new(Semaphore::new(1)),
idle_process_semaphore: Arc::new(Semaphore::new(0)), idle_processes: Arc::new(HeapQueue::new()),
shared_stdout: Arc::new(Mutex::new(FxIndexSet::default())), shared_stdout: Arc::new(Mutex::new(FxIndexSet::default())),
shared_stderr: Arc::new(Mutex::new(FxIndexSet::default())), shared_stderr: Arc::new(Mutex::new(FxIndexSet::default())),
debug, debug,
...@@ -796,20 +791,8 @@ impl NodeJsPool { ...@@ -796,20 +791,8 @@ impl NodeJsPool {
}; };
select! { select! {
idle_process_permit = self.idle_process_semaphore.clone().acquire_owned() => { idle_process_result = self.idle_processes.pop(&ACTIVE_POOLS) => {
let idle_process_permit = idle_process_permit.context("acquiring idle process permit")?; let process = idle_process_result.context("acquiring idle process permit")?;
let process = {
let mut processes = self.processes.lock();
let process = processes.pop().unwrap();
if processes.is_empty() {
let mut pools = ACTIVE_POOLS.lock();
if let Some(idx) = pools.iter().position(|p| Arc::ptr_eq(p, &self.processes)) {
pools.swap_remove(idx);
}
}
process
};
idle_process_permit.forget();
Ok((process, AcquiredPermits::Idle { concurrency_permit })) Ok((process, AcquiredPermits::Idle { concurrency_permit }))
}, },
bootup_permit = bootup => { bootup_permit = bootup => {
...@@ -856,8 +839,7 @@ impl NodeJsPool { ...@@ -856,8 +839,7 @@ impl NodeJsPool {
Ok(NodeJsOperation { Ok(NodeJsOperation {
process: Some(process), process: Some(process),
permits, permits,
processes: self.processes.clone(), idle_processes: self.idle_processes.clone(),
idle_process_semaphore: self.idle_process_semaphore.clone(),
start: Instant::now(), start: Instant::now(),
stats: self.stats.clone(), stats: self.stats.clone(),
allow_process_reuse: true, allow_process_reuse: true,
...@@ -867,20 +849,14 @@ impl NodeJsPool { ...@@ -867,20 +849,14 @@ impl NodeJsPool {
pub fn scale_down() { pub fn scale_down() {
let pools = ACTIVE_POOLS.lock().clone(); let pools = ACTIVE_POOLS.lock().clone();
for pool in pools { for pool in pools {
let mut pool = pool.lock(); pool.reduce_to_one();
let best = pool.pop().unwrap();
pool.clear();
pool.push(best);
pool.shrink_to_fit();
} }
} }
pub fn scale_zero() { pub fn scale_zero() {
let pools = take(&mut *ACTIVE_POOLS.lock()); let pools = ACTIVE_POOLS.lock().clone();
for pool in pools { for pool in pools {
let mut pool = pool.lock(); pool.reduce_to_zero(&ACTIVE_POOLS);
pool.clear();
pool.shrink_to_fit();
} }
} }
} }
...@@ -890,8 +866,7 @@ pub struct NodeJsOperation { ...@@ -890,8 +866,7 @@ pub struct NodeJsOperation {
// This is used for drop // This is used for drop
#[allow(dead_code)] #[allow(dead_code)]
permits: AcquiredPermits, permits: AcquiredPermits,
processes: Arc<Mutex<BinaryHeap<NodeJsPoolProcess>>>, idle_processes: Arc<HeapQueue<NodeJsPoolProcess>>,
idle_process_semaphore: Arc<Semaphore>,
start: Instant, start: Instant,
stats: Arc<Mutex<NodeJsPoolStats>>, stats: Arc<Mutex<NodeJsPoolStats>>,
allow_process_reuse: bool, allow_process_reuse: bool,
...@@ -1005,14 +980,7 @@ impl Drop for NodeJsOperation { ...@@ -1005,14 +980,7 @@ impl Drop for NodeJsOperation {
} }
if self.allow_process_reuse { if self.allow_process_reuse {
process.cpu_time_invested += elapsed; process.cpu_time_invested += elapsed;
{ self.idle_processes.push(process, &ACTIVE_POOLS);
let mut processes = self.processes.lock();
if processes.is_empty() {
ACTIVE_POOLS.lock().push(self.processes.clone());
}
processes.push(process);
}
self.idle_process_semaphore.add_permits(1);
} }
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment