Unverified Commit 80cb3447 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

[ci] Add an `TURBO_TASKS_AVAILABLE_PARALLELISM` environment variable that…

[ci] Add an `TURBO_TASKS_AVAILABLE_PARALLELISM` environment variable that overrides `std::thread::available_parallelism` callsites (#93995)

We want to use this to potentially improve efficiency in e2e tests where
we're running many instances of Turbopack at once. By default,
turbo-tasks will try to use every available CPU core.

**This environment variable does not strictly limit Turbopack to using 4
cores, it just restricts the number of main tokio worker threads and
changes how we shard dashmaps.**

This option will not make sense for the vast majority of users, it only
makes sense when you are running *many* instances of Turbopack at once,
and the exact semantics are potentially confusing, as it is not a hard
limit.

## Test Plan

Build vercel-site with this set to 1 and see a little over 100% CPU
usage. Set it to 8 and see around 800% CPU usage.

```
TURBO_TASKS_AVAILABLE_PARALLELISM=1 pnpm next build --experimental-build-mode=compile
```
parent 7c4b0525
...@@ -588,9 +588,10 @@ impl ProjectContainer { ...@@ -588,9 +588,10 @@ impl ProjectContainer {
node_version = options.current_node_js_version.as_str(), node_version = options.current_node_js_version.as_str(),
os = std::env::consts::OS, os = std::env::consts::OS,
arch = std::env::consts::ARCH, arch = std::env::consts::ARCH,
cpu_cores = std::thread::available_parallelism() turbo_tasks_available_parallelism =
.map(|n| n.get()) turbo_tasks::parallel::available_parallelism().map(|n| n.get()).unwrap_or(0),
.unwrap_or(0), std_thread_available_parallelism =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0),
dev = options.dev, dev = options.dev,
env_diff = Empty env_diff = Empty
); );
......
...@@ -74,7 +74,6 @@ fn init() { ...@@ -74,7 +74,6 @@ fn init() {
use std::{ use std::{
cell::RefCell, cell::RefCell,
panic::{set_hook, take_hook}, panic::{set_hook, take_hook},
thread::available_parallelism,
time::{Duration, Instant}, time::{Duration, Instant},
}; };
...@@ -83,7 +82,7 @@ fn init() { ...@@ -83,7 +82,7 @@ fn init() {
} }
use tokio::runtime::Builder; use tokio::runtime::Builder;
use turbo_tasks::panic_hooks::handle_panic; use turbo_tasks::{panic_hooks::handle_panic, parallel::available_parallelism};
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
let prev_hook = take_hook(); let prev_hook = take_hook();
......
...@@ -40,6 +40,7 @@ class TestProfile { ...@@ -40,6 +40,7 @@ class TestProfile {
'NEXT_E2E_TEST_TIMEOUT', 'NEXT_E2E_TEST_TIMEOUT',
'NEXT_TURBOPACK_IO_CONCURRENCY', 'NEXT_TURBOPACK_IO_CONCURRENCY',
'NEXT_TEST_PASSED_FILE', 'NEXT_TEST_PASSED_FILE',
'TURBO_TASKS_AVAILABLE_PARALLELISM',
]) ])
// All key=value pairs identifying this test profile, sorted by key. // All key=value pairs identifying this test profile, sorted by key.
......
...@@ -2,7 +2,6 @@ use std::{ ...@@ -2,7 +2,6 @@ use std::{
cmp::max, cmp::max,
path::PathBuf, path::PathBuf,
sync::Arc, sync::Arc,
thread::available_parallelism,
time::{Duration, Instant, SystemTime}, time::{Duration, Instant, SystemTime},
}; };
...@@ -13,6 +12,7 @@ use turbo_persistence::{ ...@@ -13,6 +12,7 @@ use turbo_persistence::{
}; };
use turbo_tasks::{ use turbo_tasks::{
message_queue::{TimingEvent, TraceEvent}, message_queue::{TimingEvent, TraceEvent},
parallel::available_parallelism,
turbo_tasks, turbo_tasks,
}; };
......
use std::thread::available_parallelism; use turbo_tasks::parallel::available_parallelism;
/// Compute a good number of shards to use for sharded data structures. /// Compute a good number of shards to use for sharded data structures.
/// The number of shards is computed based on the number of worker threads /// The number of shards is computed based on the number of worker threads
......
...@@ -5,11 +5,41 @@ ...@@ -5,11 +5,41 @@
//! //!
//! See also: <https://pwy.io/posts/mimalloc-cigarette/> //! See also: <https://pwy.io/posts/mimalloc-cigarette/>
use std::{
env, io,
num::NonZeroUsize,
sync::{Arc, OnceLock},
thread,
};
use crate::{ use crate::{
scope::scope_and_block, scope::scope_and_block,
util::{Chunk, good_chunk_size, into_chunks}, util::{Chunk, good_chunk_size, into_chunks},
}; };
/// Returns the recommended amount of parallelism for the current process. Typically the number of
/// available CPU cores.
///
/// This wraps [`std::thread::available_parallelism`] with a couple extras:
///
/// - If the `TURBO_TASKS_AVAILABLE_PARALLELISM` env var is set, overrides the value. Panics if this
/// env var fails to parse.
/// - The resolved value is cached in a [`OnceLock`]
pub fn available_parallelism() -> Result<NonZeroUsize, Arc<io::Error>> {
static CACHED: OnceLock<Result<NonZeroUsize, Arc<io::Error>>> = OnceLock::new();
CACHED
.get_or_init(|| {
if let Ok(raw) = env::var("TURBO_TASKS_AVAILABLE_PARALLELISM") {
Ok(raw.parse::<NonZeroUsize>().unwrap_or_else(|err| {
panic!("Invalid TURBO_TASKS_AVAILABLE_PARALLELISM={raw:?}: {err}")
}))
} else {
thread::available_parallelism().map_err(Arc::new)
}
})
.clone()
}
struct Chunked { struct Chunked {
chunk_size: usize, chunk_size: usize,
chunk_count: usize, chunk_count: usize,
......
...@@ -9,7 +9,7 @@ use std::{ ...@@ -9,7 +9,7 @@ use std::{
Arc, LazyLock, Arc, LazyLock,
atomic::{AtomicUsize, Ordering}, atomic::{AtomicUsize, Ordering},
}, },
thread::{self, Thread, available_parallelism}, thread::{self, Thread},
time::{Duration, Instant}, time::{Duration, Instant},
}; };
...@@ -17,7 +17,9 @@ use parking_lot::{Condvar, Mutex}; ...@@ -17,7 +17,9 @@ use parking_lot::{Condvar, Mutex};
use tokio::{runtime::Handle, task::block_in_place}; use tokio::{runtime::Handle, task::block_in_place};
use tracing::{Span, info_span}; use tracing::{Span, info_span};
use crate::{TurboTasksApi, manager::try_turbo_tasks, turbo_tasks_scope}; use crate::{
TurboTasksApi, manager::try_turbo_tasks, parallel::available_parallelism, turbo_tasks_scope,
};
/// Number of worker tasks to spawn that process jobs. It's 1 less than the number of cpus as we /// Number of worker tasks to spawn that process jobs. It's 1 less than the number of cpus as we
/// also use the current task as worker. /// also use the current task as worker.
......
...@@ -9,7 +9,6 @@ use std::{ ...@@ -9,7 +9,6 @@ use std::{
pin::Pin, pin::Pin,
sync::{Arc, LazyLock}, sync::{Arc, LazyLock},
task::{Context, Poll}, task::{Context, Poll},
thread::available_parallelism,
time::Duration, time::Duration,
}; };
...@@ -22,6 +21,7 @@ use bincode::{ ...@@ -22,6 +21,7 @@ use bincode::{
}; };
use pin_project_lite::pin_project; use pin_project_lite::pin_project;
use crate::parallel::available_parallelism;
pub use crate::{ pub use crate::{
id_factory::{IdFactory, IdFactoryWithReuse}, id_factory::{IdFactory, IdFactoryWithReuse},
once_map::*, once_map::*,
......
use std::{cell::RefCell, path::Path, thread::available_parallelism, time::Instant}; use std::{cell::RefCell, path::Path, time::Instant};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use clap::Parser; use clap::Parser;
use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt}; use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt};
use turbo_tasks::parallel::available_parallelism;
use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_malloc::TurboMalloc;
use turbopack_cli::arguments::Arguments; use turbopack_cli::arguments::Arguments;
use turbopack_trace_utils::{ use turbopack_trace_utils::{
......
use std::{iter, process::ExitStatus, sync::Arc, thread::available_parallelism, time::Duration}; use std::{iter, process::ExitStatus, sync::Arc, time::Duration};
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use async_trait::async_trait; use async_trait::async_trait;
...@@ -11,7 +11,7 @@ use turbo_rcstr::{RcStr, rcstr}; ...@@ -11,7 +11,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
Completion, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ResolvedVc, TaskInput, Completion, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ResolvedVc, TaskInput,
TryJoinIterExt, ValueToString, Vc, duration_span, fxindexmap, mark_top_level_task, TryJoinIterExt, ValueToString, Vc, duration_span, fxindexmap, mark_top_level_task,
take_effects, trace::TraceRawVcs, parallel::available_parallelism, take_effects, trace::TraceRawVcs,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path}; use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment