Unverified Commit 80cb3447 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

[ci] Add an `TURBO_TASKS_AVAILABLE_PARALLELISM` environment variable that…

[ci] Add an `TURBO_TASKS_AVAILABLE_PARALLELISM` environment variable that overrides `std::thread::available_parallelism` callsites (#93995)

We want to use this to potentially improve efficiency in e2e tests where
we're running many instances of Turbopack at once. By default,
turbo-tasks will try to use every available CPU core.

**This environment variable does not strictly limit Turbopack to using 4
cores, it just restricts the number of main tokio worker threads and
changes how we shard dashmaps.**

This option will not make sense for the vast majority of users, it only
makes sense when you are running *many* instances of Turbopack at once,
and the exact semantics are potentially confusing, as it is not a hard
limit.

## Test Plan

Build vercel-site with this set to 1 and see a little over 100% CPU
usage. Set it to 8 and see around 800% CPU usage.

```
TURBO_TASKS_AVAILABLE_PARALLELISM=1 pnpm next build --experimental-build-mode=compile
```
parent 7c4b0525
......@@ -588,9 +588,10 @@ impl ProjectContainer {
node_version = options.current_node_js_version.as_str(),
os = std::env::consts::OS,
arch = std::env::consts::ARCH,
cpu_cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(0),
turbo_tasks_available_parallelism =
turbo_tasks::parallel::available_parallelism().map(|n| n.get()).unwrap_or(0),
std_thread_available_parallelism =
std::thread::available_parallelism().map(|n| n.get()).unwrap_or(0),
dev = options.dev,
env_diff = Empty
);
......
......@@ -74,7 +74,6 @@ fn init() {
use std::{
cell::RefCell,
panic::{set_hook, take_hook},
thread::available_parallelism,
time::{Duration, Instant},
};
......@@ -83,7 +82,7 @@ fn init() {
}
use tokio::runtime::Builder;
use turbo_tasks::panic_hooks::handle_panic;
use turbo_tasks::{panic_hooks::handle_panic, parallel::available_parallelism};
use turbo_tasks_malloc::TurboMalloc;
let prev_hook = take_hook();
......
......@@ -40,6 +40,7 @@ class TestProfile {
'NEXT_E2E_TEST_TIMEOUT',
'NEXT_TURBOPACK_IO_CONCURRENCY',
'NEXT_TEST_PASSED_FILE',
'TURBO_TASKS_AVAILABLE_PARALLELISM',
])
// All key=value pairs identifying this test profile, sorted by key.
......
......@@ -2,7 +2,6 @@ use std::{
cmp::max,
path::PathBuf,
sync::Arc,
thread::available_parallelism,
time::{Duration, Instant, SystemTime},
};
......@@ -13,6 +12,7 @@ use turbo_persistence::{
};
use turbo_tasks::{
message_queue::{TimingEvent, TraceEvent},
parallel::available_parallelism,
turbo_tasks,
};
......
use std::thread::available_parallelism;
use turbo_tasks::parallel::available_parallelism;
/// Compute a good number of shards to use for sharded data structures.
/// The number of shards is computed based on the number of worker threads
......
......@@ -5,11 +5,41 @@
//!
//! See also: <https://pwy.io/posts/mimalloc-cigarette/>
use std::{
env, io,
num::NonZeroUsize,
sync::{Arc, OnceLock},
thread,
};
use crate::{
scope::scope_and_block,
util::{Chunk, good_chunk_size, into_chunks},
};
/// Returns the recommended amount of parallelism for the current process. Typically the number of
/// available CPU cores.
///
/// This wraps [`std::thread::available_parallelism`] with a couple extras:
///
/// - If the `TURBO_TASKS_AVAILABLE_PARALLELISM` env var is set, overrides the value. Panics if this
/// env var fails to parse.
/// - The resolved value is cached in a [`OnceLock`]
pub fn available_parallelism() -> Result<NonZeroUsize, Arc<io::Error>> {
static CACHED: OnceLock<Result<NonZeroUsize, Arc<io::Error>>> = OnceLock::new();
CACHED
.get_or_init(|| {
if let Ok(raw) = env::var("TURBO_TASKS_AVAILABLE_PARALLELISM") {
Ok(raw.parse::<NonZeroUsize>().unwrap_or_else(|err| {
panic!("Invalid TURBO_TASKS_AVAILABLE_PARALLELISM={raw:?}: {err}")
}))
} else {
thread::available_parallelism().map_err(Arc::new)
}
})
.clone()
}
struct Chunked {
chunk_size: usize,
chunk_count: usize,
......
......@@ -9,7 +9,7 @@ use std::{
Arc, LazyLock,
atomic::{AtomicUsize, Ordering},
},
thread::{self, Thread, available_parallelism},
thread::{self, Thread},
time::{Duration, Instant},
};
......@@ -17,7 +17,9 @@ use parking_lot::{Condvar, Mutex};
use tokio::{runtime::Handle, task::block_in_place};
use tracing::{Span, info_span};
use crate::{TurboTasksApi, manager::try_turbo_tasks, turbo_tasks_scope};
use crate::{
TurboTasksApi, manager::try_turbo_tasks, parallel::available_parallelism, turbo_tasks_scope,
};
/// Number of worker tasks to spawn that process jobs. It's 1 less than the number of cpus as we
/// also use the current task as worker.
......
......@@ -9,7 +9,6 @@ use std::{
pin::Pin,
sync::{Arc, LazyLock},
task::{Context, Poll},
thread::available_parallelism,
time::Duration,
};
......@@ -22,6 +21,7 @@ use bincode::{
};
use pin_project_lite::pin_project;
use crate::parallel::available_parallelism;
pub use crate::{
id_factory::{IdFactory, IdFactoryWithReuse},
once_map::*,
......
use std::{cell::RefCell, path::Path, thread::available_parallelism, time::Instant};
use std::{cell::RefCell, path::Path, time::Instant};
use anyhow::{Context, Result};
use clap::Parser;
use tracing_subscriber::{Registry, layer::SubscriberExt, util::SubscriberInitExt};
use turbo_tasks::parallel::available_parallelism;
use turbo_tasks_malloc::TurboMalloc;
use turbopack_cli::arguments::Arguments;
use turbopack_trace_utils::{
......
use std::{iter, process::ExitStatus, sync::Arc, thread::available_parallelism, time::Duration};
use std::{iter, process::ExitStatus, sync::Arc, time::Duration};
use anyhow::{Result, bail};
use async_trait::async_trait;
......@@ -11,7 +11,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
Completion, FxIndexMap, NonLocalValue, OperationVc, PrettyPrintError, ResolvedVc, TaskInput,
TryJoinIterExt, ValueToString, Vc, duration_span, fxindexmap, mark_top_level_task,
take_effects, trace::TraceRawVcs,
parallel::available_parallelism, take_effects, trace::TraceRawVcs,
};
use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fs::{File, FileContent, FileSystemPath, to_sys_path};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment