Unverified Commit e21ac64c authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: use parallel execution helpers (#82667)

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the PR.
- Read the Docs Contribution Guide to ensure your contribution follows the docs guidelines: https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to understand the PR)
- When linking to a Slack thread, you might want to share details of the conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->
parent 532018c8
...@@ -9253,7 +9253,6 @@ dependencies = [ ...@@ -9253,7 +9253,6 @@ dependencies = [
"parking_lot", "parking_lot",
"pot", "pot",
"rand 0.9.0", "rand 0.9.0",
"rayon",
"regex", "regex",
"ringmap", "ringmap",
"rstest", "rstest",
...@@ -9367,7 +9366,6 @@ dependencies = [ ...@@ -9367,7 +9366,6 @@ dependencies = [
"mime", "mime",
"notify", "notify",
"parking_lot", "parking_lot",
"rayon",
"regex", "regex",
"rstest", "rstest",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
......
...@@ -40,7 +40,6 @@ once_cell = { workspace = true } ...@@ -40,7 +40,6 @@ once_cell = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
pot = "3.0.0" pot = "3.0.0"
rand = { workspace = true } rand = { workspace = true }
rayon = { workspace = true }
ringmap = { workspace = true, features = ["serde"] } ringmap = { workspace = true, features = ["serde"] }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
......
...@@ -6,9 +6,8 @@ use std::{ ...@@ -6,9 +6,8 @@ use std::{
}; };
use bitfield::bitfield; use bitfield::bitfield;
use rayon::iter::{IndexedParallelIterator, IntoParallelRefIterator, ParallelIterator};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_tasks::{FxDashMap, TaskId}; use turbo_tasks::{FxDashMap, TaskId, parallel};
use crate::{ use crate::{
backend::dynamic_storage::DynamicStorage, backend::dynamic_storage::DynamicStorage,
...@@ -664,11 +663,7 @@ impl Storage { ...@@ -664,11 +663,7 @@ impl Storage {
// The number of shards is much larger than the number of threads, so the effect of the // The number of shards is much larger than the number of threads, so the effect of the
// locks held is negligible. // locks held is negligible.
self.modified parallel::map_collect::<_, _, Vec<_>>(self.modified.shards(), |shard| {
.shards()
.par_iter()
.with_max_len(1)
.map(|shard| {
let mut direct_snapshots: Vec<(TaskId, Box<InnerStorageSnapshot>)> = Vec::new(); let mut direct_snapshots: Vec<(TaskId, Box<InnerStorageSnapshot>)> = Vec::new();
let mut modified: SmallVec<[TaskId; 4]> = SmallVec::new(); let mut modified: SmallVec<[TaskId; 4]> = SmallVec::new();
{ {
...@@ -705,7 +700,6 @@ impl Storage { ...@@ -705,7 +700,6 @@ impl Storage {
process_snapshot, process_snapshot,
} }
}) })
.collect::<Vec<_>>()
} }
/// Start snapshot mode. /// Start snapshot mode.
......
use std::{ use std::{
borrow::Borrow, borrow::Borrow,
cmp::max,
env, env,
path::PathBuf, path::PathBuf,
sync::{Arc, LazyLock, Mutex, PoisonError, Weak}, sync::{Arc, LazyLock, Mutex, PoisonError, Weak},
}; };
use anyhow::{Context, Result, anyhow}; use anyhow::{Context, Result, anyhow};
use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use smallvec::SmallVec; use smallvec::SmallVec;
use tracing::Span;
use turbo_tasks::{ use turbo_tasks::{
SessionId, TaskId, SessionId, TaskId,
backend::CachedTaskType, backend::CachedTaskType,
panic_hooks::{PanicHookGuard, register_panic_hook}, panic_hooks::{PanicHookGuard, register_panic_hook},
turbo_tasks_scope, parallel,
}; };
use crate::{ use crate::{
...@@ -331,14 +328,15 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -331,14 +328,15 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
let _span = tracing::trace_span!("update task data").entered(); let _span = tracing::trace_span!("update task data").entered();
process_task_data(snapshots, Some(batch))?; process_task_data(snapshots, Some(batch))?;
let span = tracing::trace_span!("flush task data").entered(); let span = tracing::trace_span!("flush task data").entered();
[KeySpace::TaskMeta, KeySpace::TaskData] parallel::try_for_each(
.into_par_iter() &[KeySpace::TaskMeta, KeySpace::TaskData],
.try_for_each(|key_space| { |&key_space| {
let _span = span.clone().entered(); let _span = span.clone().entered();
// Safety: We already finished all processing of the task data and task // Safety: We already finished all processing of the task data and task
// meta // meta
unsafe { batch.flush(key_space) } unsafe { batch.flush(key_space) }
})?; },
)?;
} }
let mut next_task_id = get_next_free_task_id::< let mut next_task_id = get_next_free_task_id::<
...@@ -352,10 +350,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -352,10 +350,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>() items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>()
) )
.entered(); .entered();
let result = task_cache_updates let result = parallel::map_collect_owned::<_, _, Result<Vec<_>>>(
.into_par_iter() task_cache_updates,
.with_max_len(1) |updates| {
.map(|updates| {
let _span = _span.clone().entered(); let _span = _span.clone().entered();
let mut max_task_id = 0; let mut max_task_id = 0;
...@@ -390,15 +387,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed ...@@ -390,15 +387,11 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorageSealed
} }
Ok(max_task_id) Ok(max_task_id)
})
.reduce(
|| Ok(0),
|a, b| -> anyhow::Result<_> {
let a_max = a?;
let b_max = b?;
Ok(max(a_max, b_max))
}, },
)?; )?
.into_iter()
.max()
.unwrap_or(0);
next_task_id = next_task_id.max(result); next_task_id = next_task_id.max(result);
} }
...@@ -697,15 +690,7 @@ where ...@@ -697,15 +690,7 @@ where
> + Send > + Send
+ Sync, + Sync,
{ {
let span = Span::current(); parallel::map_collect_owned::<_, _, Result<Vec<_>>>(tasks, |tasks| {
let turbo_tasks = turbo_tasks::turbo_tasks();
let handle = tokio::runtime::Handle::current();
tasks
.into_par_iter()
.map(|tasks| {
let _span = span.clone().entered();
let _guard = handle.clone().enter();
turbo_tasks_scope(turbo_tasks.clone(), || {
let mut result = Vec::new(); let mut result = Vec::new();
for (task_id, meta, data) in tasks { for (task_id, meta, data) in tasks {
if let Some(batch) = batch { if let Some(batch) = batch {
...@@ -737,8 +722,6 @@ where ...@@ -737,8 +722,6 @@ where
Ok(result) Ok(result)
}) })
})
.collect::<Result<Vec<_>>>()
} }
fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>> { fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 16]>> {
......
...@@ -37,7 +37,6 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] } ...@@ -37,7 +37,6 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] }
mime = { workspace = true } mime = { workspace = true }
notify = { workspace = true } notify = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
rayon = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
......
...@@ -46,7 +46,6 @@ use dunce::simplified; ...@@ -46,7 +46,6 @@ use dunce::simplified;
use indexmap::IndexSet; use indexmap::IndexSet;
use jsonc_parser::{ParseOptions, parse_to_serde_value}; use jsonc_parser::{ParseOptions, parse_to_serde_value};
use mime::Mime; use mime::Mime;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
...@@ -56,7 +55,7 @@ use turbo_rcstr::{RcStr, rcstr}; ...@@ -56,7 +55,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
ApplyEffectsContext, Completion, InvalidationReason, Invalidator, NonLocalValue, ReadRef, ApplyEffectsContext, Completion, InvalidationReason, Invalidator, NonLocalValue, ReadRef,
ResolvedVc, TaskInput, ValueToString, Vc, debug::ValueDebugFormat, effect, ResolvedVc, TaskInput, ValueToString, Vc, debug::ValueDebugFormat, effect,
mark_session_dependent, mark_stateful, trace::TraceRawVcs, mark_session_dependent, mark_stateful, parallel, trace::TraceRawVcs,
}; };
use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64}; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, hash_xxh3_hash64};
use turbo_unix_path::{ use turbo_unix_path::{
...@@ -309,19 +308,14 @@ impl DiskFileSystemInner { ...@@ -309,19 +308,14 @@ impl DiskFileSystemInner {
fn invalidate(&self) { fn invalidate(&self) {
let _span = tracing::info_span!("invalidate filesystem", name = &*self.root).entered(); let _span = tracing::info_span!("invalidate filesystem", name = &*self.root).entered();
let span = tracing::Span::current();
let handle = tokio::runtime::Handle::current();
let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap()); let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());
let iter = invalidator_map let invalidators = invalidator_map
.into_par_iter() .into_iter()
.chain(dir_invalidator_map.into_par_iter()) .chain(dir_invalidator_map)
.flat_map(|(_, invalidators)| invalidators.into_par_iter()); .flat_map(|(_, invalidators)| invalidators.into_keys())
iter.for_each(|(i, _)| { .collect::<Vec<_>>();
let _span = span.clone().entered(); parallel::for_each_owned(invalidators, |invalidator| invalidator.invalidate());
let _guard = handle.enter();
i.invalidate()
});
} }
/// Invalidates every tracked file in the filesystem. /// Invalidates every tracked file in the filesystem.
...@@ -332,23 +326,19 @@ impl DiskFileSystemInner { ...@@ -332,23 +326,19 @@ impl DiskFileSystemInner {
reason: impl Fn(&Path) -> R + Sync, reason: impl Fn(&Path) -> R + Sync,
) { ) {
let _span = tracing::info_span!("invalidate filesystem", name = &*self.root).entered(); let _span = tracing::info_span!("invalidate filesystem", name = &*self.root).entered();
let span = tracing::Span::current();
let handle = tokio::runtime::Handle::current();
let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap()); let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());
let iter = invalidator_map let invalidators = invalidator_map
.into_par_iter() .into_iter()
.chain(dir_invalidator_map.into_par_iter()) .chain(dir_invalidator_map)
.flat_map(|(path, invalidators)| { .flat_map(|(path, invalidators)| {
let _span = span.clone().entered();
let reason_for_path = reason(&path); let reason_for_path = reason(&path);
invalidators invalidators
.into_par_iter() .into_keys()
.map(move |i| (reason_for_path.clone(), i)) .map(move |i| (reason_for_path.clone(), i))
}); })
iter.for_each(|(reason, (invalidator, _))| { .collect::<Vec<_>>();
let _span = span.clone().entered(); parallel::for_each_owned(invalidators, |(reason, invalidator)| {
let _guard = handle.enter();
invalidator.invalidate_with_reason(reason) invalidator.invalidate_with_reason(reason)
}); });
} }
......
...@@ -16,13 +16,12 @@ use notify::{ ...@@ -16,13 +16,12 @@ use notify::{
Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher, Config, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher,
event::{MetadataKind, ModifyKind, RenameMode}, event::{MetadataKind, ModifyKind, RenameMode},
}; };
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::instrument; use tracing::instrument;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{ use turbo_tasks::{
FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, spawn_thread, FxIndexSet, InvalidationReason, InvalidationReasonKind, Invalidator, parallel, spawn_thread,
util::StaticOrArc, util::StaticOrArc,
}; };
...@@ -397,40 +396,30 @@ impl DiskWatcher { ...@@ -397,40 +396,30 @@ impl DiskWatcher {
// //
// Best is to start_watching before starting to read // Best is to start_watching before starting to read
{ {
let span = tracing::info_span!("invalidate filesystem"); let _span = tracing::info_span!("invalidate filesystem").entered();
let _span = span.clone().entered();
let invalidator_map = take(&mut *fs_inner.invalidator_map.lock().unwrap()); let invalidator_map = take(&mut *fs_inner.invalidator_map.lock().unwrap());
let dir_invalidator_map = take(&mut *fs_inner.dir_invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *fs_inner.dir_invalidator_map.lock().unwrap());
let iter = invalidator_map let iter = invalidator_map.into_iter().chain(dir_invalidator_map);
.into_par_iter()
.chain(dir_invalidator_map.into_par_iter());
let handle = tokio::runtime::Handle::current();
if report_invalidation_reason { if report_invalidation_reason {
iter.flat_map(|(path, invalidators)| { let invalidators = iter
let _span = span.clone().entered(); .flat_map(|(path, invalidators)| {
let reason = WatchStart { let reason = WatchStart {
name: fs_inner.name.clone(), name: fs_inner.name.clone(),
// this path is just used for display purposes // this path is just used for display purposes
path: RcStr::from(path.to_string_lossy()), path: RcStr::from(path.to_string_lossy()),
}; };
invalidators invalidators.into_iter().map(move |i| (reason.clone(), i))
.into_par_iter()
.map(move |i| (reason.clone(), i))
}) })
.for_each(|(reason, (invalidator, _))| { .collect::<Vec<_>>();
let _span = span.clone().entered(); parallel::for_each_owned(invalidators, |(reason, (invalidator, _))| {
let _guard = handle.enter(); invalidator.invalidate_with_reason(reason);
invalidator.invalidate_with_reason(reason)
}); });
} else { } else {
iter.flat_map(|(_, invalidators)| { let invalidators = iter
let _span = span.clone().entered(); .flat_map(|(_, invalidators)| invalidators.into_keys())
invalidators.into_par_iter().map(move |i| i) .collect::<Vec<_>>();
}) parallel::for_each_owned(invalidators, |invalidator| {
.for_each(|(invalidator, _)| { invalidator.invalidate();
let _span = span.clone().entered();
let _guard = handle.enter();
invalidator.invalidate()
}); });
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment