Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
0eaca9c7
Unverified
Commit
0eaca9c7
authored
Oct 08, 2026
by
vercel[bot]
Committed by
GitHub
Oct 08, 2026
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Reduce Turbopack event delivery and filesystem lock contention
Co-authored-by:
Jimmy Miller
<
james.miller@vercel.com
>
parent
7549879a
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
291 additions
and
39 deletions
+291
-39
mod.rs
turbopack/crates/turbo-tasks-fs/benches/mod.rs
+3
-1
path_locks.rs
turbopack/crates/turbo-tasks-fs/benches/path_locks.rs
+43
-0
mutex_map.rs
turbopack/crates/turbo-tasks-fs/src/mutex_map.rs
+54
-11
message_queue.rs
turbopack/crates/turbo-tasks/benches/message_queue.rs
+38
-0
mod.rs
turbopack/crates/turbo-tasks/benches/mod.rs
+2
-1
message_queue.rs
turbopack/crates/turbo-tasks/src/message_queue.rs
+151
-26
No files found.
turbopack/crates/turbo-tasks-fs/benches/mod.rs
View file @
0eaca9c7
mod
path_locks
;
use
std
::{
use
std
::{
fs
,
fs
,
sync
::{
Arc
,
mpsc
::
channel
},
sync
::{
Arc
,
mpsc
::
channel
},
...
@@ -103,6 +105,6 @@ fn bench_rope_iteration(c: &mut Criterion) {
...
@@ -103,6 +105,6 @@ fn bench_rope_iteration(c: &mut Criterion) {
criterion_group!
(
criterion_group!
(
name
=
benches
;
name
=
benches
;
config
=
Criterion
::
default
();
config
=
Criterion
::
default
();
targets
=
bench_file_watching
,
bench_rope_iteration
targets
=
bench_file_watching
,
bench_rope_iteration
,
path_locks
::
overhead
);
);
criterion_main!
(
benches
);
criterion_main!
(
benches
);
turbopack/crates/turbo-tasks-fs/benches/path_locks.rs
0 → 100644
View file @
0eaca9c7
#
[
allow
(
unused_imports
)]
// Inline unit tests are not run by the benchmark harness.
#[path
=
"../src/mutex_map.rs"
]
mod
mutex_map
;
use
std
::{
path
::
PathBuf
,
sync
::
Arc
,
thread
,
time
::
Instant
};
use
criterion
::{
BenchmarkId
,
Criterion
,
Throughput
};
use
futures
::
executor
::
block_on
;
use
mutex_map
::
MutexMap
;
pub
fn
overhead
(
c
:
&
mut
Criterion
)
{
let
mut
group
=
c
.benchmark_group
(
"filesystem_path_locks"
);
for
workers
in
[
1
,
6
]
{
group
.throughput
(
Throughput
::
Elements
(
workers
));
group
.bench_with_input
(
BenchmarkId
::
from_parameter
(
workers
),
&
workers
,
|
b
,
&
workers
|
{
let
map
=
MutexMap
::
<
Arc
<
PathBuf
>>
::
default
();
b
.iter_custom
(|
iters
|
{
let
start
=
Instant
::
now
();
thread
::
scope
(|
scope
|
{
for
worker
in
0
..
workers
{
let
map
=
&
map
;
scope
.spawn
(
move
||
{
let
path
=
Arc
::
new
(
PathBuf
::
from
(
format!
(
"/project/node_modules/package-{worker}/dist/index.js"
)));
for
_
in
0
..
iters
{
let
guard
=
block_on
(
map
.lock
(
path
.clone
()));
std
::
hint
::
black_box
(
&
guard
);
drop
(
guard
);
}
});
}
});
start
.elapsed
()
});
},
);
}
group
.finish
();
}
turbopack/crates/turbo-tasks-fs/src/mutex_map.rs
View file @
0eaca9c7
use
std
::
{
collections
::
hash_map
::
Entry
,
hash
::
Hash
}
;
use
std
::
hash
::
Hash
;
use
parking_lot
::
Mutex
;
use
dashmap
::
mapref
::
entry
::
Entry
;
use
rustc_hash
::
FxHashMap
;
use
turbo_tasks
::{
FxDashMap
,
event
::
Event
};
use
turbo_tasks
::
event
::
Event
;
pub
struct
MutexMap
<
K
>
{
pub
struct
MutexMap
<
K
>
{
map
:
Mutex
<
FxHashMap
<
K
,
Option
<
(
Event
,
usize
)
>>>
,
// Unrelated paths should not contend on a single filesystem-wide lock.
map
:
FxDashMap
<
K
,
Option
<
(
Event
,
usize
)
>>
,
}
}
impl
<
K
>
Default
for
MutexMap
<
K
>
{
impl
<
K
:
Eq
+
Hash
>
Default
for
MutexMap
<
K
>
{
fn
default
()
->
Self
{
fn
default
()
->
Self
{
Self
{
Self
{
map
:
Mutex
::
new
(
FxHashMap
::
default
()
),
map
:
FxDashMap
::
default
(
),
}
}
}
}
}
}
...
@@ -19,8 +19,7 @@ impl<K> Default for MutexMap<K> {
...
@@ -19,8 +19,7 @@ impl<K> Default for MutexMap<K> {
impl
<
'a
,
K
:
Eq
+
Hash
+
Clone
>
MutexMap
<
K
>
{
impl
<
'a
,
K
:
Eq
+
Hash
+
Clone
>
MutexMap
<
K
>
{
pub
async
fn
lock
(
&
'a
self
,
key
:
K
)
->
MutexMapGuard
<
'a
,
K
>
{
pub
async
fn
lock
(
&
'a
self
,
key
:
K
)
->
MutexMapGuard
<
'a
,
K
>
{
let
listener
=
{
let
listener
=
{
let
mut
map
=
self
.map
.lock
();
match
self
.map
.entry
(
key
.clone
())
{
match
map
.entry
(
key
.clone
())
{
Entry
::
Occupied
(
mut
e
)
=>
{
Entry
::
Occupied
(
mut
e
)
=>
{
let
state
=
e
.get_mut
();
let
state
=
e
.get_mut
();
Some
(
match
state
{
Some
(
match
state
{
...
@@ -60,8 +59,7 @@ pub struct MutexMapGuard<'a, K: Eq + Hash> {
...
@@ -60,8 +59,7 @@ pub struct MutexMapGuard<'a, K: Eq + Hash> {
impl
<
K
:
Eq
+
Hash
>
Drop
for
MutexMapGuard
<
'_
,
K
>
{
impl
<
K
:
Eq
+
Hash
>
Drop
for
MutexMapGuard
<
'_
,
K
>
{
fn
drop
(
&
mut
self
)
{
fn
drop
(
&
mut
self
)
{
if
let
Some
(
key
)
=
self
.key
.take
()
{
if
let
Some
(
key
)
=
self
.key
.take
()
{
let
mut
map
=
self
.map.map
.lock
();
if
let
Entry
::
Occupied
(
mut
e
)
=
self
.map.map
.entry
(
key
)
{
if
let
Entry
::
Occupied
(
mut
e
)
=
map
.entry
(
key
)
{
let
value
=
e
.get_mut
();
let
value
=
e
.get_mut
();
match
value
{
match
value
{
Some
((
event
,
count
))
=>
{
Some
((
event
,
count
))
=>
{
...
@@ -80,3 +78,48 @@ impl<K: Eq + Hash> Drop for MutexMapGuard<'_, K> {
...
@@ -80,3 +78,48 @@ impl<K: Eq + Hash> Drop for MutexMapGuard<'_, K> {
}
}
}
}
}
}
#[cfg(test)]
mod
tests
{
use
std
::
sync
::{
Arc
,
atomic
::{
AtomicUsize
,
Ordering
},
};
use
futures
::
FutureExt
;
use
super
::
MutexMap
;
#[tokio
::
test]
async
fn
distinct_paths_do_not_wait_on_each_other
()
{
let
map
=
MutexMap
::
default
();
let
first
=
map
.lock
(
1u
size
)
.await
;
let
second
=
map
.lock
(
2u
size
)
.now_or_never
()
.unwrap
();
assert_eq!
(
map
.map
.len
(),
2
);
drop
(
first
);
drop
(
second
);
assert
!
(
map
.map
.is_empty
());
}
#[tokio
::
test(flavor
=
"multi_thread"
,
worker_threads
=
4
)]
async
fn
contended_path_remains_exclusive_and_is_removed
()
{
let
map
=
Arc
::
new
(
MutexMap
::
default
());
let
active
=
Arc
::
new
(
AtomicUsize
::
new
(
0
));
let
mut
tasks
=
Vec
::
new
();
for
_
in
0
..
64
{
let
map
=
map
.clone
();
let
active
=
active
.clone
();
tasks
.push
(
tokio
::
spawn
(
async
move
{
let
guard
=
map
.lock
(
1u
size
)
.await
;
assert_eq!
(
active
.fetch_add
(
1
,
Ordering
::
SeqCst
),
0
);
tokio
::
task
::
yield_now
()
.await
;
assert_eq!
(
active
.fetch_sub
(
1
,
Ordering
::
SeqCst
),
1
);
drop
(
guard
);
}));
}
for
task
in
tasks
{
task
.await
.unwrap
();
}
assert
!
(
map
.map
.is_empty
());
}
}
turbopack/crates/turbo-tasks/benches/message_queue.rs
0 → 100644
View file @
0eaca9c7
use
std
::{
sync
::
Arc
,
time
::
Duration
};
use
criterion
::{
BenchmarkId
,
Criterion
};
use
turbo_tasks
::
message_queue
::{
CompilationEventQueue
,
TimingEvent
};
pub
fn
history_replay
(
c
:
&
mut
Criterion
)
{
let
rt
=
tokio
::
runtime
::
Builder
::
new_current_thread
()
.enable_all
()
.build
()
.unwrap
();
let
queue
=
CompilationEventQueue
::
default
();
rt
.block_on
(
async
{
for
_
in
0
..
256
{
queue
.send
(
Arc
::
new
(
TimingEvent
::
new
(
String
::
new
(),
Duration
::
ZERO
)))
.unwrap
();
}
queue
.flush_and_close
()
.await
;
});
let
mut
group
=
c
.benchmark_group
(
"compilation_event_history_replay"
);
for
event_type
in
[
"NoSuchEvent"
,
"TimingEvent"
]
{
group
.bench_with_input
(
BenchmarkId
::
from_parameter
(
event_type
),
&
event_type
,
|
b
,
ty
|
{
b
.to_async
(
&
rt
)
.iter
(||
async
{
let
mut
rx
=
queue
.subscribe
(
Some
(
vec!
[(
*
ty
)
.to_owned
()]));
let
mut
count
=
0
;
while
rx
.recv
()
.await
.is_some
()
{
count
+=
1
;
}
assert_eq!
(
count
,
if
*
ty
==
"TimingEvent"
{
256
}
else
{
0
});
});
},
);
}
group
.finish
();
}
turbopack/crates/turbo-tasks/benches/mod.rs
View file @
0eaca9c7
...
@@ -2,11 +2,12 @@
...
@@ -2,11 +2,12 @@
use
criterion
::{
Criterion
,
criterion_group
,
criterion_main
};
use
criterion
::{
Criterion
,
criterion_group
,
criterion_main
};
pub
(
crate
)
mod
message_queue
;
pub
(
crate
)
mod
scope
;
pub
(
crate
)
mod
scope
;
criterion_group!
(
criterion_group!
(
name
=
turbo_tasks
;
name
=
turbo_tasks
;
config
=
Criterion
::
default
();
config
=
Criterion
::
default
();
targets
=
scope
::
overhead
targets
=
scope
::
overhead
,
message_queue
::
history_replay
);
);
criterion_main!
(
turbo_tasks
);
criterion_main!
(
turbo_tasks
);
turbopack/crates/turbo-tasks/src/message_queue.rs
View file @
0eaca9c7
...
@@ -97,34 +97,30 @@ impl CompilationEventQueue {
...
@@ -97,34 +97,30 @@ impl CompilationEventQueue {
drop
(
history
);
drop
(
history
);
if
deliver
{
if
deliver
{
// Send to all active receivers of the same message type
for
channel_type
in
[
if
let
Some
(
mut
type_subscribers
)
=
subscribers
.get_mut
(
&
EventChannelType
::
Type
(
EventChannelType
::
Type
(
message_clone
.type_name
()
.to_owned
()),
message_clone
.type_name
()
.to_owned
(),
EventChannelType
::
Global
,
))
{
]
{
let
mut
removal_indices
=
Vec
::
new
();
// Snapshot the senders before awaiting backpressure. Holding a DashMap
for
(
ix
,
sender
)
in
type_subscribers
.iter
()
.enumerate
()
{
// guard across an await can block a runtime worker when another delivery
// or subscription needs the same shard.
let
senders
=
subscribers
.get
(
&
channel_type
)
.map
(|
channels
|
channels
.clone
())
.unwrap_or_default
();
let
mut
has_closed_sender
=
false
;
for
sender
in
senders
{
if
sender
.send
(
message_clone
.clone
())
.await
.is_err
()
{
if
sender
.send
(
message_clone
.clone
())
.await
.is_err
()
{
removal_indices
.push
(
ix
)
;
has_closed_sender
=
true
;
}
}
}
}
if
has_closed_sender
{
for
ix
in
removal_indices
.iter
()
.rev
()
{
if
let
Some
(
mut
channels
)
=
subscribers
.get_mut
(
&
channel_type
)
{
type_subscribers
.remove
(
*
ix
);
// Do not remove by snapshot indices: subscriptions may have been
}
// added or removed while we were delivering the event.
}
channels
.retain
(|
sender
|
!
sender
.is_closed
());
// Send to all global message subscribers
if
let
Some
(
mut
all_channel
)
=
subscribers
.get_mut
(
&
EventChannelType
::
Global
)
{
let
mut
removal_indices
=
Vec
::
new
();
for
(
ix
,
sender
)
in
all_channel
.iter_mut
()
.enumerate
()
{
if
sender
.send
(
message_clone
.clone
())
.await
.is_err
()
{
removal_indices
.push
(
ix
);
}
}
}
}
for
ix
in
removal_indices
.iter
()
.rev
()
{
all_channel
.remove
(
*
ix
);
}
}
}
}
}
...
@@ -183,8 +179,11 @@ impl CompilationEventQueue {
...
@@ -183,8 +179,11 @@ impl CompilationEventQueue {
}
}
}
}
for
event
in
event_history
.lock
()
.await
.iter
()
{
// History is bounded; clone it rather than holding the lock while a slow
if
event_types
.contains
(
&
event
.type_name
()
.to_string
())
{
// receiver drains its channel.
let
history
=
event_history
.lock
()
.await
.clone
();
for
event
in
history
.iter
()
{
if
event_types
.iter
()
.any
(|
ty
|
ty
==
event
.type_name
())
{
let
_
=
tx_clone
.send
(
event
.clone
())
.await
;
let
_
=
tx_clone
.send
(
event
.clone
())
.await
;
}
}
}
}
...
@@ -195,7 +194,10 @@ impl CompilationEventQueue {
...
@@ -195,7 +194,10 @@ impl CompilationEventQueue {
global_subscribers
.push
(
tx_clone
.clone
());
global_subscribers
.push
(
tx_clone
.clone
());
}
}
for
event
in
event_history
.lock
()
.await
.iter
()
{
// History is bounded; clone it rather than holding the lock while a slow
// receiver drains its channel.
let
history
=
event_history
.lock
()
.await
.clone
();
for
event
in
history
.iter
()
{
let
_
=
tx_clone
.send
(
event
.clone
())
.await
;
let
_
=
tx_clone
.send
(
event
.clone
())
.await
;
}
}
}
}
...
@@ -430,6 +432,129 @@ mod tests {
...
@@ -430,6 +432,129 @@ mod tests {
assert
!
(
rx2
.recv
()
.await
.is_none
());
assert
!
(
rx2
.recv
()
.await
.is_none
());
}
}
// A current-thread runtime makes accidental blocking shard locks observable: the
// receiver must be able to run while multiple deliveries wait for channel capacity.
#[tokio
::
test]
async
fn
test_backpressured_deliveries_do_not_block_worker
()
{
let
queue
=
CompilationEventQueue
::
default
();
let
(
tx
,
mut
rx
)
=
mpsc
::
channel
(
1
);
let
event
=
Arc
::
new
(
TimingEvent
::
new
(
"test"
.to_string
(),
Duration
::
ZERO
));
tx
.send
(
event
.clone
()
as
Arc
<
dyn
CompilationEvent
>
)
.await
.unwrap
();
queue
.subscribers
.get_mut
(
&
EventChannelType
::
Global
)
.unwrap
()
.push
(
tx
);
queue
.send
(
event
.clone
())
.unwrap
();
queue
.send
(
event
)
.unwrap
();
// Both delivery tasks should reach the backpressured send without blocking
// the runtime. The original implementation deadlocks on the second task.
tokio
::
task
::
yield_now
()
.await
;
assert_eq!
(
queue
.event_history
.lock
()
.await
.len
(),
2
);
for
_
in
0
..
3
{
rx
.recv
()
.await
.unwrap
();
}
queue
.flush_and_close
()
.await
;
assert
!
(
rx
.recv
()
.await
.is_none
());
}
#[tokio
::
test]
async
fn
test_closed_receiver_is_pruned_without_removing_live_subscriber
()
{
let
queue
=
CompilationEventQueue
::
default
();
let
(
tx
,
rx
)
=
mpsc
::
channel
(
1
);
drop
(
rx
);
let
(
live_tx
,
mut
live_rx
)
=
mpsc
::
channel
(
1
);
queue
.subscribers
.get_mut
(
&
EventChannelType
::
Global
)
.unwrap
()
.extend
([
tx
,
live_tx
]);
queue
.send
(
Arc
::
new
(
TimingEvent
::
new
(
"test"
.to_string
(),
Duration
::
ZERO
,
)))
.unwrap
();
live_rx
.recv
()
.await
.unwrap
();
assert_eq!
(
queue
.subscribers
.get
(
&
EventChannelType
::
Global
)
.unwrap
()
.len
(),
1
);
queue
.flush_and_close
()
.await
;
assert
!
(
live_rx
.recv
()
.await
.is_none
());
}
#[tokio
::
test]
async
fn
test_filtered_history_replay_after_close
()
{
let
queue
=
CompilationEventQueue
::
default
();
queue
.send
(
Arc
::
new
(
TimingEvent
::
new
(
"timing"
.to_string
(),
Duration
::
ZERO
,
)))
.unwrap
();
queue
.send
(
Arc
::
new
(
DiagnosticEvent
::
new
(
Severity
::
Warning
,
"diagnostic"
.to_string
(),
)))
.unwrap
();
queue
.flush_and_close
()
.await
;
let
mut
rx
=
queue
.subscribe
(
Some
(
vec!
[
"TimingEvent"
.to_string
()]));
assert_eq!
(
rx
.recv
()
.await
.unwrap
()
.type_name
(),
"TimingEvent"
);
assert
!
(
rx
.recv
()
.await
.is_none
());
let
mut
rx
=
queue
.subscribe
(
Some
(
vec!
[
"NoSuchEvent"
.to_string
()]));
assert
!
(
rx
.recv
()
.await
.is_none
());
}
#[tokio
::
test]
async
fn
test_backpressured_replay_does_not_lock_history
()
{
let
queue
=
CompilationEventQueue
::
default
();
let
event
:
Arc
<
dyn
CompilationEvent
>
=
Arc
::
new
(
TimingEvent
::
new
(
"test"
.to_string
(),
Duration
::
ZERO
));
// Hold the history lock until the subscription is registered and its
// channel is full, forcing replay to wait for the receiver.
let
mut
history
=
queue
.event_history
.lock
()
.await
;
history
.push_back
(
event
.clone
());
let
rx
=
queue
.subscribe
(
None
);
let
sender
=
loop
{
let
sender
=
queue
.subscribers
.get
(
&
EventChannelType
::
Global
)
.unwrap
()
.first
()
.cloned
();
if
let
Some
(
sender
)
=
sender
{
break
sender
;
}
tokio
::
task
::
yield_now
()
.await
;
};
for
_
in
0
..
MAX_QUEUE_SIZE
{
sender
.send
(
event
.clone
())
.await
.unwrap
();
}
drop
(
history
);
tokio
::
task
::
yield_now
()
.await
;
queue
.send
(
event
)
.unwrap
();
tokio
::
task
::
yield_now
()
.await
;
let
history_len
=
tokio
::
time
::
timeout
(
Duration
::
from_secs
(
1
),
async
{
queue
.event_history
.lock
()
.await
.len
()
})
.await
.unwrap
();
assert_eq!
(
history_len
,
2
);
drop
(
sender
);
drop
(
rx
);
queue
.flush_and_close
()
.await
;
}
#[test]
#[test]
fn
test_trace_event_new_with_duration
()
{
fn
test_trace_event_new_with_duration
()
{
let
start
=
SystemTime
::
UNIX_EPOCH
+
Duration
::
from_secs
(
10
);
let
start
=
SystemTime
::
UNIX_EPOCH
+
Duration
::
from_secs
(
10
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment