Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
492c2f21
Commit
492c2f21
authored
Aug 22, 2025
by
Tobias Koppers
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Turbopack: add async scopes
parent
3596c033
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
128 additions
and
10 deletions
+128
-10
parallel.rs
turbopack/crates/turbo-tasks/src/parallel.rs
+36
-2
scope.rs
turbopack/crates/turbo-tasks/src/scope.rs
+92
-8
No files found.
turbopack/crates/turbo-tasks/src/parallel.rs
View file @
492c2f21
...
...
@@ -6,7 +6,10 @@
use
std
::{
sync
::
LazyLock
,
thread
::
available_parallelism
};
use
crate
::{
scope
::
scope_and_block
,
util
::
into_chunks
};
use
crate
::{
scope
::{
future_scope
,
scope_and_block
},
util
::{
Chunk
,
into_chunks
},
};
/// Calculates a good chunk size for parallel processing based on the number of available threads.
/// This is used to ensure that the workload is evenly distributed across the threads.
...
...
@@ -58,7 +61,6 @@ where
let
_results
=
scope_and_block
(
len
.div_ceil
(
chunk_size
),
|
scope
|
{
for
chunk
in
into_chunks
(
items
,
chunk_size
)
{
scope
.spawn
(
async
move
{
// SAFETY: Even when f() panics we drop all items in the chunk.
for
item
in
chunk
{
f
(
item
);
}
...
...
@@ -67,6 +69,29 @@ where
});
}
pub
async
fn
for_each_chunk_owned_async
<
T
>
(
items
:
Vec
<
T
>
,
f
:
impl
Fn
(
Chunk
<
T
>
)
+
Send
+
Sync
)
where
T
:
Send
+
Sync
,
{
let
len
=
items
.len
();
if
len
<=
1
{
for
chunk
in
into_chunks
(
items
,
1
)
{
f
(
chunk
);
}
return
;
}
let
chunk_size
=
good_chunk_size
(
len
);
let
f
=
&
f
;
let
_results
=
future_scope
(
len
.div_ceil
(
chunk_size
),
|
scope
|
{
for
chunk
in
into_chunks
(
items
,
chunk_size
)
{
scope
.spawn
(
async
move
{
f
(
chunk
);
})
}
})
.await
;
}
pub
fn
try_for_each
<
'l
,
T
,
E
>
(
items
:
&
'l
[
T
],
f
:
impl
(
Fn
(
&
'l
T
)
->
Result
<
(),
E
>
)
+
Send
+
Sync
,
...
...
@@ -287,6 +312,15 @@ mod tests {
assert_eq!
(
result
,
vec!
[
2
;
1000
]);
}
#[tokio
::
test(flavor
=
"multi_thread"
,
worker_threads
=
2
)]
async
fn
test_parallel_different_counts
()
{
for
len
in
0
..
1000
{
let
input
=
vec!
[
1
;
len
];
let
result
:
Vec
<
_
>
=
map_collect_owned
(
input
,
|
x
|
x
*
2
);
assert_eq!
(
result
,
vec!
[
2
;
len
]);
}
}
#[tokio
::
test(flavor
=
"multi_thread"
,
worker_threads
=
2
)]
async
fn
test_panic_in_scope
()
{
let
result
=
catch_unwind
(
AssertUnwindSafe
(||
{
...
...
turbopack/crates/turbo-tasks/src/scope.rs
View file @
492c2f21
...
...
@@ -14,7 +14,7 @@ use std::{
use
futures
::
FutureExt
;
use
parking_lot
::
Mutex
;
use
tokio
::{
runtime
::
Handle
,
task
::
block_in_place
};
use
tokio
::{
runtime
::
Handle
,
sync
::
Notify
,
task
::
block_in_place
};
use
tracing
::{
Instrument
,
Span
,
info_span
};
use
crate
::{
...
...
@@ -22,8 +22,13 @@ use crate::{
manager
::{
try_turbo_tasks
,
turbo_tasks_future_scope
},
};
enum
ScopeInnerType
{
Blocking
{
main_thread
:
Thread
},
Future
{
notify
:
Notify
},
}
struct
ScopeInner
{
main_thread
:
Thread
,
ty
:
ScopeInnerType
,
remaining_tasks
:
AtomicUsize
,
/// The first panic that occurred in the tasks, by task index.
/// The usize value is the index of the task.
...
...
@@ -39,13 +44,30 @@ impl ScopeInner {
}
}
if
self
.remaining_tasks
.fetch_sub
(
1
,
Ordering
::
Release
)
==
1
{
self
.main_thread
.unpark
();
match
&
self
.ty
{
ScopeInnerType
::
Blocking
{
main_thread
}
=>
{
main_thread
.unpark
();
}
ScopeInnerType
::
Future
{
notify
}
=>
{
notify
.notify_one
();
}
}
}
}
fn
wait
(
&
self
)
{
async
fn
future_wait
(
&
self
)
{
let
ScopeInnerType
::
Future
{
notify
}
=
&
self
.ty
else
{
panic!
(
"Cannot call future() on a blocking scope"
);
};
while
self
.remaining_tasks
.load
(
Ordering
::
Acquire
)
!=
0
{
notify
.notified
()
.await
;
}
}
fn
blocking_wait
(
&
self
)
{
let
_span
=
info_span!
(
"blocking"
)
.entered
();
while
self
.remaining_tasks
.load
(
Ordering
::
Acquire
)
!=
0
{
assert
!
(
matches!
(
self
.ty
,
ScopeInnerType
::
Blocking
{
..
}));
thread
::
park
();
}
if
let
Some
((
err
,
_
))
=
self
.panic
.lock
()
.take
()
{
...
...
@@ -77,12 +99,20 @@ impl<'scope, 'env: 'scope, R: Send + 'env> Scope<'scope, 'env, R> {
/// # Safety
///
/// The caller must ensure `Scope` is dropped and not forgotten.
unsafe
fn
new
(
results
:
&
'scope
[
Mutex
<
Option
<
R
>>
])
->
Self
{
unsafe
fn
new
(
results
:
&
'scope
[
Mutex
<
Option
<
R
>>
]
,
blocking
:
bool
)
->
Self
{
Self
{
results
,
index
:
AtomicUsize
::
new
(
0
),
inner
:
Arc
::
new
(
ScopeInner
{
main_thread
:
thread
::
current
(),
ty
:
if
blocking
{
ScopeInnerType
::
Blocking
{
main_thread
:
thread
::
current
(),
}
}
else
{
ScopeInnerType
::
Future
{
notify
:
Notify
::
new
(),
}
},
remaining_tasks
:
AtomicUsize
::
new
(
0
),
panic
:
Mutex
::
new
(
None
),
}),
...
...
@@ -143,11 +173,15 @@ impl<'scope, 'env: 'scope, R: Send + 'env> Scope<'scope, 'env, R> {
inner
.on_task_finished
(
panic
);
});
}
pub
async
fn
future_wait
(
&
self
)
{
self
.inner
.future_wait
()
.await
;
}
}
impl
<
'scope
,
'env
:
'scope
,
R
:
Send
+
'env
>
Drop
for
Scope
<
'scope
,
'env
,
R
>
{
fn
drop
(
&
mut
self
)
{
self
.inner
.wait
();
self
.inner
.
blocking_
wait
();
}
}
...
...
@@ -171,7 +205,7 @@ where
let
results
=
results
.into_boxed_slice
();
let
result
=
{
// SAFETY: We drop the Scope later.
let
scope
=
unsafe
{
Scope
::
new
(
&
results
)
};
let
scope
=
unsafe
{
Scope
::
new
(
&
results
,
true
)
};
catch_unwind
(
AssertUnwindSafe
(||
f
(
&
scope
)))
};
if
let
Err
(
panic
)
=
result
{
...
...
@@ -185,12 +219,48 @@ where
})
}
pub
async
fn
future_scope
<
'env
,
F
,
R
>
(
number_of_tasks
:
usize
,
f
:
F
,
)
->
impl
Iterator
<
Item
=
R
>
+
'env
where
R
:
Send
+
'env
,
F
:
for
<
'scope
>
FnOnce
(
&
'scope
Scope
<
'scope
,
'env
,
R
>
)
+
'env
,
{
let
mut
results
=
Vec
::
with_capacity
(
number_of_tasks
);
for
_
in
0
..
number_of_tasks
{
results
.push
(
Mutex
::
new
(
None
));
}
let
results
=
results
.into_boxed_slice
();
let
result
=
{
// SAFETY: We drop the Scope later.
let
scope
=
unsafe
{
Scope
::
new
(
&
results
,
false
)
};
let
result
=
catch_unwind
(
AssertUnwindSafe
(||
f
(
&
scope
)));
scope
.future_wait
()
.await
;
result
};
if
let
Err
(
panic
)
=
result
{
panic
::
resume_unwind
(
panic
);
}
results
.into_iter
()
.map
(|
mutex
|
{
mutex
.into_inner
()
.expect
(
"All values are set when the scope returns without panic"
)
})
}
#[cfg(test)]
mod
tests
{
use
std
::
panic
::{
AssertUnwindSafe
,
catch_unwind
};
use
super
::
*
;
#[test]
fn
scope_is_send
()
{
fn
is_send
<
T
:
Send
+
Sync
>
()
{}
is_send
::
<
Scope
<
'static
,
'static
,
()
>>
();
}
#[tokio
::
test(flavor
=
"multi_thread"
,
worker_threads
=
2
)]
async
fn
test_scope
()
{
let
results
=
scope_and_block
(
1000
,
|
scope
|
{
...
...
@@ -286,4 +356,18 @@ mod tests {
Some
(
&
"Intentional panic"
)
);
}
#[tokio
::
test(flavor
=
"multi_thread"
)]
async
fn
test_future_scope
()
{
let
results
=
future_scope
(
1000
,
|
scope
|
{
for
i
in
0
..
1000
{
scope
.spawn
(
async
move
{
i
});
}
})
.await
.collect
::
<
Vec
<
_
>>
();
results
.iter
()
.enumerate
()
.for_each
(|(
i
,
&
result
)|
{
assert_eq!
(
result
,
i
);
});
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment