Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
N
next.js
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Alex
next.js
Commits
5c43a3a4
Unverified
Commit
5c43a3a4
authored
Oct 25, 2024
by
Tobias Koppers
Committed by
GitHub
Oct 25, 2024
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[Turbopack] add optimization based on upper count (#71606)
### What? Performance optimization
parent
5ba53703
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
560 additions
and
68 deletions
+560
-68
Cargo.toml
Cargo.toml
+1
-0
Cargo.toml
turbopack/crates/turbo-tasks-backend/Cargo.toml
+2
-1
aggregation_update.rs
...tasks-backend/src/backend/operation/aggregation_update.rs
+404
-49
cleanup_old_edges.rs
...-tasks-backend/src/backend/operation/cleanup_old_edges.rs
+2
-6
connect_child.rs
...urbo-tasks-backend/src/backend/operation/connect_child.rs
+7
-10
storage.rs
turbopack/crates/turbo-tasks-backend/src/backend/storage.rs
+66
-2
data.rs
turbopack/crates/turbo-tasks-backend/src/data.rs
+7
-0
deque_set.rs
turbopack/crates/turbo-tasks-backend/src/utils/deque_set.rs
+70
-0
mod.rs
turbopack/crates/turbo-tasks-backend/src/utils/mod.rs
+1
-0
No files found.
Cargo.toml
View file @
5c43a3a4
...
@@ -204,6 +204,7 @@ sourcemap = "9.0.0"
...
@@ -204,6 +204,7 @@ sourcemap = "9.0.0"
strsim
=
"0.11.1"
strsim
=
"0.11.1"
syn
=
"1.0.107"
syn
=
"1.0.107"
tempfile
=
"3.3.0"
tempfile
=
"3.3.0"
thread_local
=
"1.1.8"
thiserror
=
"1.0.48"
thiserror
=
"1.0.48"
tiny-gradient
=
"0.1.0"
tiny-gradient
=
"0.1.0"
tokio
=
"1.25.0"
tokio
=
"1.25.0"
...
...
turbopack/crates/turbo-tasks-backend/Cargo.toml
View file @
5c43a3a4
...
@@ -15,6 +15,7 @@ workspace = true
...
@@ -15,6 +15,7 @@ workspace = true
[features]
[features]
default
=
[]
default
=
[]
verify_serialization
=
[]
verify_serialization
=
[]
trace_aggregation_update
=
[]
[dependencies]
[dependencies]
anyhow
=
{
workspace
=
true
}
anyhow
=
{
workspace
=
true
}
...
@@ -39,7 +40,7 @@ smallvec = { workspace = true }
...
@@ -39,7 +40,7 @@ smallvec = { workspace = true }
tokio
=
{
workspace
=
true
}
tokio
=
{
workspace
=
true
}
tokio-scoped
=
"0.2.0"
tokio-scoped
=
"0.2.0"
tracing
=
{
workspace
=
true
}
tracing
=
{
workspace
=
true
}
thread_local
=
{
version
=
"1.1.8"
}
thread_local
=
{
workspace
=
true
}
turbo-prehash
=
{
workspace
=
true
}
turbo-prehash
=
{
workspace
=
true
}
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks
=
{
workspace
=
true
}
turbo-tasks-hash
=
{
workspace
=
true
}
turbo-tasks-hash
=
{
workspace
=
true
}
...
...
turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs
View file @
5c43a3a4
This diff is collapsed.
Click to expand it.
turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs
View file @
5c43a3a4
...
@@ -13,7 +13,7 @@ use crate::{
...
@@ -13,7 +13,7 @@ use crate::{
invalidate
::{
make_task_dirty
,
TaskDirtyCause
},
invalidate
::{
make_task_dirty
,
TaskDirtyCause
},
AggregatedDataUpdate
,
ExecuteContext
,
Operation
,
TaskGuard
,
AggregatedDataUpdate
,
ExecuteContext
,
Operation
,
TaskGuard
,
},
},
storage
::{
update
,
update_coun
t
},
storage
::{
update
_count
,
update_ucount_and_ge
t
},
TaskDataCategory
,
TaskDataCategory
,
},
},
data
::{
CachedDataItemKey
,
CellRef
,
CollectibleRef
,
CollectiblesRef
},
data
::{
CachedDataItemKey
,
CellRef
,
CollectibleRef
,
CollectiblesRef
},
...
@@ -83,11 +83,7 @@ impl Operation for CleanupOldEdgesOperation {
...
@@ -83,11 +83,7 @@ impl Operation for CleanupOldEdgesOperation {
task
.remove
(
&
CachedDataItemKey
::
Child
{
task
:
child_id
});
task
.remove
(
&
CachedDataItemKey
::
Child
{
task
:
child_id
});
}
}
let
remove_children_count
=
u32
::
try_from
(
children
.len
())
.unwrap
();
let
remove_children_count
=
u32
::
try_from
(
children
.len
())
.unwrap
();
update!
(
task
,
ChildrenCount
,
|
count
:
Option
<
u32
>
|
{
update_ucount_and_get!
(
task
,
ChildrenCount
,
-
remove_children_count
);
// If this underflows, we messed up counting somewhere
let
count
=
count
.unwrap_or_default
()
-
remove_children_count
;
(
count
!=
0
)
.then_some
(
count
)
});
if
is_aggregating_node
(
get_aggregation_number
(
&
task
))
{
if
is_aggregating_node
(
get_aggregation_number
(
&
task
))
{
queue
.push
(
AggregationUpdateJob
::
InnerOfUpperLostFollowers
{
queue
.push
(
AggregationUpdateJob
::
InnerOfUpperLostFollowers
{
upper_id
:
task_id
,
upper_id
:
task_id
,
...
...
turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs
View file @
5c43a3a4
...
@@ -12,7 +12,7 @@ use crate::{
...
@@ -12,7 +12,7 @@ use crate::{
},
},
is_root_node
,
ExecuteContext
,
Operation
,
TaskGuard
,
is_root_node
,
ExecuteContext
,
Operation
,
TaskGuard
,
},
},
storage
::{
get
,
update
},
storage
::{
get
,
update
_ucount_and_get
},
TaskDataCategory
,
TaskDataCategory
,
},
},
data
::{
CachedDataItem
,
CachedDataItemKey
},
data
::{
CachedDataItem
,
CachedDataItemKey
},
...
@@ -46,17 +46,12 @@ impl ConnectChildOperation {
...
@@ -46,17 +46,12 @@ impl ConnectChildOperation {
task
:
child_task_id
,
task
:
child_task_id
,
value
:
(),
value
:
(),
})
{
})
{
// Update the children count
let
mut
children_count
=
0
;
update!
(
parent_task
,
ChildrenCount
,
|
count
:
Option
<
u32
>
|
{
children_count
=
count
.unwrap_or_default
()
+
1
;
Some
(
children_count
)
});
// Update the task aggregation
let
mut
queue
=
AggregationUpdateQueue
::
new
();
let
mut
queue
=
AggregationUpdateQueue
::
new
();
// Compute new parent aggregation number based on the number of children
// Update the children count
let
children_count
=
update_ucount_and_get!
(
parent_task
,
ChildrenCount
,
1
);
// Compute future parent aggregation number based on the number of children
let
current_parent_aggregation
=
get!
(
parent_task
,
AggregationNumber
)
let
current_parent_aggregation
=
get!
(
parent_task
,
AggregationNumber
)
.copied
()
.copied
()
.unwrap_or_default
();
.unwrap_or_default
();
...
@@ -130,6 +125,8 @@ impl ConnectChildOperation {
...
@@ -130,6 +125,8 @@ impl ConnectChildOperation {
}
}
}
}
#[cfg(feature
=
"trace_aggregation_update"
)]
let
_span
=
tracing
::
trace_span!
(
"connect_child"
)
.entered
();
ConnectChildOperation
::
UpdateAggregation
{
ConnectChildOperation
::
UpdateAggregation
{
aggregation_update
:
queue
,
aggregation_update
:
queue
,
}
}
...
...
turbopack/crates/turbo-tasks-backend/src/backend/storage.rs
View file @
5c43a3a4
...
@@ -487,12 +487,73 @@ macro_rules! update {
...
@@ -487,12 +487,73 @@ macro_rules! update {
};
};
}
}
macro_rules!
update_ucount_and_get
{
(
$task:ident
,
$key:ident
$input:tt
,
-
$update:expr
)
=>
{
match
$update
{
update
=>
{
let
mut
value
=
0
;
$crate
::
backend
::
storage
::
update!
(
$task
,
$key
$input
,
|
old
:
Option
<
_
>
|
{
if
let
Some
(
old
)
=
old
{
value
=
old
-
update
;
(
value
!=
0
)
.then_some
(
value
)
}
else
{
None
}
});
value
}
}
};
(
$task:ident
,
$key:ident
$input:tt
,
$update:expr
)
=>
{
match
$update
{
update
=>
{
let
mut
value
=
0
;
$crate
::
backend
::
storage
::
update!
(
$task
,
$key
$input
,
|
old
:
Option
<
_
>
|
{
if
let
Some
(
old
)
=
old
{
value
=
old
+
update
;
(
value
!=
0
)
.then_some
(
value
)
}
else
{
value
=
update
;
(
update
!=
0
)
.then_some
(
update
)
}
});
value
}
}
};
(
$task:ident
,
$key:ident
,
-
$update:expr
)
=>
{
$crate
::
backend
::
storage
::
update_ucount_and_get!
(
$task
,
$key
{},
-
$update
)
};
(
$task:ident
,
$key:ident
,
$update:expr
)
=>
{
$crate
::
backend
::
storage
::
update_ucount_and_get!
(
$task
,
$key
{},
$update
)
};
}
macro_rules!
update_count
{
macro_rules!
update_count
{
(
$task:ident
,
$key:ident
$input:tt
,
-
$update:expr
)
=>
{
match
$update
{
update
=>
{
let
mut
state_change
=
false
;
$crate
::
backend
::
storage
::
update!
(
$task
,
$key
$input
,
|
old
:
Option
<
_
>
|
{
#[allow(unused_comparisons,
reason
=
"type of update might be unsigned, where update < 0 is always false"
)]
if
let
Some
(
old
)
=
old
{
let
new
=
old
-
update
;
state_change
=
old
<=
0
&&
new
>
0
||
old
>
0
&&
new
<=
0
;
(
new
!=
0
)
.then_some
(
new
)
}
else
{
state_change
=
update
<
0
;
(
update
!=
0
)
.then_some
(
-
update
)
}
});
state_change
}
}
};
(
$task:ident
,
$key:ident
$input:tt
,
$update:expr
)
=>
{
(
$task:ident
,
$key:ident
$input:tt
,
$update:expr
)
=>
{
match
$update
{
match
$update
{
update
=>
{
update
=>
{
let
mut
state_change
=
false
;
let
mut
state_change
=
false
;
$crate
::
backend
::
storage
::
update!
(
$task
,
$key
$input
,
|
old
:
Option
<
i32
>
|
{
$crate
::
backend
::
storage
::
update!
(
$task
,
$key
$input
,
|
old
:
Option
<
_
>
|
{
if
let
Some
(
old
)
=
old
{
if
let
Some
(
old
)
=
old
{
let
new
=
old
+
update
;
let
new
=
old
+
update
;
state_change
=
old
<=
0
&&
new
>
0
||
old
>
0
&&
new
<=
0
;
state_change
=
old
<=
0
&&
new
>
0
||
old
>
0
&&
new
<=
0
;
...
@@ -506,7 +567,9 @@ macro_rules! update_count {
...
@@ -506,7 +567,9 @@ macro_rules! update_count {
}
}
}
}
};
};
(
$task:ident
,
$key:ident
,
$update:expr
)
=>
{
(
$task:ident
,
$key:ident
,
-
$update:expr
)
=>
{
$crate
::
backend
::
storage
::
update_count!
(
$task
,
$key
{},
-
$update
)
};
(
$task:ident
,
$key:ident
,
$update:expr
)
=>
{
$crate
::
backend
::
storage
::
update_count!
(
$task
,
$key
{},
$update
)
$crate
::
backend
::
storage
::
update_count!
(
$task
,
$key
{},
$update
)
};
};
}
}
...
@@ -535,3 +598,4 @@ pub(crate) use iter_many;
...
@@ -535,3 +598,4 @@ pub(crate) use iter_many;
pub
(
crate
)
use
remove
;
pub
(
crate
)
use
remove
;
pub
(
crate
)
use
update
;
pub
(
crate
)
use
update
;
pub
(
crate
)
use
update_count
;
pub
(
crate
)
use
update_count
;
pub
(
crate
)
use
update_ucount_and_get
;
turbopack/crates/turbo-tasks-backend/src/data.rs
View file @
5c43a3a4
...
@@ -369,6 +369,10 @@ pub enum CachedDataItem {
...
@@ -369,6 +369,10 @@ pub enum CachedDataItem {
task
:
TaskId
,
task
:
TaskId
,
value
:
i32
,
value
:
i32
,
},
},
PersistentUpperCount
{
// Only counting persistent tasks
value
:
u32
,
},
// Aggregated Data
// Aggregated Data
AggregatedDirtyContainer
{
AggregatedDirtyContainer
{
...
@@ -453,6 +457,7 @@ impl CachedDataItem {
...
@@ -453,6 +457,7 @@ impl CachedDataItem {
CachedDataItem
::
AggregationNumber
{
..
}
=>
true
,
CachedDataItem
::
AggregationNumber
{
..
}
=>
true
,
CachedDataItem
::
Follower
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
Follower
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
Upper
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
Upper
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
PersistentUpperCount
{
..
}
=>
true
,
CachedDataItem
::
AggregatedDirtyContainer
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
AggregatedDirtyContainer
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItem
::
AggregatedCollectible
{
collectible
,
..
}
=>
{
CachedDataItem
::
AggregatedCollectible
{
collectible
,
..
}
=>
{
!
collectible
.cell.task
.is_transient
()
!
collectible
.cell.task
.is_transient
()
...
@@ -518,6 +523,7 @@ impl CachedDataItemKey {
...
@@ -518,6 +523,7 @@ impl CachedDataItemKey {
CachedDataItemKey
::
AggregationNumber
{
..
}
=>
true
,
CachedDataItemKey
::
AggregationNumber
{
..
}
=>
true
,
CachedDataItemKey
::
Follower
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
Follower
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
Upper
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
Upper
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
PersistentUpperCount
{}
=>
true
,
CachedDataItemKey
::
AggregatedDirtyContainer
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
AggregatedDirtyContainer
{
task
,
..
}
=>
!
task
.is_transient
(),
CachedDataItemKey
::
AggregatedCollectible
{
collectible
,
..
}
=>
{
CachedDataItemKey
::
AggregatedCollectible
{
collectible
,
..
}
=>
{
!
collectible
.cell.task
.is_transient
()
!
collectible
.cell.task
.is_transient
()
...
@@ -562,6 +568,7 @@ impl CachedDataItemKey {
...
@@ -562,6 +568,7 @@ impl CachedDataItemKey {
|
CachedDataItemKey
::
Dirty
{
..
}
|
CachedDataItemKey
::
Dirty
{
..
}
|
CachedDataItemKey
::
Follower
{
..
}
|
CachedDataItemKey
::
Follower
{
..
}
|
CachedDataItemKey
::
Upper
{
..
}
|
CachedDataItemKey
::
Upper
{
..
}
|
CachedDataItemKey
::
PersistentUpperCount
{
..
}
|
CachedDataItemKey
::
AggregatedDirtyContainer
{
..
}
|
CachedDataItemKey
::
AggregatedDirtyContainer
{
..
}
|
CachedDataItemKey
::
AggregatedCollectible
{
..
}
|
CachedDataItemKey
::
AggregatedCollectible
{
..
}
|
CachedDataItemKey
::
AggregatedDirtyContainerCount
{
..
}
|
CachedDataItemKey
::
AggregatedDirtyContainerCount
{
..
}
...
...
turbopack/crates/turbo-tasks-backend/src/utils/deque_set.rs
0 → 100644
View file @
5c43a3a4
use
std
::{
collections
::{
HashSet
,
VecDeque
},
hash
::{
BuildHasher
,
BuildHasherDefault
,
Hash
},
};
use
rustc_hash
::
FxHasher
;
use
serde
::{
Deserialize
,
Serialize
};
// TODO This could be more efficent:
// - Not storing items twice
// - Not serializing items twice
#[derive(Clone,
Serialize,
Deserialize)]
#[serde(bound(
deserialize
=
"T: Hash + Eq + Deserialize<'de>, B: BuildHasher + Default"
,
serialize
=
"T: Hash + Eq + Serialize, B: BuildHasher + Default"
))]
pub
struct
DequeSet
<
T
,
B
:
BuildHasher
=
BuildHasherDefault
<
FxHasher
>>
{
set
:
HashSet
<
T
,
B
>
,
queue
:
VecDeque
<
T
>
,
}
impl
<
T
,
B
:
BuildHasher
+
Default
>
Default
for
DequeSet
<
T
,
B
>
{
fn
default
()
->
Self
{
Self
{
set
:
Default
::
default
(),
queue
:
Default
::
default
(),
}
}
}
impl
<
T
,
B
:
BuildHasher
>
DequeSet
<
T
,
B
>
{
pub
fn
is_empty
(
&
self
)
->
bool
{
self
.queue
.is_empty
()
}
#[allow(dead_code)]
pub
fn
len
(
&
self
)
->
usize
{
self
.queue
.len
()
}
}
impl
<
T
:
Hash
+
Eq
+
Clone
,
B
:
BuildHasher
>
DequeSet
<
T
,
B
>
{
pub
fn
insert_back
(
&
mut
self
,
item
:
T
)
->
bool
{
if
self
.set
.insert
(
item
.clone
())
{
self
.queue
.push_back
(
item
);
true
}
else
{
false
}
}
pub
fn
pop_front
(
&
mut
self
)
->
Option
<
T
>
{
if
let
Some
(
item
)
=
self
.queue
.pop_front
()
{
self
.set
.remove
(
&
item
);
Some
(
item
)
}
else
{
None
}
}
}
impl
<
T
:
Hash
+
Eq
+
Clone
,
B
:
BuildHasher
>
Extend
<
T
>
for
DequeSet
<
T
,
B
>
{
fn
extend
<
I
:
IntoIterator
<
Item
=
T
>>
(
&
mut
self
,
iter
:
I
)
{
self
.queue
.extend
(
iter
.into_iter
()
.filter
(|
item
|
self
.set
.insert
(
item
.clone
())),
);
}
}
turbopack/crates/turbo-tasks-backend/src/utils/mod.rs
View file @
5c43a3a4
pub
mod
bi_map
;
pub
mod
bi_map
;
pub
mod
chunked_vec
;
pub
mod
chunked_vec
;
pub
mod
dash_map_multi
;
pub
mod
dash_map_multi
;
pub
mod
deque_set
;
pub
mod
ptr_eq_arc
;
pub
mod
ptr_eq_arc
;
pub
mod
sharded
;
pub
mod
sharded
;
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment