Unverified Commit ee3a50cc authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

Upgrade Turbopack to hashbrown 0.15 (#97808)

### What?

Upgrade Turbopack's workspace dependencies to DashMap 7.0.0-rc2 and
Hashbrown 0.15.4, removing Hashbrown 0.14 from the `turbo-tasks-backend`
dependency graph.

### Why?

`turbo-tasks-backend` previously compiled two Hashbrown versions because
its custom DashMap helpers shared Hashbrown 0.14's private raw table
types. Converging the backend graph on Hashbrown 0.15 removes that
duplicate backend codegen path and slightly reduces the shipped native
addon's size.

A clean release A/B of the unstripped x86-64 `next-napi-bindings` cdylib
measured:

| Revision | `libnext_napi_bindings.so` |
| --- | ---: |
| canary base `80576037` | 147,824,920 bytes |
| PR | 147,818,576 bytes |
| **Delta** | **−6,344 bytes (−0.0043%)** |

The final gain is small because unrelated SWC dependencies still pull
DashMap 6 / Hashbrown 0.14 into the same addon; this change removes only
backend-specific Hashbrown 0.14 codegen. Plus... nearly everything is
monomorphized anyway

### How?

DashMap 7 replaces its Hashbrown 0.14 raw table with Hashbrown 0.15's
public `HashTable` API. The backend helpers now use that public API
while retaining their existing shard locking and allocation behavior:

- heterogeneous cache lookup and insertion still reuse a pre-located
shard and precomputed hash;
- the entry helper passes Hashbrown's native `hash_table::Entry` into a
closure, preserving precise lock-release timing without a custom
self-referential entry type or cloned hasher;
- nonblocking removal still avoids lock-order deadlocks;
- two-key mutation uses a release-mode disjointness assertion followed
by `get_many_unchecked_mut` to avoid a redundant duplicate-pointer scan;
- snapshot draining still transfers ownership of each shard table and
releases task memory incrementally;
- custom mutable guards remain non-`Send`.

The migration also replaces several raw-pointer iteration and removal
paths with safe `HashTable` operations. Existing raw-entry users in
`auto-hash-map` and `turbopack-trace-server` explicitly enable Hashbrown
0.15's compatibility feature.

### Performance

Performance testing with the overhead.rs benchmark revealed no
regression/progression
<!-- NEXT_JS_LLM -->


<!-- fleet 53ff0bed-46a9-4361-befa-a1791d585f27 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarLuke Sandberg <210140+lukesandberg@users.noreply.github.com>
parent 49d3043e
...@@ -405,7 +405,7 @@ version = "0.1.0" ...@@ -405,7 +405,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"hashbrown 0.14.5", "hashbrown 0.15.4",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
"serde", "serde",
"shrink-to-fit", "shrink-to-fit",
...@@ -2123,6 +2123,20 @@ dependencies = [ ...@@ -2123,6 +2123,20 @@ dependencies = [
"parking_lot_core", "parking_lot_core",
] ]
[[package]]
name = "dashmap"
version = "7.0.0-rc2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a1e35a65fe0538a60167f0ada6e195ad5d477f6ddae273943596d4a1a5730b"
dependencies = [
"cfg-if",
"crossbeam-utils",
"equivalent",
"hashbrown 0.15.4",
"lock_api",
"parking_lot_core",
]
[[package]] [[package]]
name = "data-encoding" name = "data-encoding"
version = "2.4.0" version = "2.4.0"
...@@ -2529,7 +2543,7 @@ checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" ...@@ -2529,7 +2543,7 @@ checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.3.5",
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
...@@ -3147,7 +3161,6 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" ...@@ -3147,7 +3161,6 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [ dependencies = [
"ahash 0.8.12", "ahash 0.8.12",
"allocator-api2", "allocator-api2",
"serde",
] ]
[[package]] [[package]]
...@@ -4389,11 +4402,10 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" ...@@ -4389,11 +4402,10 @@ checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
version = "0.4.10" version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [ dependencies = [
"autocfg",
"scopeguard", "scopeguard",
"serde", "serde",
] ]
...@@ -5552,15 +5564,15 @@ dependencies = [ ...@@ -5552,15 +5564,15 @@ dependencies = [
[[package]] [[package]]
name = "parking_lot_core" name = "parking_lot_core"
version = "0.9.8" version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"libc", "libc",
"redox_syscall", "redox_syscall 0.5.18",
"smallvec", "smallvec",
"windows-targets 0.48.5", "windows-link 0.2.1",
] ]
[[package]] [[package]]
...@@ -6469,6 +6481,15 @@ dependencies = [ ...@@ -6469,6 +6481,15 @@ dependencies = [
"bitflags 1.3.2", "bitflags 1.3.2",
] ]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags 2.9.1",
]
[[package]] [[package]]
name = "ref-cast" name = "ref-cast"
version = "1.0.24" version = "1.0.24"
...@@ -9948,7 +9969,7 @@ dependencies = [ ...@@ -9948,7 +9969,7 @@ dependencies = [
"byteorder", "byteorder",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"crc32fast", "crc32fast",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"either", "either",
"fs-err", "fs-err",
"jiff", "jiff",
...@@ -10020,7 +10041,7 @@ dependencies = [ ...@@ -10020,7 +10041,7 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"concurrent-queue", "concurrent-queue",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"either", "either",
"erased-serde", "erased-serde",
"event-listener", "event-listener",
...@@ -10063,10 +10084,10 @@ dependencies = [ ...@@ -10063,10 +10084,10 @@ dependencies = [
"bitfield", "bitfield",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"crossbeam-utils", "crossbeam-utils",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"fs-err", "fs-err",
"futures", "futures",
"hashbrown 0.14.5", "hashbrown 0.15.4",
"indexmap 2.14.0", "indexmap 2.14.0",
"indoc", "indoc",
"jiff", "jiff",
...@@ -10170,7 +10191,7 @@ dependencies = [ ...@@ -10170,7 +10191,7 @@ dependencies = [
"bytes", "bytes",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"concurrent-queue", "concurrent-queue",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"fs-err", "fs-err",
"futures", "futures",
"include_dir", "include_dir",
...@@ -10596,7 +10617,7 @@ dependencies = [ ...@@ -10596,7 +10617,7 @@ dependencies = [
"bumpalo", "bumpalo",
"bytes-str", "bytes-str",
"codspeed-criterion-compat", "codspeed-criterion-compat",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"data-encoding", "data-encoding",
"either", "either",
"forked_react_compiler", "forked_react_compiler",
...@@ -10796,7 +10817,7 @@ dependencies = [ ...@@ -10796,7 +10817,7 @@ dependencies = [
"bincode 2.0.1", "bincode 2.0.1",
"bytes", "bytes",
"const_format", "const_format",
"dashmap 6.1.0", "dashmap 7.0.0-rc2",
"either", "either",
"futures", "futures",
"futures-retry", "futures-retry",
...@@ -10954,7 +10975,7 @@ dependencies = [ ...@@ -10954,7 +10975,7 @@ dependencies = [
"anyhow", "anyhow",
"either", "either",
"flate2", "flate2",
"hashbrown 0.14.5", "hashbrown 0.15.4",
"indexmap 2.14.0", "indexmap 2.14.0",
"itertools 0.10.5", "itertools 0.10.5",
"postcard", "postcard",
......
...@@ -268,7 +268,7 @@ criterion = { package = "codspeed-criterion-compat", version = "4.3.0" } ...@@ -268,7 +268,7 @@ criterion = { package = "codspeed-criterion-compat", version = "4.3.0" }
ctor = "1.0.7" ctor = "1.0.7"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
crossbeam-utils = "0.8" crossbeam-utils = "0.8"
dashmap = "6.1.0" dashmap = "7.0.0-rc2"
data-encoding = "2.3.3" data-encoding = "2.3.3"
dhat = { version = "0.3.2" } dhat = { version = "0.3.2" }
dunce = "1.0.3" dunce = "1.0.3"
...@@ -280,7 +280,7 @@ fs-err = "3.1.1" ...@@ -280,7 +280,7 @@ fs-err = "3.1.1"
futures = "0.3.31" futures = "0.3.31"
futures-retry = "0.6.0" futures-retry = "0.6.0"
futures-util = "0.3.31" futures-util = "0.3.31"
hashbrown = "0.14.5" hashbrown = "0.15.4"
image = { version = "0.25.8", default-features = false } image = { version = "0.25.8", default-features = false }
indexmap = "2.14.0" indexmap = "2.14.0"
indoc = "2.0.0" indoc = "2.0.0"
......
...@@ -10,7 +10,7 @@ workspace = true ...@@ -10,7 +10,7 @@ workspace = true
[dependencies] [dependencies]
bincode = { workspace = true } bincode = { workspace = true }
hashbrown = { workspace = true, features = ["serde"]} hashbrown = { workspace = true, features = ["raw-entry", "serde"]}
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }
shrink-to-fit = { workspace = true, features = ["hashbrown"] } shrink-to-fit = { workspace = true, features = ["hashbrown"] }
......
...@@ -43,7 +43,7 @@ bitfield = { workspace = true } ...@@ -43,7 +43,7 @@ bitfield = { workspace = true }
crossbeam-utils = { workspace = true } crossbeam-utils = { workspace = true }
dashmap = { workspace = true, features = ["raw-api"]} dashmap = { workspace = true, features = ["raw-api"]}
fs-err = { workspace = true } fs-err = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
jiff = "0.2.10" jiff = "0.2.10"
lzzzz = { workspace = true, optional = true } lzzzz = { workspace = true, optional = true }
......
...@@ -22,6 +22,7 @@ use std::{ ...@@ -22,6 +22,7 @@ use std::{
use anyhow::{Context, Result, bail}; use anyhow::{Context, Result, bail};
use auto_hash_map::{AutoMap, AutoSet}; use auto_hash_map::{AutoMap, AutoSet};
use hashbrown::hash_table::Entry;
use indexmap::IndexSet; use indexmap::IndexSet;
use parking_lot::Mutex; use parking_lot::Mutex;
use rustc_hash::{FxHashMap, FxHashSet, FxHasher}; use rustc_hash::{FxHashMap, FxHashSet, FxHasher};
...@@ -80,7 +81,7 @@ use crate::{ ...@@ -80,7 +81,7 @@ use crate::{
error::TaskError, error::TaskError,
kv_backing_storage::TurboBackingStorage, kv_backing_storage::TurboBackingStorage,
utils::{ utils::{
dash_map_raw_entry::{RawEntry, get_shard, raw_entry_in_shard, raw_get_in_shard}, dash_map_entry::{get_in_shard, get_shard, with_entry_in_shard},
shard_amount::compute_shard_amount, shard_amount::compute_shard_amount,
stopwatch::Stopwatch, stopwatch::Stopwatch,
}, },
...@@ -1584,7 +1585,7 @@ impl TurboTasksBackend { ...@@ -1584,7 +1585,7 @@ impl TurboTasksBackend {
// Use a read lock rather than a write lock to avoid contention. connect_child // Use a read lock rather than a write lock to avoid contention. connect_child
// may re-enter task_cache with a write lock, so we must not hold a write lock here. // may re-enter task_cache with a write lock, so we must not hold a write lock here.
if let Some(task_id) = if let Some(task_id) =
raw_get_in_shard(shard, hash, |k| k.eq_components(native_fn, this, arg_ref)) get_in_shard(shard, hash, |k| k.eq_components(native_fn, this, arg_ref))
{ {
self.track_cache_hit_by_fn(native_fn); self.track_cache_hit_by_fn(native_fn);
operation::ConnectChildOperation::run(parent_task, task_id, ctx); operation::ConnectChildOperation::run(parent_task, task_id, ctx);
...@@ -1601,31 +1602,36 @@ impl TurboTasksBackend { ...@@ -1601,31 +1602,36 @@ impl TurboTasksBackend {
self.track_cache_hit_by_fn(native_fn); self.track_cache_hit_by_fn(native_fn);
// Step 3a: Insert into in-memory cache using the pre-located shard. // Step 3a: Insert into in-memory cache using the pre-located shard.
// Use the existing Arc from storage to avoid a duplicate allocation. // Use the existing Arc from storage to avoid a duplicate allocation.
match raw_entry_in_shard(shard, self.storage.task_cache.hasher(), hash, |k| { with_entry_in_shard(
k.eq_components(native_fn, this, arg_ref) shard,
}) { self.storage.task_cache.hasher(),
RawEntry::Occupied(_) => {} hash,
RawEntry::Vacant(e) => { arg,
e.insert(stored_type, task_id); |k, arg| k.eq_components(native_fn, this, arg.as_ref()),
|entry, _arg| {
if let Entry::Vacant(entry) = entry {
entry.insert((stored_type, task_id));
} }
}; },
);
task_id task_id
} else { } else {
match raw_entry_in_shard(shard, self.storage.task_cache.hasher(), hash, |k| { let (task_id, created) = with_entry_in_shard(
k.eq_components(native_fn, this, arg_ref) shard,
}) { self.storage.task_cache.hasher(),
RawEntry::Occupied(e) => { hash,
arg,
|k, arg| k.eq_components(native_fn, this, arg.as_ref()),
|entry, arg| match entry {
Entry::Occupied(entry) => {
// Another thread beat us to creating this task — use their task_id. // Another thread beat us to creating this task — use their task_id.
// They will handle logging the new task as modified. // They will handle logging the new task as modified.
let task_id = *e.get(); (entry.get().1, false)
drop(e);
self.track_cache_hit_by_fn(native_fn);
task_id
} }
RawEntry::Vacant(e) => { Entry::Vacant(entry) => {
// Only now do we force the allocation. // Only now do we force the allocation.
// NOTE: if our caller had to perform resolution, then this will have already // NOTE: if our caller had to perform resolution, then this will have
// been boxed and take_box just takes it. // already been boxed and take_box just takes it.
let task_type = CachedTaskTypeArc::new(CachedTaskType { let task_type = CachedTaskTypeArc::new(CachedTaskType {
native_fn, native_fn,
this, this,
...@@ -1641,12 +1647,18 @@ impl TurboTasksBackend { ...@@ -1641,12 +1647,18 @@ impl TurboTasksBackend {
// the storage entry already initialized (restored flags set). // the storage entry already initialized (restored flags set).
self.storage self.storage
.initialize_new_task(task_id, Some(task_type.clone())); .initialize_new_task(task_id, Some(task_type.clone()));
// insert() consumes e, releasing the shard write lock. entry.insert((task_type, task_id));
e.insert(task_type, task_id); (task_id, true)
}
},
);
// The entry closure has returned, so the task_cache shard lock is released before
// cache tracking or aggregation updates can re-enter the backend.
if created {
self.track_cache_miss_by_fn(native_fn); self.track_cache_miss_by_fn(native_fn);
// Update the aggregation number before connecting the child // Update the aggregation number before connecting the child. We don't need this on
// We don't need this on any of the task recovery paths above because the // recovery paths because the aggregation number will already be set.
// aggregation number will already be set.
if is_root { if is_root {
AggregationUpdateQueue::run( AggregationUpdateQueue::run(
AggregationUpdateJob::UpdateAggregationNumber { AggregationUpdateJob::UpdateAggregationNumber {
...@@ -1666,11 +1678,12 @@ impl TurboTasksBackend { ...@@ -1666,11 +1678,12 @@ impl TurboTasksBackend {
}, },
&mut ctx, &mut ctx,
); );
};
task_id
} }
} else {
self.track_cache_hit_by_fn(native_fn);
} }
task_id
}; };
operation::ConnectChildOperation::run(parent_task, task_id, ctx); operation::ConnectChildOperation::run(parent_task, task_id, ctx);
......
...@@ -9,8 +9,7 @@ use std::{ ...@@ -9,8 +9,7 @@ use std::{
}, },
}; };
use dashmap::SharedValue; use hashbrown::hash_table;
use hashbrown::raw::RawIntoIter;
use thread_local::ThreadLocal; use thread_local::ThreadLocal;
use tracing::span::Id; use tracing::span::Id;
use turbo_bincode::TurboBincodeBuffer; use turbo_bincode::TurboBincodeBuffer;
...@@ -24,8 +23,8 @@ use crate::{ ...@@ -24,8 +23,8 @@ use crate::{
database::key_value_database::KeySpace, database::key_value_database::KeySpace,
utils::{ utils::{
dash_map_drop_contents::drop_contents, dash_map_drop_contents::drop_contents,
dash_map_multi::{RefMut, get_multiple_mut}, dash_map_entry::{TryLockAndRemove, try_lock_and_remove},
dash_map_raw_entry::{TryLockAndRemove, try_lock_and_remove}, dash_map_multi::{RefMut, get_disjoint_mut},
}, },
}; };
...@@ -340,25 +339,18 @@ impl Storage { ...@@ -340,25 +339,18 @@ impl Storage {
let work = { let work = {
let mut shard_guard = shard.write(); let mut shard_guard = shard.write();
if drain_entries { if drain_entries {
// SAFETY: shard_guard outlives the iterator and we hold it for the whole scan. shard_guard.retain(|(key, task)| {
for bucket in unsafe { shard_guard.iter() } { let modified_task = task.flags.any_modified();
// Read the key and modified flag, then drop the borrow before any erase.
// SAFETY: the guard outlives the bucket reference.
let (key, modified_task) = {
let (key, shared_value) = unsafe { bucket.as_ref() };
(*key, shared_value.get().flags.any_modified())
};
if modified_task { if modified_task {
debug_assert!( debug_assert!(
!key.is_transient(), !key.is_transient(),
"found a modified transient task: {key:?}" "found a modified transient task: {key:?}"
); );
} else {
// Unmodified entries are not part of the snapshot. Erase and free them
// now so the table we move out below holds only modified entries.
unsafe { shard_guard.erase(bucket) };
}
} }
// Unmodified entries are not part of the snapshot. Remove and free them
// now so the table we move out below holds only modified entries.
modified_task
});
if shard_guard.is_empty() { if shard_guard.is_empty() {
// The shard held only unmodified entries, which we've now erased and freed. // The shard held only unmodified entries, which we've now erased and freed.
// No iterator is created for an empty shard. // No iterator is created for an empty shard.
...@@ -369,15 +361,12 @@ impl Storage { ...@@ -369,15 +361,12 @@ impl Storage {
ShardWork::Drain(std::mem::take(&mut *shard_guard).into_iter()) ShardWork::Drain(std::mem::take(&mut *shard_guard).into_iter())
} else { } else {
let mut modified = Vec::with_capacity(modified_count as usize); let mut modified = Vec::with_capacity(modified_count as usize);
// SAFETY: shard_guard outlives the iterator and we hold it for the whole scan. for (key, task) in shard_guard.iter() {
for bucket in unsafe { shard_guard.iter() } {
// SAFETY: the guard outlives the bucket reference.
let (key, shared_value) = unsafe { bucket.as_ref() };
// Only check modified flags — transient tasks never have modified flags set // Only check modified flags — transient tasks never have modified flags set
// (track_modification guards against it), so this naturally excludes them. // (track_modification guards against it), so this naturally excludes them.
// new_task always comes with modified flags (set_persistent_task_type calls // new_task always comes with modified flags (set_persistent_task_type calls
// track_modification), so any_modified() is sufficient. // track_modification), so any_modified() is sufficient.
if shared_value.get().flags.any_modified() { if task.flags.any_modified() {
debug_assert!( debug_assert!(
!key.is_transient(), !key.is_transient(),
"found a modified transient task: {key:?}" "found a modified transient task: {key:?}"
...@@ -473,7 +462,7 @@ impl Storage { ...@@ -473,7 +462,7 @@ impl Storage {
let snap_shard = &snapshot_shards[shard_idx]; let snap_shard = &snapshot_shards[shard_idx];
// Acquire in documented order: map first, snapshots second. // Acquire in documented order: map first, snapshots second.
let map_guard = map_shard.write(); let mut map_guard = map_shard.write();
let mut snap_guard = snap_shard.write(); let mut snap_guard = snap_shard.write();
for (key, _) in snap_guard.drain() { for (key, _) in snap_guard.drain() {
...@@ -482,11 +471,8 @@ impl Storage { ...@@ -482,11 +471,8 @@ impl Storage {
// through `self.map.get_mut`, which would attempt to re-acquire this shard's // through `self.map.get_mut`, which would attempt to re-acquire this shard's
// write lock and would also obscure the pairing. // write lock and would also obscure the pairing.
let hash = self.map.hasher().hash_one(key); let hash = self.map.hasher().hash_one(key);
if let Some(bucket) = map_guard.find(hash, |(k, _)| *k == key) { if let Some((_, task)) = map_guard.find_mut(hash, |(k, _)| *k == key) {
// SAFETY: We hold `map_shard`'s write lock for the duration of this self.promote_during_snapshot_flags(task, shard_idx);
// access, so the bucket pointer is valid and no other thread can alias it.
let (_, shared_value) = unsafe { bucket.as_mut() };
self.promote_during_snapshot_flags(shared_value.get_mut(), shard_idx);
} }
} }
// If we are saving a non-trivial amount of memory just clear it out. // If we are saving a non-trivial amount of memory just clear it out.
...@@ -531,7 +517,7 @@ impl Storage { ...@@ -531,7 +517,7 @@ impl Storage {
key1: TaskId, key1: TaskId,
key2: TaskId, key2: TaskId,
) -> (StorageWriteGuard<'_>, StorageWriteGuard<'_>) { ) -> (StorageWriteGuard<'_>, StorageWriteGuard<'_>) {
let (a, b) = get_multiple_mut(&self.map, key1, key2, || Box::new(TaskStorage::new())); let (a, b) = get_disjoint_mut(&self.map, key1, key2, || Box::new(TaskStorage::new()));
( (
StorageWriteGuard { StorageWriteGuard {
storage: self, storage: self,
...@@ -587,21 +573,18 @@ impl Storage { ...@@ -587,21 +573,18 @@ impl Storage {
// avoid a lock cycle with get_or_create_persistent_task, which takes task_cache // avoid a lock cycle with get_or_create_persistent_task, which takes task_cache
// before map. Allocated lazily on first conflict. // before map. Allocated lazily on first conflict.
let mut deferred_task_cache_removals: Vec<CachedTaskTypeArc> = Vec::new(); let mut deferred_task_cache_removals: Vec<CachedTaskTypeArc> = Vec::new();
// SAFETY: We hold the write lock for the duration of iteration. shard.retain(|(task_id, task)| {
for bucket in unsafe { shard.iter() } {
// SAFETY: The write lock guard outlives the bucket reference.
let (task_id, task) = unsafe { bucket.as_mut() };
if task_id.is_transient() { if task_id.is_transient() {
evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1; evicted.unevictable_reasons[UnevictableReason::Transient.index()] += 1;
continue; return true;
} }
let (key_evictability, value_evictability) = task.get().evictability(); let (key_evictability, value_evictability) = task.evictability();
match key_evictability { match key_evictability {
KeyEvictability::Evictable => { KeyEvictability::Evictable => {
// The task type is persisted to backing storage (new_task = false), // The task type is persisted to backing storage (new_task = false),
// so task_cache is a pure perf cache. Remove it now; it will be // so task_cache is a pure perf cache. Remove it now; it will be
// re-populated by task_by_type() on the next cache miss. // re-populated by task_by_type() on the next cache miss.
let task_type = task.get().get_persistent_task_type().unwrap(); let task_type = task.get_persistent_task_type().unwrap();
// Only try to acquire the lock, if we cannot just remove at the end // Only try to acquire the lock, if we cannot just remove at the end
// Because `get_or_create_task` acquires 'task_cache' then `storage.map` and // Because `get_or_create_task` acquires 'task_cache' then `storage.map` and
// we do the opposite we need to be defensive here. Attempting here is just // we do the opposite we need to be defensive here. Attempting here is just
...@@ -624,12 +607,10 @@ impl Storage { ...@@ -624,12 +607,10 @@ impl Storage {
} }
match value_evictability { match value_evictability {
ValueEvictability::Evictable { meta, data } => { ValueEvictability::Evictable { meta, data } => {
match task.get_mut().drop_partial(data, meta) { match task.drop_partial(data, meta) {
DropPartialOutcome::Empty => { DropPartialOutcome::Empty => {
unsafe {
shard.erase(bucket);
}
evicted.full += 1; evicted.full += 1;
return false;
} }
DropPartialOutcome::HasResidue => { DropPartialOutcome::HasResidue => {
if data && meta { if data && meta {
...@@ -647,7 +628,8 @@ impl Storage { ...@@ -647,7 +628,8 @@ impl Storage {
evicted.unevictable_reasons[reason.index()] += 1; evicted.unevictable_reasons[reason.index()] += 1;
} }
} }
} true
});
// Shrink the shard if it's less than half full, to reclaim slack capacity // Shrink the shard if it's less than half full, to reclaim slack capacity
// after bulk evictions. We already hold the write lock, so this is free // after bulk evictions. We already hold the write lock, so this is free
// from a locking perspective. TaskId hashing is cheap (it's just an integer). // from a locking perspective. TaskId hashing is cheap (it's just an integer).
...@@ -932,7 +914,7 @@ enum ShardWork { ...@@ -932,7 +914,7 @@ enum ShardWork {
/// (modified-only) shard table out of the map. The iterator owns that table and drains it /// (modified-only) shard table out of the map. The iterator owns that table and drains it
/// directly, freeing each task box as it is serialized. No second map lookup, no flag /// directly, freeing each task box as it is serialized. No second map lookup, no flag
/// bookkeeping (the whole map is discarded right after this snapshot). /// bookkeeping (the whole map is discarded right after this snapshot).
Drain(RawIntoIter<(TaskId, SharedValue<Box<TaskStorage>>)>), Drain(hash_table::IntoIter<(TaskId, Box<TaskStorage>)>),
} }
pub struct SnapshotShard<'l, P> { pub struct SnapshotShard<'l, P> {
...@@ -1016,7 +998,6 @@ where ...@@ -1016,7 +998,6 @@ where
// bookkeeping the normal path does, since the entire map is discarded right after // bookkeeping the normal path does, since the entire map is discarded right after
// this snapshot. // this snapshot.
let (task_id, inner) = entries.next()?; let (task_id, inner) = entries.next()?;
let inner = inner.into_inner();
Some(serialize_task(task_id, &inner)) Some(serialize_task(task_id, &inner))
// we don't need to update any bits because everything is getting dropped. // we don't need to update any bits because everything is getting dropped.
} }
......
use std::hash::{BuildHasher, Hash}; use std::hash::{BuildHasher, Hash};
use crossbeam_utils::CachePadded; use crossbeam_utils::CachePadded;
use dashmap::{DashMap, RwLock, RwLockWriteGuard, SharedValue}; use dashmap::{DashMap, RwLock};
use hashbrown::raw::{Bucket, InsertSlot, RawTable}; use hashbrown::{HashTable, hash_table};
/// The type of a single shard inside a [`DashMap`]. /// The type of a single shard inside a [`DashMap`].
/// ///
/// `dashmap::HashMap<K, V>` is a private alias for `RawTable<(K, SharedValue<V>)>`. /// `dashmap::HashMap<K, V>` is a private alias for `HashTable<(K, V)>`.
pub type Shard<K, V> = CachePadded<RwLock<RawTable<(K, SharedValue<V>)>>>; pub type Shard<K, V> = CachePadded<RwLock<HashTable<(K, V)>>>;
/// Returns a reference to the shard that owns the given pre-computed hash, /// Returns a reference to the shard that owns the given pre-computed hash,
/// without locking anything. /// without locking anything.
/// ///
/// Pass the returned reference to [`raw_get_in_shard`] and /// Pass the returned reference to [`get_in_shard`] and
/// [`raw_entry_in_shard`] so that the shard is only located once even when a /// [`with_entry_in_shard`] so that the shard is only located once even when a
/// read-lock miss is followed by a write-lock retry. /// read-lock miss is followed by a write-lock retry.
pub fn get_shard<K: Eq + Hash, V, S: BuildHasher + Clone>( pub fn get_shard<K: Eq + Hash, V, S: BuildHasher + Clone>(
map: &DashMap<K, V, S>, map: &DashMap<K, V, S>,
...@@ -25,43 +25,35 @@ pub fn get_shard<K: Eq + Hash, V, S: BuildHasher + Clone>( ...@@ -25,43 +25,35 @@ pub fn get_shard<K: Eq + Hash, V, S: BuildHasher + Clone>(
/// Read-only heterogeneous lookup using a pre-located shard reference. /// Read-only heterogeneous lookup using a pre-located shard reference.
/// Returns `Some(value)` on hit, `None` on miss. Uses only a read lock. /// Returns `Some(value)` on hit, `None` on miss. Uses only a read lock.
pub fn raw_get_in_shard<K: Eq + Hash, V: Copy>( pub fn get_in_shard<K: Eq + Hash, V: Copy>(
shard: &Shard<K, V>, shard: &Shard<K, V>,
hash: u64, hash: u64,
eq: impl Fn(&K) -> bool, eq: impl Fn(&K) -> bool,
) -> Option<V> { ) -> Option<V> {
let guard = shard.read(); let guard = shard.read();
// Safety: We have a read lock on the shard. guard.find(hash, |(k, _v)| eq(k)).map(|(_k, v)| *v)
guard
.find(hash, |(k, _v)| eq(k))
.map(|bucket| *unsafe { bucket.as_ref() }.1.get())
} }
/// Write-lock entry lookup using a pre-located shard reference and /// Runs `then` with the native Hashbrown entry for a pre-located DashMap shard.
/// heterogeneous equality.
/// ///
/// Takes a pre-located `shard` (from [`get_shard`]) and `hash` so the shard is /// The shard write lock is held only for the duration of `then`. The caller
/// not located a second time on a read-miss / write-retry path. /// controls the precise point where the entry is consumed and the lock is
pub fn raw_entry_in_shard<'l, K: Eq + Hash, V, S: BuildHasher + Clone>( /// released by returning from the closure.
shard: &'l Shard<K, V>, pub fn with_entry_in_shard<K: Eq + Hash, V, S: BuildHasher + Clone, Q: ?Sized, R>(
shard: &Shard<K, V>,
map_hasher: &S, map_hasher: &S,
hash: u64, hash: u64,
eq: impl Fn(&K) -> bool, query: &mut Q,
) -> RawEntry<'l, K, V> { eq: impl Fn(&K, &Q) -> bool,
then: impl FnOnce(hash_table::Entry<'_, (K, V)>, &mut Q) -> R,
) -> R {
let mut guard = shard.write(); let mut guard = shard.write();
let result = let entry = guard.entry(
guard.find_or_find_insert_slot(hash, |(k, _v)| eq(k), |(k, _v)| map_hasher.hash_one(k));
match result {
Ok(bucket) => RawEntry::Occupied(OccupiedEntry {
bucket,
shard: guard,
}),
Err(insert_slot) => RawEntry::Vacant(VacantEntry {
hash, hash,
insert_slot, |(k, _v)| eq(k, query),
shard: guard, |(k, _v)| map_hasher.hash_one(k),
}), );
} then(entry, query)
} }
/// Outcome of [`try_lock_and_remove`]. /// Outcome of [`try_lock_and_remove`].
...@@ -95,49 +87,11 @@ pub fn try_lock_and_remove< ...@@ -95,49 +87,11 @@ pub fn try_lock_and_remove<
let Some(mut shard) = map.shards()[shard_idx].try_write() else { let Some(mut shard) = map.shards()[shard_idx].try_write() else {
return TryLockAndRemove::WouldBlock; return TryLockAndRemove::WouldBlock;
}; };
// SAFETY: we hold the write lock for the duration of the find/erase. match shard.find_entry(hash, |(k, _v)| k.as_ref() == key) {
match shard.find(hash, |(k, _v)| k.as_ref() == key) { Ok(entry) => {
Some(bucket) => { entry.remove();
unsafe { shard.erase(bucket) };
TryLockAndRemove::Removed TryLockAndRemove::Removed
} }
None => TryLockAndRemove::NotFound, Err(_) => TryLockAndRemove::NotFound,
}
}
pub enum RawEntry<'l, K, V> {
Occupied(OccupiedEntry<'l, K, V>),
Vacant(VacantEntry<'l, K, V>),
}
pub struct OccupiedEntry<'l, K, V> {
bucket: Bucket<(K, SharedValue<V>)>,
#[allow(dead_code, reason = "kept to ensure the lock lives long enough")]
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> OccupiedEntry<'l, K, V> {
pub fn get(&self) -> &V {
// Safety: We have a write lock on the shard, so no other references to the value can
// exist.
unsafe { self.bucket.as_ref().1.get() }
}
}
pub struct VacantEntry<'l, K, V> {
hash: u64,
insert_slot: InsertSlot,
shard: RwLockWriteGuard<'l, RawTable<(K, SharedValue<V>)>>,
}
impl<'l, K, V> VacantEntry<'l, K, V> {
pub fn insert(mut self, key: K, value: V) {
let shared_value = SharedValue::new(value);
// Safety: The insert slot is valid and the map has not been modified since we obtained it
// (we hold the write lock).
unsafe {
self.shard
.insert_in_slot(self.hash, self.insert_slot, (key, shared_value));
}
} }
} }
...@@ -2,23 +2,25 @@ use std::{ ...@@ -2,23 +2,25 @@ use std::{
hash::{BuildHasher, Hash}, hash::{BuildHasher, Hash},
marker::PhantomData, marker::PhantomData,
ops::{Deref, DerefMut}, ops::{Deref, DerefMut},
ptr::NonNull,
sync::Arc, sync::Arc,
}; };
use dashmap::{DashMap, RwLockWriteGuard, SharedValue}; use dashmap::{DashMap, RawRwLock};
use hashbrown::raw::{Bucket, RawTable}; use hashbrown::HashTable;
use parking_lot::lock_api::RwLockWriteGuard;
type RwLockWriteTableGuard<'a, K, V> = RwLockWriteGuard<'a, RawTable<(K, SharedValue<V>)>>; type RwLockWriteTableGuard<'a, K, V> = RwLockWriteGuard<'a, RawRwLock, HashTable<(K, V)>>;
pub enum RefMut<'a, K, V> { pub enum RefMut<'a, K, V> {
Base(dashmap::mapref::one::RefMut<'a, K, V>), Base(dashmap::mapref::one::RefMut<'a, K, V>),
Simple { Simple {
_guard: RwLockWriteTableGuard<'a, K, V>, _guard: RwLockWriteTableGuard<'a, K, V>,
bucket: Bucket<(K, SharedValue<V>)>, entry: NonNull<(K, V)>,
}, },
Shared { Shared {
_guard: Arc<RwLockWriteTableGuard<'a, K, V>>, _guard: Arc<RwLockWriteTableGuard<'a, K, V>>,
bucket: Bucket<(K, SharedValue<V>)>, entry: NonNull<(K, V)>,
// Ensures that RefMut is !Send, preventing holding RefMut across .await points in async // Ensures that RefMut is !Send, preventing holding RefMut across .await points in async
// code, which can cause deadlocks. See safety comment on `unsafe impl Sync for RefMut` // code, which can cause deadlocks. See safety comment on `unsafe impl Sync for RefMut`
// below. // below.
...@@ -34,13 +36,13 @@ pub enum RefMut<'a, K, V> { ...@@ -34,13 +36,13 @@ pub enum RefMut<'a, K, V> {
// while every other tokio worker piles up trying to take the same lock — leaving no thread free // while every other tokio worker piles up trying to take the same lock — leaving no thread free
// to poll the parked future. Marking the type `!Send` makes the borrow checker reject those call // to poll the parked future. Marking the type `!Send` makes the borrow checker reject those call
// sites at compile time. // sites at compile time.
// SAFETY (Sync): `RefMut` contains a raw `Bucket` pointer into a `DashMap` shard's `RawTable`. // SAFETY (Sync): `RefMut` contains a non-null pointer into a `DashMap` shard's `HashTable`.
// Sharing `&RefMut` is safe because: // Sharing `&RefMut` is safe because:
// - `Simple` variant: The `Bucket` is accessed under an exclusive `RwLockWriteGuard` on a single // - `Simple` variant: The entry is accessed under an exclusive `RwLockWriteGuard` on a single
// shard. The guard provides exclusive access to all data in that shard. // shard. The guard provides exclusive access to all data in that shard.
// - `Shared` variant: The `Bucket` is accessed under an `Arc<RwLockWriteGuard>`. The // - `Shared` variant: The entry is accessed under an `Arc<RwLockWriteGuard>`. The
// `get_multiple_mut` function asserts that bucket pointers do not alias, so each `RefMut` has // `get_disjoint_mut` function validates that the keys differ before obtaining both references
// exclusive access to its bucket even when sharing a guard. // through `HashTable::get_many_unchecked_mut`.
// - `K: Sync + V: Sync` bounds ensure the key and value types are safe to share across threads. // - `K: Sync + V: Sync` bounds ensure the key and value types are safe to share across threads.
unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for RefMut<'_, K, V> {} unsafe impl<K: Eq + Hash + Sync, V: Sync> Sync for RefMut<'_, K, V> {}
...@@ -60,14 +62,10 @@ impl<K: Eq + Hash, V> RefMut<'_, K, V> { ...@@ -60,14 +62,10 @@ impl<K: Eq + Hash, V> RefMut<'_, K, V> {
pub fn pair(&self) -> (&K, &V) { pub fn pair(&self) -> (&K, &V) {
match self { match self {
RefMut::Base(r) => r.pair(), RefMut::Base(r) => r.pair(),
RefMut::Simple { bucket, .. } | RefMut::Shared { bucket, .. } => { RefMut::Simple { entry, .. } | RefMut::Shared { entry, .. } => {
// SAFETY: // SAFETY: The entry remains valid while the shard write guard is held.
// - The bucket is still valid, as we're holding a write guard on the shard let entry = unsafe { entry.as_ref() };
// - These bucket pointers are convertible to references (&entry.0, &entry.1)
//
// https://doc.rust-lang.org/std/ptr/index.html#pointer-to-reference-conversion
let entry = unsafe { bucket.as_ref() };
(&entry.0, entry.1.get())
} }
} }
} }
...@@ -75,13 +73,13 @@ impl<K: Eq + Hash, V> RefMut<'_, K, V> { ...@@ -75,13 +73,13 @@ impl<K: Eq + Hash, V> RefMut<'_, K, V> {
pub fn pair_mut(&mut self) -> (&K, &mut V) { pub fn pair_mut(&mut self) -> (&K, &mut V) {
match self { match self {
RefMut::Base(r) => r.pair_mut(), RefMut::Base(r) => r.pair_mut(),
RefMut::Simple { bucket, .. } | RefMut::Shared { bucket, .. } => { RefMut::Simple { entry, .. } | RefMut::Shared { entry, .. } => {
// SAFETY: Same as above in `pair`, plus aliasing is prevented via: // SAFETY: Same as above in `pair`, plus aliasing is prevented via:
// 1. The lifetime of `&mut self`. // 1. The lifetime of `&mut self`.
// 2. `Simple` values come from separate shards (no aliasing possible) // 2. `Simple` values come from separate shards (no aliasing possible).
// 3. `Shared` values are asserted in `get_multiple_mut` to have unique pointers // 3. `Shared` values were validated as disjoint before the pointers were created.
let entry = unsafe { bucket.as_mut() }; let entry = unsafe { entry.as_mut() };
(&entry.0, entry.1.get_mut()) (&entry.0, &mut entry.1)
} }
} }
} }
...@@ -110,7 +108,7 @@ where ...@@ -110,7 +108,7 @@ where
} }
} }
pub fn get_multiple_mut<K, V>( pub fn get_disjoint_mut<K, V>(
map: &DashMap<K, V, impl BuildHasher + Clone>, map: &DashMap<K, V, impl BuildHasher + Clone>,
key1: K, key1: K,
key2: K, key2: K,
...@@ -135,49 +133,43 @@ where ...@@ -135,49 +133,43 @@ where
let shards = map.shards(); let shards = map.shards();
if s1 == s2 { if s1 == s2 {
let mut guard = shards[s1].write(); // Equal keys would resolve to a single entry below. This must be a release-mode assertion
// because the unchecked lookup relies on it for memory safety.
// we need to call `find_or_find_insert_slot` to avoid overwriting existing entries, but we assert!(
// can't use the returned bucket until after we get `bucket2` (below) key1 != key2,
let _ = guard "`get_disjoint_mut` was called with equal keys, which breaks mutable referencing rules"
.find_or_find_insert_slot(h1, eq1, hash_entry) );
.unwrap_or_else(|slot| unsafe {
// SAFETY: This slot was previously returned by `find_or_find_insert_slot`, and no
// mutation of the table has occurred since that call.
guard.insert_in_slot(h1, slot, (key1.clone(), SharedValue::new(insert_with())))
});
let bucket2 = guard let mut guard = shards[s1].write();
.find_or_find_insert_slot(h2, eq2, hash_entry)
.unwrap_or_else(|slot| unsafe {
// SAFETY: See previous call above
guard.insert_in_slot(h2, slot, (key2.clone(), SharedValue::new(insert_with())))
});
// Getting `bucket2` might invalidate the bucket pointer of the first entry, *even if no if guard.find(h1, eq1).is_none() {
// insert happens* as `RawTable::find_or_find_insert_slot` will *sometimes* resize the guard.insert_unique(h1, (key1.clone(), insert_with()), hash_entry);
// table, as it unconditionally reserves space for a potential insertion. }
let bucket1 = guard.find(h1, eq1).expect( if guard.find(h2, eq2).is_none() {
"failed to find bucket of previously inserted item, is the hash or eq implementation \ guard.insert_unique(h2, (key2.clone(), insert_with()), hash_entry);
incorrect?", }
);
// this assertion is needed for memory safety reasons // SAFETY: `key1 != key2` was asserted above. Since `K: Eq`, the two equality closures
assert!( // cannot select the same entry, even when the hashes collide.
!std::ptr::eq(bucket1.as_ptr(), bucket2.as_ptr()), let [entry1, entry2] =
"`get_multiple_mut` was called with equal keys, which breaks mutable referencing rules" unsafe {
); guard.get_many_unchecked_mut([h1, h2], |index, entry| {
if index == 0 { eq1(entry) } else { eq2(entry) }
})
};
let entry1 = NonNull::from(entry1.expect("the first entry was inserted above"));
let entry2 = NonNull::from(entry2.expect("the second entry was inserted above"));
let guard = Arc::new(guard); let guard = Arc::new(guard);
( (
RefMut::Shared { RefMut::Shared {
_guard: guard.clone(), _guard: guard.clone(),
bucket: bucket1, entry: entry1,
phantom: PhantomData, phantom: PhantomData,
}, },
RefMut::Shared { RefMut::Shared {
_guard: guard, _guard: guard,
bucket: bucket2, entry: entry2,
phantom: PhantomData, phantom: PhantomData,
}, },
) )
...@@ -197,27 +189,31 @@ where ...@@ -197,27 +189,31 @@ where
} }
}; };
let bucket1 = guard1 if guard1.find(h1, eq1).is_none() {
.find_or_find_insert_slot(h1, eq1, hash_entry) guard1.insert_unique(h1, (key1.clone(), insert_with()), hash_entry);
.unwrap_or_else(|slot| unsafe { }
// SAFETY: See first insert_in_slot call if guard2.find(h2, eq2).is_none() {
guard1.insert_in_slot(h1, slot, (key1.clone(), SharedValue::new(insert_with()))) guard2.insert_unique(h2, (key2.clone(), insert_with()), hash_entry);
}); }
let bucket2 = guard2 let entry1 = NonNull::from(
.find_or_find_insert_slot(h2, eq2, hash_entry) guard1
.unwrap_or_else(|slot| unsafe { .find_mut(h1, eq1)
// SAFETY: See first insert_in_slot call .expect("the first entry was inserted"),
guard2.insert_in_slot(h2, slot, (key2.clone(), SharedValue::new(insert_with()))) );
}); let entry2 = NonNull::from(
guard2
.find_mut(h2, eq2)
.expect("the second entry was inserted"),
);
( (
RefMut::Simple { RefMut::Simple {
_guard: guard1, _guard: guard1,
bucket: bucket1, entry: entry1,
}, },
RefMut::Simple { RefMut::Simple {
_guard: guard2, _guard: guard2,
bucket: bucket2, entry: entry2,
}, },
) )
} }
...@@ -250,7 +246,7 @@ mod tests { ...@@ -250,7 +246,7 @@ mod tests {
for indices in indices { for indices in indices {
s.spawn(|| { s.spawn(|| {
for i in indices { for i in indices {
let (mut a, mut b) = get_multiple_mut(map, i, i + 1, || 0); let (mut a, mut b) = get_disjoint_mut(map, i, i + 1, || 0);
*a += 1; *a += 1;
*b += 1; *b += 1;
} }
......
pub mod dash_map_drop_contents; pub mod dash_map_drop_contents;
pub mod dash_map_entry;
pub mod dash_map_multi; pub mod dash_map_multi;
pub mod dash_map_raw_entry;
pub mod markdown_table; pub mod markdown_table;
pub mod ptr_eq_arc; pub mod ptr_eq_arc;
pub mod shard_amount; pub mod shard_amount;
......
...@@ -19,7 +19,7 @@ bench = false ...@@ -19,7 +19,7 @@ bench = false
anyhow = { workspace = true } anyhow = { workspace = true }
either = { workspace = true } either = { workspace = true }
flate2 = { workspace = true } flate2 = { workspace = true }
hashbrown = { workspace = true, features = ["raw"] } hashbrown = { workspace = true, features = ["raw-entry"] }
indexmap = { workspace = true, features = ["serde"] } indexmap = { workspace = true, features = ["serde"] }
itertools = { workspace = true } itertools = { workspace = true }
postcard = { workspace = true } postcard = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment