Unverified Commit 2ff2a7aa authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Add `turbo-frozenmap` crate with `FrozenMap` and `FrozenSet` implementations (#87042)

This implements a `FrozenMap` type as a flat sorted boxed slice of `(K, V)` tuples.

This is intended to be used with `TaskInput`s and `turbo_task::Value`s, where the data is stored immutably, and where we already try to call `shrink_to_fit`.

This map is smaller than `HashMap` or `BTreeMap`, and provides `O(log n)` lookups with excellent cache locality (so it's probably faster than `HashMap` in most cases).

Since the map is sorted, equality doesn't care about order, so this is a better implementation of `Eq` for our use-cases than what `IndexMap` provides.

It has a variety of ways it can be constructed: A `HashMap`, a `Vec<(K, V)>`, a `BTreeMap`, etc. There are pros and cons of all of these:
- `Vec<(K, V)>` is good if you don't need lookups until after freezing, and you don't expect many duplicates, since sorting at the end is cheaper than maintaining a map.
- A `BTreeMap` is good if you need a real map, because we can avoid a sort at the end.
- `HashMap`/`IndexMap` are okay if you're getting this data structure from somewhere else.

`IndexSet` is also provided as a thin wrapper around `IndexMap`.

## Remaining Issues

This can't be used with `ResolvedVc` as keys, because `ResolvedVc` does not implement `Ord`. I think it should be okay to do that because it's no worse than our current implementations of `Eq` and `Hash`, but I know this is contentious, so I'm leaving it out of this PR.

## Implementation Strategy

I used Claude Code to write most of this:

- Provided it rust's stdlib implementation of BTreeMap.
- Asked it to extract the public interface into a separate file, excluding nightly-only features.
- Asked it to remove all `&mut` methods.
- Asked it to implement `FrozenMap` with the same API.

Then I did a bunch of manual cleanup:
- Replaced a bunch of manual trait implementations with derives.
- Added bincode and serde traits.
- Added various `turbo-tasks` trait implementations.
- Added some methods/impls for getting the underlying slice.
- Optimized a bunch of the constructors.

Then I asked Claude Code to implement the Set version, and did another round of cleanup:

- Removed bunch of unsafe code where it was trying to transmute `(T, ())` to `T`. The memory layout of a tuple is technically undefined, so this wasn't safe, and the APIs it was trying to create doing this weren't really needed either.
- Replaced some of the iterator newtype wrappers with simpler type aliases. The newtype was overkill for our use-case.
parent ae0603c4
......@@ -9129,6 +9129,15 @@ dependencies = [
"turbo-tasks",
]
[[package]]
name = "turbo-frozenmap"
version = "0.1.0"
dependencies = [
"bincode 2.0.1",
"indexmap 2.9.0",
"serde",
]
[[package]]
name = "turbo-persistence"
version = "0.1.0"
......@@ -9242,6 +9251,7 @@ dependencies = [
"triomphe 0.1.12",
"turbo-bincode",
"turbo-dyn-eq-hash",
"turbo-frozenmap",
"turbo-rcstr",
"turbo-tasks-hash",
"turbo-tasks-macros",
......
......@@ -294,13 +294,12 @@ next-taskless = { path = "crates/next-taskless" }
# Turbopack
auto-hash-map = { path = "turbopack/crates/turbo-tasks-auto-hash-map" }
turbo-bincode = { path = "turbopack/crates/turbo-bincode" }
turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
turbo-rcstr = { path = "turbopack/crates/turbo-rcstr" }
turbo-dyn-eq-hash = { path = "turbopack/crates/turbo-dyn-eq-hash" }
turbo-esregex = { path = "turbopack/crates/turbo-esregex" }
turbo-frozenmap = { path = "turbopack/crates/turbo-frozenmap" }
turbo-persistence = { path = "turbopack/crates/turbo-persistence" }
turbo-unix-path = { path = "turbopack/crates/turbo-unix-path" }
turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
turbo-prehash = { path = "turbopack/crates/turbo-prehash" }
turbo-rcstr = { path = "turbopack/crates/turbo-rcstr" }
turbo-tasks = { path = "turbopack/crates/turbo-tasks" }
turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" }
turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" }
......@@ -309,7 +308,9 @@ turbo-tasks-fetch = { path = "turbopack/crates/turbo-tasks-fetch" }
turbo-tasks-fs = { path = "turbopack/crates/turbo-tasks-fs" }
turbo-tasks-hash = { path = "turbopack/crates/turbo-tasks-hash" }
turbo-tasks-macros = { path = "turbopack/crates/turbo-tasks-macros" }
turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false }
turbo-tasks-testing = { path = "turbopack/crates/turbo-tasks-testing" }
turbo-unix-path = { path = "turbopack/crates/turbo-unix-path" }
turbopack = { path = "turbopack/crates/turbopack" }
turbopack-bench = { path = "turbopack/crates/turbopack-bench" }
turbopack-nodejs = { path = "turbopack/crates/turbopack-nodejs" }
......
[package]
name = "turbo-frozenmap"
version = "0.1.0"
edition = "2024"
license = "MIT"
[dependencies]
bincode = { workspace = true }
indexmap = { workspace = true }
serde = { workspace = true }
[dev-dependencies]
[lints]
workspace = true
//! A frozen (immutable) ordered map and set implementation.
pub mod map;
pub mod set;
pub use crate::{map::FrozenMap, set::FrozenSet};
This diff is collapsed.
This diff is collapsed.
......@@ -51,6 +51,7 @@ tracing = { workspace = true }
triomphe = { workspace = true, features = ["unsize", "unstable"] }
turbo-bincode = { workspace = true }
turbo-dyn-eq-hash = { workspace = true }
turbo-frozenmap = { workspace = true }
turbo-rcstr = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-tasks-macros = { workspace = true }
......
......@@ -55,11 +55,13 @@ macro_rules! impl_auto_marker_trait {
unsafe impl<T: $trait, S, const I: usize> $trait for ::auto_hash_map::AutoSet<T, S, I> {}
unsafe impl<T: $trait> $trait for ::std::collections::BTreeSet<T> {}
unsafe impl<T: $trait, S> $trait for ::indexmap::IndexSet<T, S> {}
unsafe impl<T: $trait> $trait for ::turbo_frozenmap::FrozenSet<T> {}
unsafe impl<K: $trait, V: $trait, S> $trait for ::std::collections::HashMap<K, V, S> {}
unsafe impl<K: $trait, V: $trait, S, const I: usize> $trait
for ::auto_hash_map::AutoMap<K, V, S, I> {}
unsafe impl<K: $trait, V: $trait> $trait for ::std::collections::BTreeMap<K, V> {}
unsafe impl<K: $trait, V: $trait, S> $trait for ::indexmap::IndexMap<K, V, S> {}
unsafe impl<K: $trait, V: $trait> $trait for ::turbo_frozenmap::FrozenMap<K, V> {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::boxed::Box<T> {}
unsafe impl<T: $trait + ?Sized> $trait for ::std::sync::Arc<T> {}
unsafe impl<B: $trait + ::std::borrow::ToOwned + ?Sized> $trait
......
......@@ -16,6 +16,7 @@ use bincode::{
error::{DecodeError, EncodeError},
};
use either::Either;
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr;
// This import is necessary for derive macros to work, as their expansion refers to the crate
......@@ -284,11 +285,60 @@ where
T: TaskInput + Ord,
{
async fn resolve_input(&self) -> Result<Self> {
let mut new_map = BTreeSet::new();
let mut new_set = BTreeSet::new();
for value in self {
new_map.insert(TaskInput::resolve_input(value).await?);
new_set.insert(TaskInput::resolve_input(value).await?);
}
Ok(new_map)
Ok(new_set)
}
fn is_resolved(&self) -> bool {
self.iter().all(TaskInput::is_resolved)
}
fn is_transient(&self) -> bool {
self.iter().any(TaskInput::is_transient)
}
}
impl<K, V> TaskInput for FrozenMap<K, V>
where
K: TaskInput + Ord + 'static,
V: TaskInput + 'static,
{
async fn resolve_input(&self) -> Result<Self> {
let mut new_entries = Vec::with_capacity(self.len());
for (k, v) in self {
new_entries.push((
TaskInput::resolve_input(k).await?,
TaskInput::resolve_input(v).await?,
));
}
// note: resolving might deduplicate `Vc`s in keys
Ok(Self::from(new_entries))
}
fn is_resolved(&self) -> bool {
self.iter()
.all(|(k, v)| TaskInput::is_resolved(k) && TaskInput::is_resolved(v))
}
fn is_transient(&self) -> bool {
self.iter()
.any(|(k, v)| TaskInput::is_transient(k) || TaskInput::is_transient(v))
}
}
impl<T> TaskInput for FrozenSet<T>
where
T: TaskInput + Ord + 'static,
{
async fn resolve_input(&self) -> Result<Self> {
let mut new_set = Vec::with_capacity(self.len());
for value in self {
new_set.push(TaskInput::resolve_input(value).await?);
}
Ok(Self::from_iter(new_set))
}
fn is_resolved(&self) -> bool {
......
......@@ -12,6 +12,7 @@ use auto_hash_map::{AutoMap, AutoSet};
use either::Either;
use indexmap::{IndexMap, IndexSet};
use smallvec::SmallVec;
use turbo_frozenmap::{FrozenMap, FrozenSet};
use turbo_rcstr::RcStr;
use crate::RawVc;
......@@ -200,6 +201,14 @@ impl<T: TraceRawVcs, S> TraceRawVcs for IndexSet<T, S> {
}
}
impl<T: TraceRawVcs> TraceRawVcs for FrozenSet<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
for item in self.iter() {
TraceRawVcs::trace_raw_vcs(item, trace_context);
}
}
}
impl<K: TraceRawVcs, V: TraceRawVcs, S> TraceRawVcs for HashMap<K, V, S> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
for (key, value) in self.iter() {
......@@ -236,6 +245,15 @@ impl<K: TraceRawVcs, V: TraceRawVcs, S> TraceRawVcs for IndexMap<K, V, S> {
}
}
impl<K: TraceRawVcs, V: TraceRawVcs> TraceRawVcs for FrozenMap<K, V> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
for (key, value) in self.iter() {
TraceRawVcs::trace_raw_vcs(key, trace_context);
TraceRawVcs::trace_raw_vcs(value, trace_context);
}
}
}
impl<T: TraceRawVcs + ?Sized> TraceRawVcs for Box<T> {
fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) {
TraceRawVcs::trace_raw_vcs(&**self, trace_context);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment