Unverified Commit 68bac2a7 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

turbopack: reduce EffectStateStorage memory by storing u128 content hashes (#92902)

### What?

In `turbo-tasks-fs`, the `WriteEffect` and `WriteLinkEffect` structs now
store a precomputed `u128` xxh3 content hash instead of returning the
full `ReadRef<PersistedFileContent>` / `ReadRef<LinkContent>` from their
`Effect::value()` implementations.

The associated type `Effect::Value` is changed from
`ReadRef<PersistedFileContent>` / `ReadRef<LinkContent>` to `u128` for
both effect types.

`LinkContent` gains a `DeterministicHash` derive to support hashing via
`hash_xxh3_hash128`.

### Why?

`EffectStateStorage` stores the last-applied value for each effect key
in `last_applied: Mutex<Option<Box<dyn Any + Send + Sync>>>` to enable a
double-checked deduplication fast path (skip re-writing a file if
content hasn't changed). In long-running sessions this map grows
continuously and holds onto a lot of memory because it stores full file
content (`ReadRef<PersistedFileContent>`) or full link content
(`ReadRef<LinkContent>`) for every file ever written.

Storing a `u128` hash (16 bytes) instead of the full content
significantly reduces memory usage while preserving the deduplication
semantics (hash equality implies content equality with negligible
collision probability).

### How?

- Added `content_hash: u128` field to `WriteEffect` and
`WriteLinkEffect`, computed at construction time using
`hash_xxh3_hash128` (xxh3 128-bit non-cryptographic hash, already used
elsewhere in turbopack for content addressing).
- Changed `Effect::Value = u128` for both effect types; `value()`
returns `&self.content_hash`.
- The `EffectStateStorage` now stores `Box<u128>` (24 bytes on the heap)
per entry instead of `Box<ReadRef<...>>` which held a reference-counted
pointer into potentially large file content.
- No changes to `EffectStateStorage`, `Effects::apply()`, or `DynEffect`
vtable — the existing `Any`/`DynPartialEq` machinery works transparently
with `u128`.

<!-- NEXT_JS_LLM_PR -->
Co-authored-by: 's avatarTobias Koppers <sokra@users.noreply.github.com>
Co-authored-by: 's avatarClaude <noreply@anthropic.com>
parent 7b52536c
...@@ -69,6 +69,7 @@ use turbo_tasks::{ ...@@ -69,6 +69,7 @@ use turbo_tasks::{
}; };
use turbo_tasks_hash::{ use turbo_tasks_hash::{
DeterministicHash, DeterministicHasher, HashAlgorithm, deterministic_hash, hash_xxh3_hash64, DeterministicHash, DeterministicHasher, HashAlgorithm, deterministic_hash, hash_xxh3_hash64,
hash_xxh3_hash128,
}; };
use turbo_unix_path::{ use turbo_unix_path::{
get_parent_path, get_relative_path_to, join_path, normalize_path, sys_to_unix, unix_to_sys, get_parent_path, get_relative_path_to, join_path, normalize_path, sys_to_unix, unix_to_sys,
...@@ -939,18 +940,19 @@ impl FileSystem for DiskFileSystem { ...@@ -939,18 +940,19 @@ impl FileSystem for DiskFileSystem {
full_path: PathBuf, full_path: PathBuf,
inner: Arc<DiskFileSystemInner>, inner: Arc<DiskFileSystemInner>,
content: ReadRef<PersistedFileContent>, content: ReadRef<PersistedFileContent>,
content_hash: u128,
} }
impl Effect for WriteEffect { impl Effect for WriteEffect {
type Error = AnyhowWrapper; type Error = AnyhowWrapper;
type Value = ReadRef<PersistedFileContent>; type Value = u128;
fn key(&self) -> Vec<u8> { fn key(&self) -> Vec<u8> {
self.full_path.as_os_str().as_encoded_bytes().to_vec() self.full_path.as_os_str().as_encoded_bytes().to_vec()
} }
fn value(&self) -> &ReadRef<PersistedFileContent> { fn value(&self) -> &u128 {
&self.content &self.content_hash
} }
fn state_storage(&self) -> &EffectStateStorage { fn state_storage(&self) -> &EffectStateStorage {
...@@ -1082,10 +1084,12 @@ impl FileSystem for DiskFileSystem { ...@@ -1082,10 +1084,12 @@ impl FileSystem for DiskFileSystem {
} }
} }
let content_hash = u128::from_le_bytes(hash_xxh3_hash128(&*content));
emit_effect(WriteEffect { emit_effect(WriteEffect {
full_path, full_path,
inner, inner,
content, content,
content_hash,
}); });
Ok(()) Ok(())
...@@ -1112,18 +1116,19 @@ impl FileSystem for DiskFileSystem { ...@@ -1112,18 +1116,19 @@ impl FileSystem for DiskFileSystem {
full_path: PathBuf, full_path: PathBuf,
inner: Arc<DiskFileSystemInner>, inner: Arc<DiskFileSystemInner>,
content: ReadRef<LinkContent>, content: ReadRef<LinkContent>,
content_hash: u128,
} }
impl Effect for WriteLinkEffect { impl Effect for WriteLinkEffect {
type Error = AnyhowWrapper; type Error = AnyhowWrapper;
type Value = ReadRef<LinkContent>; type Value = u128;
fn key(&self) -> Vec<u8> { fn key(&self) -> Vec<u8> {
self.full_path.as_os_str().as_encoded_bytes().to_vec() self.full_path.as_os_str().as_encoded_bytes().to_vec()
} }
fn value(&self) -> &ReadRef<LinkContent> { fn value(&self) -> &u128 {
&self.content &self.content_hash
} }
fn state_storage(&self) -> &EffectStateStorage { fn state_storage(&self) -> &EffectStateStorage {
...@@ -1361,10 +1366,12 @@ impl FileSystem for DiskFileSystem { ...@@ -1361,10 +1366,12 @@ impl FileSystem for DiskFileSystem {
} }
} }
let content_hash = u128::from_le_bytes(hash_xxh3_hash128(&*content));
emit_effect(WriteLinkEffect { emit_effect(WriteLinkEffect {
full_path, full_path,
inner, inner,
content, content,
content_hash,
}); });
Ok(()) Ok(())
} }
...@@ -2092,7 +2099,7 @@ bitflags! { ...@@ -2092,7 +2099,7 @@ bitflags! {
/// creating a new link, we always create junction points, because symlink creation may fail if /// creating a new link, we always create junction points, because symlink creation may fail if
/// Windows "developer mode" is not enabled and we're running in an unprivileged environment. /// Windows "developer mode" is not enabled and we're running in an unprivileged environment.
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(Debug)] #[derive(Debug, DeterministicHash)]
pub enum LinkContent { pub enum LinkContent {
/// A valid symbolic link pointing to `target`. /// A valid symbolic link pointing to `target`.
/// ///
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment