Unverified Commit 9b44e3e3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: avoid storing task data and task cache concurrenctly (#78775)

### What?

Avoid to concurrently write task data and task cache as it increases memory usage by a lot.
parent 837e0864
...@@ -177,19 +177,16 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage ...@@ -177,19 +177,16 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
{ {
let _span = tracing::trace_span!("save snapshot", session_id = ?session_id, operations = operations.len()); let _span = tracing::trace_span!("save snapshot", session_id = ?session_id, operations = operations.len());
let mut batch = self.database.write_batch()?; let mut batch = self.database.write_batch()?;
let mut task_items_result = Ok(Vec::new());
// Start organizing the updates in parallel // Start organizing the updates in parallel
match &mut batch { match &mut batch {
WriteBatch::Concurrent(ref batch, _) => { WriteBatch::Concurrent(ref batch, _) => {
turbo_tasks::scope(|s| { {
s.spawn(|_| { let _span = tracing::trace_span!("update task data").entered();
let _span = tracing::trace_span!("update task meta").entered(); process_task_data(snapshots, Some(batch))?;
task_items_result = process_task_data(snapshots, Some(batch)); }
});
let mut next_task_id = let mut next_task_id = get_next_free_task_id::<
get_next_free_task_id::<
T::SerialWriteBatch<'_>, T::SerialWriteBatch<'_>,
T::ConcurrentWriteBatch<'_>, T::ConcurrentWriteBatch<'_>,
>(&mut WriteBatchRef::concurrent(batch))?; >(&mut WriteBatchRef::concurrent(batch))?;
...@@ -219,8 +216,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage ...@@ -219,8 +216,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
) )
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Unable to write task cache {task_type:?} => \ "Unable to write task cache {task_type:?} => {task_id}"
{task_id}"
) )
})?; })?;
batch batch
...@@ -231,8 +227,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage ...@@ -231,8 +227,7 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
) )
.with_context(|| { .with_context(|| {
anyhow!( anyhow!(
"Unable to write task cache {task_id} => \ "Unable to write task cache {task_id} => {task_type:?}"
{task_type:?}"
) )
})?; })?;
max_task_id = max_task_id.max(task_id + 1); max_task_id = max_task_id.max(task_id + 1);
...@@ -257,12 +252,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage ...@@ -257,12 +252,9 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
session_id, session_id,
operations, operations,
)?; )?;
anyhow::Ok(())
})?;
task_items_result?;
} }
WriteBatch::Serial(batch) => { WriteBatch::Serial(batch) => {
let mut task_items_result = Ok(Vec::new());
turbo_tasks::scope(|s| { turbo_tasks::scope(|s| {
s.spawn(|_| { s.spawn(|_| {
task_items_result = task_items_result =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment