Unverified Commit 9b44e3e3 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: avoid storing task data and task cache concurrenctly (#78775)

### What?

Avoid to concurrently write task data and task cache as it increases memory usage by a lot.
parent 837e0864
...@@ -177,92 +177,84 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage ...@@ -177,92 +177,84 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
{ {
let _span = tracing::trace_span!("save snapshot", session_id = ?session_id, operations = operations.len()); let _span = tracing::trace_span!("save snapshot", session_id = ?session_id, operations = operations.len());
let mut batch = self.database.write_batch()?; let mut batch = self.database.write_batch()?;
let mut task_items_result = Ok(Vec::new());
// Start organizing the updates in parallel // Start organizing the updates in parallel
match &mut batch { match &mut batch {
WriteBatch::Concurrent(ref batch, _) => { WriteBatch::Concurrent(ref batch, _) => {
turbo_tasks::scope(|s| { {
s.spawn(|_| { let _span = tracing::trace_span!("update task data").entered();
let _span = tracing::trace_span!("update task meta").entered(); process_task_data(snapshots, Some(batch))?;
task_items_result = process_task_data(snapshots, Some(batch)); }
});
let mut next_task_id = let mut next_task_id = get_next_free_task_id::<
get_next_free_task_id::< T::SerialWriteBatch<'_>,
T::SerialWriteBatch<'_>, T::ConcurrentWriteBatch<'_>,
T::ConcurrentWriteBatch<'_>, >(&mut WriteBatchRef::concurrent(batch))?;
>(&mut WriteBatchRef::concurrent(batch))?;
{ {
let _span = tracing::trace_span!( let _span = tracing::trace_span!(
"update task cache", "update task cache",
items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>() items = task_cache_updates.iter().map(|m| m.len()).sum::<usize>()
) )
.entered(); .entered();
let result = task_cache_updates let result = task_cache_updates
.into_par_iter() .into_par_iter()
.with_max_len(1) .with_max_len(1)
.map(|updates| { .map(|updates| {
let mut max_task_id = 0; let mut max_task_id = 0;
let mut task_type_bytes = Vec::new(); let mut task_type_bytes = Vec::new();
for (task_type, task_id) in updates { for (task_type, task_id) in updates {
let task_id: u32 = *task_id; let task_id: u32 = *task_id;
serialize_task_type(&task_type, &mut task_type_bytes, task_id)?; serialize_task_type(&task_type, &mut task_type_bytes, task_id)?;
batch batch
.put( .put(
KeySpace::ForwardTaskCache, KeySpace::ForwardTaskCache,
WriteBuffer::Borrowed(&task_type_bytes), WriteBuffer::Borrowed(&task_type_bytes),
WriteBuffer::Borrowed(&task_id.to_le_bytes()), WriteBuffer::Borrowed(&task_id.to_le_bytes()),
)
.with_context(|| {
anyhow!(
"Unable to write task cache {task_type:?} => {task_id}"
) )
.with_context(|| { })?;
anyhow!( batch
"Unable to write task cache {task_type:?} => \ .put(
{task_id}" KeySpace::ReverseTaskCache,
) WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()),
})?; WriteBuffer::Borrowed(&task_type_bytes),
batch )
.put( .with_context(|| {
KeySpace::ReverseTaskCache, anyhow!(
WriteBuffer::Borrowed(IntKey::new(task_id).as_ref()), "Unable to write task cache {task_id} => {task_type:?}"
WriteBuffer::Borrowed(&task_type_bytes),
) )
.with_context(|| { })?;
anyhow!( max_task_id = max_task_id.max(task_id + 1);
"Unable to write task cache {task_id} => \ }
{task_type:?}"
) Ok(max_task_id)
})?; })
max_task_id = max_task_id.max(task_id + 1); .reduce(
} || Ok(0),
|a, b| -> anyhow::Result<_> {
Ok(max_task_id) let a_max = a?;
}) let b_max = b?;
.reduce( Ok(max(a_max, b_max))
|| Ok(0), },
|a, b| -> anyhow::Result<_> { )?;
let a_max = a?; next_task_id = next_task_id.max(result);
let b_max = b?; }
Ok(max(a_max, b_max))
},
)?;
next_task_id = next_task_id.max(result);
}
save_infra::<T::SerialWriteBatch<'_>, T::ConcurrentWriteBatch<'_>>(
&mut WriteBatchRef::concurrent(batch),
next_task_id,
session_id,
operations,
)?;
anyhow::Ok(())
})?;
task_items_result?; save_infra::<T::SerialWriteBatch<'_>, T::ConcurrentWriteBatch<'_>>(
&mut WriteBatchRef::concurrent(batch),
next_task_id,
session_id,
operations,
)?;
} }
WriteBatch::Serial(batch) => { WriteBatch::Serial(batch) => {
let mut task_items_result = Ok(Vec::new());
turbo_tasks::scope(|s| { turbo_tasks::scope(|s| {
s.spawn(|_| { s.spawn(|_| {
task_items_result = task_items_result =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment