Unverified Commit 4ab7cacf authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: Improve errors when opening SST files (#80106)

### What?

add more context to error messages for database failures
parent 6f76e91d
...@@ -652,7 +652,8 @@ impl TurboPersistence { ...@@ -652,7 +652,8 @@ impl TurboPersistence {
max_coverage, max_coverage,
max_merge_sequence, max_merge_sequence,
max_merge_size, max_merge_size,
)?; )
.context("Failed to compact database")?;
} }
if !new_meta_files.is_empty() { if !new_meta_files.is_empty() {
...@@ -664,7 +665,8 @@ impl TurboPersistence { ...@@ -664,7 +665,8 @@ impl TurboPersistence {
blob_seq_numbers_to_delete, blob_seq_numbers_to_delete,
sequence_number: *sequence_number.get_mut(), sequence_number: *sequence_number.get_mut(),
keys_written, keys_written,
})?; })
.context("Failed to commit the database compaction")?;
} }
self.active_write_operation.store(false, Ordering::Release); self.active_write_operation.store(false, Ordering::Release);
...@@ -853,7 +855,7 @@ impl TurboPersistence { ...@@ -853,7 +855,7 @@ impl TurboPersistence {
let index_in_meta = ssts_with_ranges[index].index_in_meta; let index_in_meta = ssts_with_ranges[index].index_in_meta;
let meta = &meta_files[meta_index]; let meta = &meta_files[meta_index];
meta.entry(index_in_meta) meta.entry(index_in_meta)
.sst(&self.path)? .sst(meta)?
.iter(key_block_cache, value_block_cache) .iter(key_block_cache, value_block_cache)
}) })
.collect::<Result<Vec<_>>>()?; .collect::<Result<Vec<_>>>()?;
...@@ -983,7 +985,10 @@ impl TurboPersistence { ...@@ -983,7 +985,10 @@ impl TurboPersistence {
keys_written, keys_written,
}) })
}) })
.collect::<Result<Vec<_>>>()?; .collect::<Result<Vec<_>>>()
.with_context(|| {
format!("Failed to merge database files for family {family}")
})?;
let mut meta_file_builder = meta_file_builder.into_inner(); let mut meta_file_builder = meta_file_builder.into_inner();
...@@ -1017,8 +1022,7 @@ impl TurboPersistence { ...@@ -1017,8 +1022,7 @@ impl TurboPersistence {
let span = tracing::trace_span!("write meta file").entered(); let span = tracing::trace_span!("write meta file").entered();
let seq = sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let meta_file = let meta_file = meta_file_builder.write(&self.path, seq)?;
meta_file_builder.write(&self.path.join(format!("{seq:08}.meta")))?;
drop(span); drop(span);
let mut new_sst_files = Vec::with_capacity( let mut new_sst_files = Vec::with_capacity(
......
...@@ -68,9 +68,15 @@ impl MetaEntry { ...@@ -68,9 +68,15 @@ impl MetaEntry {
.expect("AQMF data out of bounds") .expect("AQMF data out of bounds")
} }
pub fn sst(&self, db_path: &Path) -> Result<&StaticSortedFile> { pub fn sst(&self, meta: &MetaFile) -> Result<&StaticSortedFile> {
self.sst self.sst.get_or_try_init(|| {
.get_or_try_init(|| StaticSortedFile::open(db_path, self.sst_data.clone())) StaticSortedFile::open(&meta.db_path, self.sst_data.clone()).with_context(|| {
format!(
"Unable to open static sorted file referenced from {:08}.meta",
meta.sequence_number()
)
})
})
} }
/// Returns the key family and hash range of this file. /// Returns the key family and hash range of this file.
...@@ -262,7 +268,14 @@ impl MetaFile { ...@@ -262,7 +268,14 @@ impl MetaFile {
GuardResult::Value(aqmf) => aqmf, GuardResult::Value(aqmf) => aqmf,
GuardResult::Guard(guard) => { GuardResult::Guard(guard) => {
let aqmf = entry.aqmf(self.aqmf_data()); let aqmf = entry.aqmf(self.aqmf_data());
let aqmf: Arc<qfilter::Filter> = Arc::new(pot::from_slice(aqmf)?); let aqmf: Arc<qfilter::Filter> =
Arc::new(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?);
let _ = guard.insert(aqmf.clone()); let _ = guard.insert(aqmf.clone());
aqmf aqmf
} }
...@@ -275,19 +288,23 @@ impl MetaFile { ...@@ -275,19 +288,23 @@ impl MetaFile {
} else { } else {
let aqmf = entry.aqmf.get_or_try_init(|| { let aqmf = entry.aqmf.get_or_try_init(|| {
let aqmf = entry.aqmf(self.aqmf_data()); let aqmf = entry.aqmf(self.aqmf_data());
anyhow::Ok(pot::from_slice(aqmf)?) anyhow::Ok(pot::from_slice(aqmf).with_context(|| {
format!(
"Failed to deserialize AQMF from {:08}.meta for {:08}.sst",
self.sequence_number,
entry.sequence_number()
)
})?)
})?; })?;
if !aqmf.contains_fingerprint(key_hash) { if !aqmf.contains_fingerprint(key_hash) {
miss_result = MetaLookupResult::QuickFilterMiss; miss_result = MetaLookupResult::QuickFilterMiss;
continue; continue;
} }
}; };
let result = entry.sst(&self.db_path)?.lookup( let result =
key_hash, entry
key, .sst(self)?
key_block_cache, .lookup(key_hash, key, key_block_cache, value_block_cache)?;
value_block_cache,
)?;
if !matches!(result, SstLookupResult::NotFound) { if !matches!(result, SstLookupResult::NotFound) {
return Ok(MetaLookupResult::SstLookup(result)); return Ok(MetaLookupResult::SstLookup(result));
} }
......
...@@ -4,6 +4,7 @@ use std::{ ...@@ -4,6 +4,7 @@ use std::{
path::Path, path::Path,
}; };
use anyhow::{Context, Result};
use byteorder::{BE, WriteBytesExt}; use byteorder::{BE, WriteBytesExt};
use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta; use crate::static_sorted_file_builder::StaticSortedFileBuilderMeta;
...@@ -34,7 +35,13 @@ impl MetaFileBuilder { ...@@ -34,7 +35,13 @@ impl MetaFileBuilder {
} }
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
pub fn write(&self, file: &Path) -> io::Result<File> { pub fn write(&self, db_path: &Path, seq: u32) -> Result<File> {
let file = db_path.join(format!("{seq:08}.meta"));
self.write_internal(&file)
.with_context(|| format!("Unable to write meta file {seq:08}.meta"))
}
fn write_internal(&self, file: &Path) -> io::Result<File> {
let mut file = BufWriter::new(File::create(file)?); let mut file = BufWriter::new(File::create(file)?);
file.write_u32::<BE>(0xFE4ADA4A)?; // Magic number file.write_u32::<BE>(0xFE4ADA4A)?; // Magic number
file.write_u32::<BE>(self.family)?; file.write_u32::<BE>(self.family)?;
......
...@@ -397,10 +397,7 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> { ...@@ -397,10 +397,7 @@ impl<K: StoreKey + Send + Sync, const FAMILIES: usize> WriteBatch<K, FAMILIES> {
} }
keys_written.fetch_add(entries, Ordering::Relaxed); keys_written.fetch_add(entries, Ordering::Relaxed);
let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1; let seq = self.current_sequence_number.fetch_add(1, Ordering::SeqCst) + 1;
let file = self.db_path.join(format!("{seq:08}.meta")); let file = builder.write(&self.db_path, seq)?;
let file = builder
.write(&file)
.with_context(|| format!("Unable to write meta file {seq:08}.meta"))?;
Ok((seq, file)) Ok((seq, file))
}) })
.collect::<Result<Vec<_>>>()?; .collect::<Result<Vec<_>>>()?;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment