Unverified Commit eb172198 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] Reduce memory (#75723)

### What?

* avoid cloning SourceMap for ignore list
* shrink vec before converting to Rope
* shrink Strings too
* shrink source map
parent 2da62593
......@@ -9552,6 +9552,7 @@ dependencies = [
"async-trait",
"auto-hash-map",
"browserslist-rs",
"const_format",
"either",
"futures",
"indexmap 2.7.1",
......
......@@ -133,6 +133,7 @@ clap = { version = "4.5.2", features = ["derive"] }
concurrent-queue = "2.5.0"
console = "0.15.5"
console-subscriber = "0.4.1"
const_format = "0.2.30"
criterion = "0.5.1"
crossbeam-channel = "0.5.8"
dashmap = "6.1.0"
......
......@@ -118,8 +118,22 @@ impl Rope {
}
}
impl From<Vec<u8>> for Rope {
fn from(mut bytes: Vec<u8>) -> Self {
bytes.shrink_to_fit();
Rope::from(Bytes::from(bytes))
}
}
impl From<String> for Rope {
fn from(mut bytes: String) -> Self {
bytes.shrink_to_fit();
Rope::from(Bytes::from(bytes))
}
}
impl<T: Into<Bytes>> From<T> for Rope {
fn from(bytes: T) -> Self {
default fn from(bytes: T) -> Self {
let bytes = bytes.into();
// We can't have an InnerRope which contains an empty Local section.
if bytes.is_empty() {
......@@ -309,7 +323,10 @@ impl Uncommitted {
match mem::take(self) {
Self::None => None,
Self::Static(s) => Some(s.into()),
Self::Owned(v) => Some(v.into()),
Self::Owned(mut v) => {
v.shrink_to_fit();
Some(v.into())
}
}
}
}
......
......@@ -17,6 +17,7 @@ anyhow = { workspace = true }
async-trait = { workspace = true }
auto-hash-map = { workspace = true }
browserslist-rs = { workspace = true }
const_format = { workspace = true }
either = { workspace = true }
futures = { workspace = true }
indexmap = { workspace = true }
......
......@@ -5,7 +5,7 @@ use std::{
};
use anyhow::{Context, Result};
use indexmap::{IndexMap, IndexSet};
use indexmap::IndexMap;
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{
rope::{Rope, RopeBuilder},
......@@ -181,38 +181,14 @@ impl GenerateSourceMap for Code {
last_byte_pos = *byte_pos;
let encoded = match map {
None => SourceMap::empty(),
None => SourceMap::empty().to_resolved().await?,
Some(map) => match *map.generate_source_map().await? {
None => SourceMap::empty(),
Some(map) => {
let map = &*map.await?;
let map = map.to_source_map().await?;
match map.as_regular_source_map() {
None => SourceMap::empty(),
Some(map) => {
let mut map = map.into_owned();
let mut ignored_ids = IndexSet::new();
for (src_id, src) in map.sources().enumerate() {
if src.starts_with("turbopack://[next]")
|| src.starts_with("turbopack://[turbopack]")
|| src.contains("/node_modules/")
{
ignored_ids.insert(src_id);
}
}
for ignored_id in ignored_ids {
map.add_to_ignore_list(ignored_id as _);
}
SourceMap::new_decoded(sourcemap::DecodedMap::Regular(map)).cell()
}
}
}
None => SourceMap::empty().to_resolved().await?,
Some(map) => map,
},
};
sections.push(SourceMapSection::new(pos, encoded.to_resolved().await?))
sections.push(SourceMapSection::new(pos, encoded))
}
Ok(Vc::cell(Some(
......
use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc};
use anyhow::Result;
use indexmap::IndexSet;
use once_cell::sync::Lazy;
use ref_cast::RefCast;
use regex::Regex;
......@@ -15,11 +14,12 @@ use turbo_tasks_fs::{
};
use crate::{
asset::AssetContent, source::Source, source_pos::SourcePos, virtual_source::VirtualSource,
SOURCE_MAP_PREFIX,
asset::AssetContent, source::Source, source_map::utils::add_default_ignore_list,
source_pos::SourcePos, virtual_source::VirtualSource, SOURCE_MAP_PREFIX,
};
pub(crate) mod source_map_asset;
pub mod utils;
pub use source_map_asset::SourceMapAsset;
......@@ -416,28 +416,15 @@ impl SourceMap {
.collect::<Vec<_>>();
let mut new_sources = Vec::with_capacity(count);
let mut new_source_contents = Vec::with_capacity(count);
let mut ignored_sources = IndexSet::new();
for (src_id, (source, source_content)) in sources
.into_iter()
.zip(source_contents.into_iter())
.enumerate()
{
for (source, source_content) in sources.into_iter().zip(source_contents.into_iter()) {
let (source, name) = resolve_source(source, source_content, origin).await?;
if source.starts_with("turbopack://[next]")
|| source.starts_with("turbopack://[turbopack]")
|| source.contains("/node_modules/")
{
ignored_sources.insert(src_id);
}
new_sources.push(source);
new_source_contents.push(Some(name));
}
let mut map =
RegularMap::new(file, tokens, names, new_sources, Some(new_source_contents));
for ignored_source in ignored_sources {
map.add_to_ignore_list(ignored_source as _);
}
add_default_ignore_list(&mut map);
Ok(map)
}
......
use std::collections::HashSet;
use const_format::concatcp;
use sourcemap::SourceMap;
use crate::SOURCE_MAP_PREFIX;
pub fn add_default_ignore_list(map: &mut SourceMap) {
let mut ignored_ids = HashSet::new();
for (source_id, source) in map.sources().enumerate() {
if source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[next]"))
|| source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[turbopack]"))
|| source.contains("/node_modules/")
{
ignored_ids.insert(source_id);
}
}
for ignored_id in ignored_ids {
map.add_to_ignore_list(ignored_id as _);
}
}
......@@ -31,7 +31,7 @@ use turbopack_core::{
reference_type::ImportContext,
resolve::origin::ResolveOrigin,
source::Source,
source_map::{GenerateSourceMap, OptionSourceMap},
source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap},
source_pos::SourcePos,
SOURCE_MAP_PREFIX,
};
......@@ -647,20 +647,22 @@ impl GenerateSourceMap for ParseCssResultSourceMap {
);
}
let mut map = builder.into_sourcemap();
add_default_ignore_list(&mut map);
Vc::cell(Some(
turbopack_core::source_map::SourceMap::new_regular(builder.into_sourcemap())
.resolved_cell(),
turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
))
}
ParseCssResultSourceMap::Swc {
source_map,
mappings,
} => {
let map = source_map.build_source_map_with_config(
let mut map = source_map.build_source_map_with_config(
mappings,
None,
InlineSourcesContentConfig {},
);
add_default_ignore_list(&mut map);
Vc::cell(Some(
turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
))
......
......@@ -39,7 +39,7 @@ pub async fn minify(
let code = code.await?;
let cm = Arc::new(SwcSourceMap::new(FilePathMapping::empty()));
let (src, src_map_buf) = {
let (src, mut src_map_buf) = {
let compiler = Arc::new(Compiler::new(cm.clone()));
let fm = compiler.cm.new_source_file(
FileName::Custom(path.path.to_string()).into(),
......@@ -116,6 +116,7 @@ pub async fn minify(
let mut builder = CodeBuilder::default();
if let Some(original_map) = source_maps {
src_map_buf.shrink_to_fit();
builder.push_source(
&src.into(),
Some(ResolvedVc::upcast(
......
......@@ -30,7 +30,7 @@ use turbopack_core::{
error::PrettyPrintError,
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
source::Source,
source_map::{GenerateSourceMap, OptionSourceMap, SourceMap},
source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap, SourceMap},
SOURCE_MAP_PREFIX,
};
use turbopack_swc_utils::emitter::IssueEmitter;
......@@ -124,11 +124,12 @@ impl GenerateSourceMap for ParseResultSourceMap {
} else {
None
};
let map = self.files_map.build_source_map_with_config(
let mut map = self.files_map.build_source_map_with_config(
&self.mappings,
input_map.as_deref(),
InlineSourcesContentConfig {},
);
add_default_ignore_list(&mut map);
Ok(Vc::cell(Some(SourceMap::new_regular(map).resolved_cell())))
}
}
......
......@@ -20,7 +20,7 @@ workspace = true
anyhow = { workspace = true }
async-stream = "0.3.4"
async-trait = { workspace = true }
const_format = "0.2.30"
const_format = { workspace = true }
either = { workspace = true, features = ["serde"] }
futures = { workspace = true }
futures-retry = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment