Unverified Commit eb172198 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

[Turbopack] Reduce memory (#75723)

### What?

* avoid cloning SourceMap for ignore list
* shrink vec before converting to Rope
* shrink Strings too
* shrink source map
parent 2da62593
...@@ -9552,6 +9552,7 @@ dependencies = [ ...@@ -9552,6 +9552,7 @@ dependencies = [
"async-trait", "async-trait",
"auto-hash-map", "auto-hash-map",
"browserslist-rs", "browserslist-rs",
"const_format",
"either", "either",
"futures", "futures",
"indexmap 2.7.1", "indexmap 2.7.1",
......
...@@ -133,6 +133,7 @@ clap = { version = "4.5.2", features = ["derive"] } ...@@ -133,6 +133,7 @@ clap = { version = "4.5.2", features = ["derive"] }
concurrent-queue = "2.5.0" concurrent-queue = "2.5.0"
console = "0.15.5" console = "0.15.5"
console-subscriber = "0.4.1" console-subscriber = "0.4.1"
const_format = "0.2.30"
criterion = "0.5.1" criterion = "0.5.1"
crossbeam-channel = "0.5.8" crossbeam-channel = "0.5.8"
dashmap = "6.1.0" dashmap = "6.1.0"
......
...@@ -118,8 +118,22 @@ impl Rope { ...@@ -118,8 +118,22 @@ impl Rope {
} }
} }
impl From<Vec<u8>> for Rope {
fn from(mut bytes: Vec<u8>) -> Self {
bytes.shrink_to_fit();
Rope::from(Bytes::from(bytes))
}
}
impl From<String> for Rope {
fn from(mut bytes: String) -> Self {
bytes.shrink_to_fit();
Rope::from(Bytes::from(bytes))
}
}
impl<T: Into<Bytes>> From<T> for Rope { impl<T: Into<Bytes>> From<T> for Rope {
fn from(bytes: T) -> Self { default fn from(bytes: T) -> Self {
let bytes = bytes.into(); let bytes = bytes.into();
// We can't have an InnerRope which contains an empty Local section. // We can't have an InnerRope which contains an empty Local section.
if bytes.is_empty() { if bytes.is_empty() {
...@@ -309,7 +323,10 @@ impl Uncommitted { ...@@ -309,7 +323,10 @@ impl Uncommitted {
match mem::take(self) { match mem::take(self) {
Self::None => None, Self::None => None,
Self::Static(s) => Some(s.into()), Self::Static(s) => Some(s.into()),
Self::Owned(v) => Some(v.into()), Self::Owned(mut v) => {
v.shrink_to_fit();
Some(v.into())
}
} }
} }
} }
......
...@@ -17,6 +17,7 @@ anyhow = { workspace = true } ...@@ -17,6 +17,7 @@ anyhow = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
auto-hash-map = { workspace = true } auto-hash-map = { workspace = true }
browserslist-rs = { workspace = true } browserslist-rs = { workspace = true }
const_format = { workspace = true }
either = { workspace = true } either = { workspace = true }
futures = { workspace = true } futures = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
......
...@@ -5,7 +5,7 @@ use std::{ ...@@ -5,7 +5,7 @@ use std::{
}; };
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use indexmap::{IndexMap, IndexSet}; use indexmap::IndexMap;
use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
rope::{Rope, RopeBuilder}, rope::{Rope, RopeBuilder},
...@@ -181,38 +181,14 @@ impl GenerateSourceMap for Code { ...@@ -181,38 +181,14 @@ impl GenerateSourceMap for Code {
last_byte_pos = *byte_pos; last_byte_pos = *byte_pos;
let encoded = match map { let encoded = match map {
None => SourceMap::empty(), None => SourceMap::empty().to_resolved().await?,
Some(map) => match *map.generate_source_map().await? { Some(map) => match *map.generate_source_map().await? {
None => SourceMap::empty(), None => SourceMap::empty().to_resolved().await?,
Some(map) => { Some(map) => map,
let map = &*map.await?;
let map = map.to_source_map().await?;
match map.as_regular_source_map() {
None => SourceMap::empty(),
Some(map) => {
let mut map = map.into_owned();
let mut ignored_ids = IndexSet::new();
for (src_id, src) in map.sources().enumerate() {
if src.starts_with("turbopack://[next]")
|| src.starts_with("turbopack://[turbopack]")
|| src.contains("/node_modules/")
{
ignored_ids.insert(src_id);
}
}
for ignored_id in ignored_ids {
map.add_to_ignore_list(ignored_id as _);
}
SourceMap::new_decoded(sourcemap::DecodedMap::Regular(map)).cell()
}
}
}
}, },
}; };
sections.push(SourceMapSection::new(pos, encoded.to_resolved().await?)) sections.push(SourceMapSection::new(pos, encoded))
} }
Ok(Vc::cell(Some( Ok(Vc::cell(Some(
......
use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc}; use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc};
use anyhow::Result; use anyhow::Result;
use indexmap::IndexSet;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use ref_cast::RefCast; use ref_cast::RefCast;
use regex::Regex; use regex::Regex;
...@@ -15,11 +14,12 @@ use turbo_tasks_fs::{ ...@@ -15,11 +14,12 @@ use turbo_tasks_fs::{
}; };
use crate::{ use crate::{
asset::AssetContent, source::Source, source_pos::SourcePos, virtual_source::VirtualSource, asset::AssetContent, source::Source, source_map::utils::add_default_ignore_list,
SOURCE_MAP_PREFIX, source_pos::SourcePos, virtual_source::VirtualSource, SOURCE_MAP_PREFIX,
}; };
pub(crate) mod source_map_asset; pub(crate) mod source_map_asset;
pub mod utils;
pub use source_map_asset::SourceMapAsset; pub use source_map_asset::SourceMapAsset;
...@@ -416,28 +416,15 @@ impl SourceMap { ...@@ -416,28 +416,15 @@ impl SourceMap {
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let mut new_sources = Vec::with_capacity(count); let mut new_sources = Vec::with_capacity(count);
let mut new_source_contents = Vec::with_capacity(count); let mut new_source_contents = Vec::with_capacity(count);
let mut ignored_sources = IndexSet::new(); for (source, source_content) in sources.into_iter().zip(source_contents.into_iter()) {
for (src_id, (source, source_content)) in sources
.into_iter()
.zip(source_contents.into_iter())
.enumerate()
{
let (source, name) = resolve_source(source, source_content, origin).await?; let (source, name) = resolve_source(source, source_content, origin).await?;
if source.starts_with("turbopack://[next]")
|| source.starts_with("turbopack://[turbopack]")
|| source.contains("/node_modules/")
{
ignored_sources.insert(src_id);
}
new_sources.push(source); new_sources.push(source);
new_source_contents.push(Some(name)); new_source_contents.push(Some(name));
} }
let mut map = let mut map =
RegularMap::new(file, tokens, names, new_sources, Some(new_source_contents)); RegularMap::new(file, tokens, names, new_sources, Some(new_source_contents));
for ignored_source in ignored_sources { add_default_ignore_list(&mut map);
map.add_to_ignore_list(ignored_source as _);
}
Ok(map) Ok(map)
} }
......
use std::collections::HashSet;
use const_format::concatcp;
use sourcemap::SourceMap;
use crate::SOURCE_MAP_PREFIX;
pub fn add_default_ignore_list(map: &mut SourceMap) {
let mut ignored_ids = HashSet::new();
for (source_id, source) in map.sources().enumerate() {
if source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[next]"))
|| source.starts_with(concatcp!(SOURCE_MAP_PREFIX, "[turbopack]"))
|| source.contains("/node_modules/")
{
ignored_ids.insert(source_id);
}
}
for ignored_id in ignored_ids {
map.add_to_ignore_list(ignored_id as _);
}
}
...@@ -31,7 +31,7 @@ use turbopack_core::{ ...@@ -31,7 +31,7 @@ use turbopack_core::{
reference_type::ImportContext, reference_type::ImportContext,
resolve::origin::ResolveOrigin, resolve::origin::ResolveOrigin,
source::Source, source::Source,
source_map::{GenerateSourceMap, OptionSourceMap}, source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap},
source_pos::SourcePos, source_pos::SourcePos,
SOURCE_MAP_PREFIX, SOURCE_MAP_PREFIX,
}; };
...@@ -647,20 +647,22 @@ impl GenerateSourceMap for ParseCssResultSourceMap { ...@@ -647,20 +647,22 @@ impl GenerateSourceMap for ParseCssResultSourceMap {
); );
} }
let mut map = builder.into_sourcemap();
add_default_ignore_list(&mut map);
Vc::cell(Some( Vc::cell(Some(
turbopack_core::source_map::SourceMap::new_regular(builder.into_sourcemap()) turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
.resolved_cell(),
)) ))
} }
ParseCssResultSourceMap::Swc { ParseCssResultSourceMap::Swc {
source_map, source_map,
mappings, mappings,
} => { } => {
let map = source_map.build_source_map_with_config( let mut map = source_map.build_source_map_with_config(
mappings, mappings,
None, None,
InlineSourcesContentConfig {}, InlineSourcesContentConfig {},
); );
add_default_ignore_list(&mut map);
Vc::cell(Some( Vc::cell(Some(
turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(), turbopack_core::source_map::SourceMap::new_regular(map).resolved_cell(),
)) ))
......
...@@ -39,7 +39,7 @@ pub async fn minify( ...@@ -39,7 +39,7 @@ pub async fn minify(
let code = code.await?; let code = code.await?;
let cm = Arc::new(SwcSourceMap::new(FilePathMapping::empty())); let cm = Arc::new(SwcSourceMap::new(FilePathMapping::empty()));
let (src, src_map_buf) = { let (src, mut src_map_buf) = {
let compiler = Arc::new(Compiler::new(cm.clone())); let compiler = Arc::new(Compiler::new(cm.clone()));
let fm = compiler.cm.new_source_file( let fm = compiler.cm.new_source_file(
FileName::Custom(path.path.to_string()).into(), FileName::Custom(path.path.to_string()).into(),
...@@ -116,6 +116,7 @@ pub async fn minify( ...@@ -116,6 +116,7 @@ pub async fn minify(
let mut builder = CodeBuilder::default(); let mut builder = CodeBuilder::default();
if let Some(original_map) = source_maps { if let Some(original_map) = source_maps {
src_map_buf.shrink_to_fit();
builder.push_source( builder.push_source(
&src.into(), &src.into(),
Some(ResolvedVc::upcast( Some(ResolvedVc::upcast(
......
...@@ -30,7 +30,7 @@ use turbopack_core::{ ...@@ -30,7 +30,7 @@ use turbopack_core::{
error::PrettyPrintError, error::PrettyPrintError,
issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString},
source::Source, source::Source,
source_map::{GenerateSourceMap, OptionSourceMap, SourceMap}, source_map::{utils::add_default_ignore_list, GenerateSourceMap, OptionSourceMap, SourceMap},
SOURCE_MAP_PREFIX, SOURCE_MAP_PREFIX,
}; };
use turbopack_swc_utils::emitter::IssueEmitter; use turbopack_swc_utils::emitter::IssueEmitter;
...@@ -124,11 +124,12 @@ impl GenerateSourceMap for ParseResultSourceMap { ...@@ -124,11 +124,12 @@ impl GenerateSourceMap for ParseResultSourceMap {
} else { } else {
None None
}; };
let map = self.files_map.build_source_map_with_config( let mut map = self.files_map.build_source_map_with_config(
&self.mappings, &self.mappings,
input_map.as_deref(), input_map.as_deref(),
InlineSourcesContentConfig {}, InlineSourcesContentConfig {},
); );
add_default_ignore_list(&mut map);
Ok(Vc::cell(Some(SourceMap::new_regular(map).resolved_cell()))) Ok(Vc::cell(Some(SourceMap::new_regular(map).resolved_cell())))
} }
} }
......
...@@ -20,7 +20,7 @@ workspace = true ...@@ -20,7 +20,7 @@ workspace = true
anyhow = { workspace = true } anyhow = { workspace = true }
async-stream = "0.3.4" async-stream = "0.3.4"
async-trait = { workspace = true } async-trait = { workspace = true }
const_format = "0.2.30" const_format = { workspace = true }
either = { workspace = true, features = ["serde"] } either = { workspace = true, features = ["serde"] }
futures = { workspace = true } futures = { workspace = true }
futures-retry = { workspace = true } futures-retry = { workspace = true }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment