Unverified Commit 191224e6 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: defer sourcemaps processing to codegen (#78855)

Closes PACK-3434

```
    // TODO This is too eagerly generating the source map. We should store a GenerateSourceMap
    // instead and only actually generate the SourceMap when it's needed. This would allow to avoid
    // generating the source map when a module is never included in the final bundle. It allows
    // analysis to finish earlier which makes references available earlier which benefits
    // parallelism. When SourceMaps are emitted it moves that generation work to the code generation
    // phase which is more parallelizable.
```
parent be6c8ee2
...@@ -86,7 +86,7 @@ use turbopack_core::{ ...@@ -86,7 +86,7 @@ use turbopack_core::{
FindContextFileResult, FindContextFileResult,
}, },
source::Source, source::Source,
source_map::OptionStringifiedSourceMap, source_map::GenerateSourceMap,
}; };
// TODO remove this // TODO remove this
pub use turbopack_resolve::ecmascript as resolve; pub use turbopack_resolve::ecmascript as resolve;
...@@ -819,7 +819,7 @@ pub struct EcmascriptModuleContentOptions { ...@@ -819,7 +819,7 @@ pub struct EcmascriptModuleContentOptions {
code_generation: ResolvedVc<CodeGens>, code_generation: ResolvedVc<CodeGens>,
async_module: ResolvedVc<OptionAsyncModule>, async_module: ResolvedVc<OptionAsyncModule>,
generate_source_map: bool, generate_source_map: bool,
original_source_map: ResolvedVc<OptionStringifiedSourceMap>, original_source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
exports: ResolvedVc<EcmascriptExports>, exports: ResolvedVc<EcmascriptExports>,
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>, async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
} }
...@@ -945,7 +945,7 @@ impl EcmascriptModuleContent { ...@@ -945,7 +945,7 @@ impl EcmascriptModuleContent {
specified_module_type, specified_module_type,
vec![], vec![],
generate_source_map, generate_source_map,
OptionStringifiedSourceMap::none().to_resolved().await?, None,
) )
.await?; .await?;
emit_content(content).await emit_content(content).await
...@@ -958,7 +958,7 @@ struct CodeGenResult { ...@@ -958,7 +958,7 @@ struct CodeGenResult {
comments: Either<ImmutableComments, Arc<ImmutableComments>>, comments: Either<ImmutableComments, Arc<ImmutableComments>>,
is_esm: bool, is_esm: bool,
generate_source_map: bool, generate_source_map: bool,
original_source_map: Option<ResolvedVc<OptionStringifiedSourceMap>>, original_source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
} }
async fn process_parse_result( async fn process_parse_result(
...@@ -967,7 +967,7 @@ async fn process_parse_result( ...@@ -967,7 +967,7 @@ async fn process_parse_result(
specified_module_type: SpecifiedModuleType, specified_module_type: SpecifiedModuleType,
code_gens: Vec<CodeGeneration>, code_gens: Vec<CodeGeneration>,
generate_source_map: bool, generate_source_map: bool,
original_source_map: ResolvedVc<OptionStringifiedSourceMap>, original_source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
) -> Result<CodeGenResult> { ) -> Result<CodeGenResult> {
let parsed = parsed.final_read_hint().await?; let parsed = parsed.final_read_hint().await?;
...@@ -1025,7 +1025,7 @@ async fn process_parse_result( ...@@ -1025,7 +1025,7 @@ async fn process_parse_result(
comments, comments,
is_esm, is_esm,
generate_source_map, generate_source_map,
original_source_map: Some(original_source_map), original_source_map,
} }
} }
ParseResult::Unparseable { messages } => { ParseResult::Unparseable { messages } => {
...@@ -1130,7 +1130,7 @@ async fn emit_content(content: CodeGenResult) -> Result<Vc<EcmascriptModuleConte ...@@ -1130,7 +1130,7 @@ async fn emit_content(content: CodeGenResult) -> Result<Vc<EcmascriptModuleConte
Some(generate_js_source_map( Some(generate_js_source_map(
source_map.clone(), source_map.clone(),
mappings, mappings,
original_source_map.await?.as_ref(), original_source_map.generate_source_map().await?.as_ref(),
)?) )?)
} else { } else {
Some(generate_js_source_map(source_map.clone(), mappings, None)?) Some(generate_js_source_map(source_map.clone(), mappings, None)?)
......
...@@ -56,7 +56,7 @@ use turbo_tasks::{ ...@@ -56,7 +56,7 @@ use turbo_tasks::{
trace::TraceRawVcs, FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput, trace::TraceRawVcs, FxIndexMap, FxIndexSet, NonLocalValue, ReadRef, ResolvedVc, TaskInput,
TryJoinIterExt, Upcast, Value, ValueToString, Vc, TryJoinIterExt, Upcast, Value, ValueToString, Vc,
}; };
use turbo_tasks_fs::{rope::Rope, FileSystemPath}; use turbo_tasks_fs::FileSystemPath;
use turbopack_core::{ use turbopack_core::{
compile_time_info::{ compile_time_info::{
CompileTimeInfo, DefineableNameSegment, FreeVarReference, FreeVarReferences, CompileTimeInfo, DefineableNameSegment, FreeVarReference, FreeVarReferences,
...@@ -76,9 +76,7 @@ use turbopack_core::{ ...@@ -76,9 +76,7 @@ use turbopack_core::{
resolve, FindContextFileResult, ModulePart, resolve, FindContextFileResult, ModulePart,
}, },
source::Source, source::Source,
source_map::{ source_map::GenerateSourceMap,
utils::resolve_source_map_sources, GenerateSourceMap, OptionStringifiedSourceMap,
},
}; };
use turbopack_resolve::{ use turbopack_resolve::{
ecmascript::{apply_cjs_specific_options, cjs_resolve_source}, ecmascript::{apply_cjs_specific_options, cjs_resolve_source},
...@@ -148,6 +146,7 @@ use crate::{ ...@@ -148,6 +146,7 @@ use crate::{
node::PackageJsonReference, node::PackageJsonReference,
require_context::{RequireContextAssetReference, RequireContextMap}, require_context::{RequireContextAssetReference, RequireContextMap},
type_issue::SpecifiedModuleTypeIssue, type_issue::SpecifiedModuleTypeIssue,
util::InlineSourceMap,
}, },
runtime_functions::{ runtime_functions::{
TUBROPACK_RUNTIME_FUNCTION_SHORTCUTS, TURBOPACK_EXPORT_NAMESPACE, TURBOPACK_EXPORT_VALUE, TUBROPACK_RUNTIME_FUNCTION_SHORTCUTS, TURBOPACK_EXPORT_NAMESPACE, TURBOPACK_EXPORT_VALUE,
...@@ -174,7 +173,7 @@ pub struct AnalyzeEcmascriptModuleResult { ...@@ -174,7 +173,7 @@ pub struct AnalyzeEcmascriptModuleResult {
pub has_side_effect_free_directive: bool, pub has_side_effect_free_directive: bool,
/// `true` when the analysis was successful. /// `true` when the analysis was successful.
pub successful: bool, pub successful: bool,
pub source_map: ResolvedVc<OptionStringifiedSourceMap>, pub source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -223,7 +222,7 @@ pub struct AnalyzeEcmascriptModuleResultBuilder { ...@@ -223,7 +222,7 @@ pub struct AnalyzeEcmascriptModuleResultBuilder {
exports: EcmascriptExports, exports: EcmascriptExports,
async_module: ResolvedVc<OptionAsyncModule>, async_module: ResolvedVc<OptionAsyncModule>,
successful: bool, successful: bool,
source_map: Option<ResolvedVc<OptionStringifiedSourceMap>>, source_map: Option<ResolvedVc<Box<dyn GenerateSourceMap>>>,
has_side_effect_free_directive: bool, has_side_effect_free_directive: bool,
} }
...@@ -289,7 +288,7 @@ impl AnalyzeEcmascriptModuleResultBuilder { ...@@ -289,7 +288,7 @@ impl AnalyzeEcmascriptModuleResultBuilder {
} }
/// Sets the analysis result ES export. /// Sets the analysis result ES export.
pub fn set_source_map(&mut self, source_map: ResolvedVc<OptionStringifiedSourceMap>) { pub fn set_source_map(&mut self, source_map: ResolvedVc<Box<dyn GenerateSourceMap>>) {
self.source_map = Some(source_map); self.source_map = Some(source_map);
} }
...@@ -401,12 +400,6 @@ impl AnalyzeEcmascriptModuleResultBuilder { ...@@ -401,12 +400,6 @@ impl AnalyzeEcmascriptModuleResultBuilder {
let references: Vec<_> = self.references.into_iter().collect(); let references: Vec<_> = self.references.into_iter().collect();
let source_map = if let Some(source_map) = self.source_map {
source_map
} else {
OptionStringifiedSourceMap::none().to_resolved().await?
};
self.code_gens.shrink_to_fit(); self.code_gens.shrink_to_fit();
Ok(AnalyzeEcmascriptModuleResult::cell( Ok(AnalyzeEcmascriptModuleResult::cell(
AnalyzeEcmascriptModuleResult { AnalyzeEcmascriptModuleResult {
...@@ -424,7 +417,7 @@ impl AnalyzeEcmascriptModuleResultBuilder { ...@@ -424,7 +417,7 @@ impl AnalyzeEcmascriptModuleResultBuilder {
async_module: self.async_module, async_module: self.async_module,
has_side_effect_free_directive: self.has_side_effect_free_directive, has_side_effect_free_directive: self.has_side_effect_free_directive,
successful: self.successful, successful: self.successful,
source_map, source_map: self.source_map,
}, },
)) ))
} }
...@@ -705,14 +698,16 @@ pub(crate) async fn analyse_ecmascript_module_internal( ...@@ -705,14 +698,16 @@ pub(crate) async fn analyse_ecmascript_module_internal(
.to_resolved() .to_resolved()
.await?; .await?;
analysis.add_reference(reference); analysis.add_reference(reference);
let source_map = reference.generate_source_map(); analysis.set_source_map(ResolvedVc::upcast(reference));
analysis.set_source_map(source_map.to_resolved().await?);
source_map_from_comment = true; source_map_from_comment = true;
} else if JSON_DATA_URL_BASE64.is_match(path) { } else if JSON_DATA_URL_BASE64.is_match(path) {
let source_map = maybe_decode_data_url(path.into()); analysis.set_source_map(ResolvedVc::upcast(
let source_map = InlineSourceMap {
resolve_source_map_sources(source_map.as_ref(), origin_path).await?; origin_path: origin_path.to_resolved().await?,
analysis.set_source_map(ResolvedVc::cell(source_map)); source_map: path.into(),
}
.resolved_cell(),
));
source_map_from_comment = true; source_map_from_comment = true;
} }
} }
...@@ -720,12 +715,7 @@ pub(crate) async fn analyse_ecmascript_module_internal( ...@@ -720,12 +715,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
if let Some(generate_source_map) = if let Some(generate_source_map) =
ResolvedVc::try_sidecast::<Box<dyn GenerateSourceMap>>(source) ResolvedVc::try_sidecast::<Box<dyn GenerateSourceMap>>(source)
{ {
analysis.set_source_map( analysis.set_source_map(generate_source_map);
generate_source_map
.generate_source_map()
.to_resolved()
.await?,
);
} }
} }
anyhow::Ok(()) anyhow::Ok(())
...@@ -3665,16 +3655,3 @@ fn is_invoking_node_process_eval(args: &[JsValue]) -> bool { ...@@ -3665,16 +3655,3 @@ fn is_invoking_node_process_eval(args: &[JsValue]) -> bool {
false false
} }
fn maybe_decode_data_url(url: RcStr) -> Option<Rope> {
const DATA_PREAMBLE: &str = "data:application/json;base64,";
if !url.starts_with(DATA_PREAMBLE) {
return None;
}
let data_b64 = &url[DATA_PREAMBLE.len()..];
data_encoding::BASE64
.decode(data_b64.as_bytes())
.ok()
.map(Rope::from)
}
use anyhow::Result; use anyhow::Result;
use swc_core::{ecma::ast::Expr, quote}; use swc_core::{ecma::ast::Expr, quote};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::Vc; use turbo_tasks::{ResolvedVc, Vc};
use turbopack_core::resolve::parse::Request; use turbo_tasks_fs::{rope::Rope, FileSystemPath};
use turbopack_core::{
resolve::parse::Request,
source_map::{
utils::resolve_source_map_sources, GenerateSourceMap, OptionStringifiedSourceMap,
},
};
/// Creates a IIFE expression that throws a "Cannot find module" error for the /// Creates a IIFE expression that throws a "Cannot find module" error for the
/// given request string /// given request string
...@@ -36,3 +42,35 @@ pub async fn request_to_string(request: Vc<Request>) -> Result<Vc<RcStr>> { ...@@ -36,3 +42,35 @@ pub async fn request_to_string(request: Vc<Request>) -> Result<Vc<RcStr>> {
.unwrap_or_else(|| "unknown".into()), .unwrap_or_else(|| "unknown".into()),
)) ))
} }
#[turbo_tasks::value(shared)]
#[derive(Debug, Clone)]
pub struct InlineSourceMap {
/// The file path of the module containing the sourcemap data URL
pub origin_path: ResolvedVc<FileSystemPath>,
/// The Base64 encoded JSON sourcemap string
pub source_map: RcStr,
}
#[turbo_tasks::value_impl]
impl GenerateSourceMap for InlineSourceMap {
#[turbo_tasks::function]
pub async fn generate_source_map(&self) -> Result<Vc<OptionStringifiedSourceMap>> {
let source_map = maybe_decode_data_url(&self.source_map);
let source_map = resolve_source_map_sources(source_map.as_ref(), *self.origin_path).await?;
Ok(Vc::cell(source_map))
}
}
fn maybe_decode_data_url(url: &str) -> Option<Rope> {
const DATA_PREAMBLE: &str = "data:application/json;base64,";
if !url.starts_with(DATA_PREAMBLE) {
return None;
}
let data_b64 = &url[DATA_PREAMBLE.len()..];
data_encoding::BASE64
.decode(data_b64.as_bytes())
.ok()
.map(Rope::from)
}
...@@ -11,7 +11,6 @@ use turbopack_core::{ ...@@ -11,7 +11,6 @@ use turbopack_core::{
module_graph::ModuleGraph, module_graph::ModuleGraph,
reference::ModuleReferences, reference::ModuleReferences,
resolve::ModulePart, resolve::ModulePart,
source_map::OptionStringifiedSourceMap,
}; };
use super::chunk_item::EcmascriptModuleFacadeChunkItem; use super::chunk_item::EcmascriptModuleFacadeChunkItem;
...@@ -224,7 +223,7 @@ impl EcmascriptAnalyzable for EcmascriptModuleFacadeModule { ...@@ -224,7 +223,7 @@ impl EcmascriptAnalyzable for EcmascriptModuleFacadeModule {
code_generation: CodeGens::empty().to_resolved().await?, code_generation: CodeGens::empty().to_resolved().await?,
async_module: ResolvedVc::cell(Some(self.async_module().to_resolved().await?)), async_module: ResolvedVc::cell(Some(self.async_module().to_resolved().await?)),
generate_source_map: false, generate_source_map: false,
original_source_map: OptionStringifiedSourceMap::none().to_resolved().await?, original_source_map: None,
exports: self.get_exports().to_resolved().await?, exports: self.get_exports().to_resolved().await?,
async_module_info, async_module_info,
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment