Unverified Commit 37c97d69 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: refactor codegen (#78685)

Refactor codegen in preparation for scope hoisting

Now, you can easily get the AST of the code-gen-ed module (which I need to merge them together)

Part of PACK-2960
parent e6ea4b75
......@@ -266,6 +266,9 @@ impl<T> ReadRef<T>
where
T: VcValueType,
{
/// Returns the inner value, if this [`ReadRef`] has exactly one strong reference.
///
/// Otherwise, an [`Err`] is returned with the same [`ReadRef`] that was passed in.
pub fn try_unwrap(this: Self) -> Result<VcReadTarget<T>, Self> {
match triomphe::Arc::try_unwrap(this.0) {
Ok(value) => Ok(T::Read::value_to_target(value)),
......
......@@ -50,12 +50,13 @@ pub use references::{AnalyzeEcmascriptModuleResult, TURBOPACK_HELPER};
use serde::{Deserialize, Serialize};
pub use static_code::StaticEcmascriptCode;
use swc_core::{
common::{comments::Comments, util::take::Take, Globals, Mark, GLOBALS},
common::{comments::Comments, util::take::Take, Globals, Mark, SourceMap, DUMMY_SP, GLOBALS},
ecma::{
ast::{self, ModuleItem, Program, Script},
ast::{self, Expr, ModuleItem, Program, Script},
codegen::{text_writer::JsWriter, Emitter},
visit::{VisitMutWith, VisitMutWithAstPath},
},
quote,
};
use tracing::Instrument;
pub use transform::{
......@@ -99,6 +100,7 @@ use crate::{
analyse_ecmascript_module, async_module::OptionAsyncModule, esm::base::EsmAssetReferences,
},
side_effect_optimization::reference::EcmascriptModulePartReference,
swc_comments::ImmutableComments,
transform::remove_shebang,
};
......@@ -287,7 +289,7 @@ pub trait EcmascriptParsable {
}
#[turbo_tasks::value_trait]
pub trait EcmascriptAnalyzable {
pub trait EcmascriptAnalyzable: Module + Asset {
fn analyze(self: Vc<Self>) -> Vc<AnalyzeEcmascriptModuleResult>;
/// Generates module contents without an analysis pass. This is useful for
......@@ -297,12 +299,25 @@ pub trait EcmascriptAnalyzable {
generate_source_map: bool,
) -> Result<Vc<EcmascriptModuleContent>>;
async fn module_content_options(
self: Vc<Self>,
module_graph: Vc<ModuleGraph>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
async_module_info: Option<Vc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContentOptions>>;
async fn module_content(
self: Vc<Self>,
module_graph: Vc<ModuleGraph>,
chunking_context: Vc<Box<dyn ChunkingContext>>,
async_module_info: Option<Vc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContent>>;
) -> Vc<EcmascriptModuleContent> {
EcmascriptModuleContent::new(self.module_content_options(
module_graph,
chunking_context,
async_module_info,
))
}
}
impl EcmascriptModuleAsset {
......@@ -413,12 +428,12 @@ impl EcmascriptAnalyzable for EcmascriptModuleAsset {
}
#[turbo_tasks::function]
async fn module_content(
async fn module_content_options(
self: Vc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContent>> {
) -> Result<Vc<EcmascriptModuleContentOptions>> {
let parsed = self.parse().to_resolved().await?;
let analyze = self.analyze();
......@@ -429,24 +444,23 @@ impl EcmascriptAnalyzable for EcmascriptModuleAsset {
.reference_module_source_maps(Vc::upcast(self))
.await?;
Ok(EcmascriptModuleContent::new(
EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.references().to_resolved().await?,
esm_references: analyze_ref.esm_references,
part_references: vec![],
code_generation: analyze_ref.code_generation,
async_module: analyze_ref.async_module,
generate_source_map,
original_source_map: analyze_ref.source_map,
exports: analyze_ref.exports,
async_module_info,
},
))
Ok(EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.references().to_resolved().await?,
esm_references: analyze_ref.esm_references,
part_references: vec![],
code_generation: analyze_ref.code_generation,
async_module: analyze_ref.async_module,
generate_source_map,
original_source_map: analyze_ref.source_map,
exports: analyze_ref.exports,
async_module_info,
}
.cell())
}
}
......@@ -783,7 +797,7 @@ impl EcmascriptChunkItem for ModuleChunkItem {
}
/// The transformed contents of an Ecmascript module.
#[turbo_tasks::value]
#[turbo_tasks::value(shared)]
pub struct EcmascriptModuleContent {
pub inner_code: Rope,
pub source_map: Option<Rope>,
......@@ -791,7 +805,8 @@ pub struct EcmascriptModuleContent {
// pub refresh: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TaskInput, TraceRawVcs)]
#[turbo_tasks::value(shared)]
#[derive(Clone, Debug, Hash, TaskInput)]
pub struct EcmascriptModuleContentOptions {
parsed: ResolvedVc<ParseResult>,
ident: ResolvedVc<AssetIdent>,
......@@ -809,15 +824,10 @@ pub struct EcmascriptModuleContentOptions {
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
}
#[turbo_tasks::value_impl]
impl EcmascriptModuleContent {
/// Creates a new [`Vc<EcmascriptModuleContent>`].
#[turbo_tasks::function]
pub async fn new(input: EcmascriptModuleContentOptions) -> Result<Vc<Self>> {
impl EcmascriptModuleContentOptions {
async fn merged_code_gens(&self) -> Result<Vec<CodeGeneration>> {
let EcmascriptModuleContentOptions {
parsed,
ident,
specified_module_type,
module_graph,
chunking_context,
references,
......@@ -825,21 +835,20 @@ impl EcmascriptModuleContent {
part_references,
code_generation,
async_module,
generate_source_map,
original_source_map,
exports,
async_module_info,
} = input;
..
} = self;
let (esm_code_gens, part_code_gens, additional_code_gens, code_gens) = async {
async {
let additional_code_gens = [
if let Some(async_module) = &*async_module.await? {
Some(
async_module
.code_generation(
async_module_info.map(|info| *info),
*references,
*chunking_context,
**references,
**chunking_context,
)
.await?,
)
......@@ -849,7 +858,7 @@ impl EcmascriptModuleContent {
if let EcmascriptExports::EsmExports(exports) = *exports.await? {
Some(
exports
.code_generation(*module_graph, *chunking_context, Some(*parsed))
.code_generation(**module_graph, **chunking_context, Some(**parsed))
.await?,
)
} else {
......@@ -860,47 +869,64 @@ impl EcmascriptModuleContent {
let esm_code_gens = esm_references
.await?
.iter()
.map(|r| r.code_generation(*chunking_context))
.map(|r| r.code_generation(**chunking_context))
.try_join()
.await?;
let part_code_gens = part_references
.iter()
.map(|r| r.code_generation(*chunking_context))
.map(|r| r.code_generation(**chunking_context))
.try_join()
.await?;
let code_gens = code_generation
.await?
.iter()
.map(|c| c.code_generation(*module_graph, *chunking_context))
.map(|c| c.code_generation(**module_graph, **chunking_context))
.try_join()
.await?;
anyhow::Ok((
esm_code_gens,
part_code_gens,
additional_code_gens,
code_gens,
))
anyhow::Ok(
esm_code_gens
.into_iter()
.chain(part_code_gens.into_iter())
.chain(additional_code_gens.into_iter().flatten())
.chain(code_gens.into_iter())
.collect(),
)
}
.instrument(tracing::info_span!("precompute code generation"))
.await?;
let code_gens = esm_code_gens
.iter()
.chain(part_code_gens.iter())
.chain(additional_code_gens.iter().flatten())
.chain(code_gens.iter());
.await
}
}
gen_content_with_code_gens(
#[turbo_tasks::value_impl]
impl EcmascriptModuleContent {
/// Creates a new [`Vc<EcmascriptModuleContent>`].
#[turbo_tasks::function]
pub async fn new(input: Vc<EcmascriptModuleContentOptions>) -> Result<Vc<Self>> {
let input = input.await?;
let EcmascriptModuleContentOptions {
parsed,
*ident,
ident,
specified_module_type,
code_gens,
generate_source_map,
original_source_map,
)
..
} = &*input;
let code_gens = input.merged_code_gens().await?;
async {
let content = process_parse_result(
*parsed,
**ident,
*specified_module_type,
code_gens,
*generate_source_map,
*original_source_map,
)
.await?;
emit_content(content).await
}
.instrument(tracing::info_span!("gen content with code gens"))
.await
}
......@@ -913,29 +939,39 @@ impl EcmascriptModuleContent {
specified_module_type: SpecifiedModuleType,
generate_source_map: bool,
) -> Result<Vc<Self>> {
gen_content_with_code_gens(
let content = process_parse_result(
parsed.to_resolved().await?,
ident,
specified_module_type,
&[],
vec![],
generate_source_map,
OptionStringifiedSourceMap::none().to_resolved().await?,
)
.await
.await?;
emit_content(content).await
}
}
async fn gen_content_with_code_gens(
struct CodeGenResult {
program: Program,
source_map: Arc<SourceMap>,
comments: Either<ImmutableComments, Arc<ImmutableComments>>,
is_esm: bool,
generate_source_map: bool,
original_source_map: Option<ResolvedVc<OptionStringifiedSourceMap>>,
}
async fn process_parse_result(
parsed: ResolvedVc<ParseResult>,
ident: Vc<AssetIdent>,
specified_module_type: SpecifiedModuleType,
code_gens: impl IntoIterator<Item = &CodeGeneration>,
code_gens: Vec<CodeGeneration>,
generate_source_map: bool,
original_source_map: ResolvedVc<OptionStringifiedSourceMap>,
) -> Result<Vc<EcmascriptModuleContent>> {
) -> Result<CodeGenResult> {
let parsed = parsed.final_read_hint().await?;
match &*parsed {
Ok(match &*parsed {
ParseResult::Ok { .. } => {
// We need a mutable version of the AST. We try to avoid cloning it by unwrapping the
// ReadRef.
......@@ -978,102 +1014,150 @@ async fn gen_content_with_code_gens(
}
_ => unreachable!(),
};
let top_level_mark = eval_context.top_level_mark;
let is_esm = eval_context.is_esm(specified_module_type);
process_content_with_code_gens(
&mut program,
globals,
Some(eval_context.top_level_mark),
code_gens,
);
process_content_with_code_gens(&mut program, globals, Some(top_level_mark), code_gens);
let mut bytes: Vec<u8> = vec![];
// TODO: Insert this as a sourceless segment so that sourcemaps aren't affected.
// = format!("/* {} */\n", self.module.path().to_string().await?).into_bytes();
let mut mappings = vec![];
{
let comments = match comments {
Either::Left(comments) => Either::Left(comments.into_consumable()),
Either::Right(ref comments) => Either::Right(comments.consumable()),
};
let comments: &dyn Comments = match &comments {
Either::Left(comments) => comments,
Either::Right(comments) => comments,
};
let mut emitter = Emitter {
cfg: swc_core::ecma::codegen::Config::default(),
cm: source_map.clone(),
comments: Some(&comments),
wr: JsWriter::new(
source_map.clone(),
"\n",
&mut bytes,
generate_source_map.then_some(&mut mappings),
),
};
emitter.emit_program(&program)?;
CodeGenResult {
program,
source_map: source_map.clone(),
comments,
is_esm,
generate_source_map,
original_source_map: Some(original_source_map),
}
}
ParseResult::Unparseable { messages } => {
let path = ident.path().to_string().await?;
let error_messages = messages
.as_ref()
.and_then(|m| m.first().map(|f| format!("\n{}", f)))
.unwrap_or("".into());
let msg = format!("Could not parse module '{path}'\n{error_messages}");
let body = vec![
quote!(
"const e = new Error($msg);" as Stmt,
msg: Expr = Expr::Lit(msg.into()),
),
quote!("e.code = 'MODULE_UNPARSEABLE';" as Stmt),
quote!("throw e;" as Stmt),
];
let source_map = if generate_source_map {
Some(generate_js_source_map(
source_map.clone(),
mappings,
original_source_map.await?.as_ref(),
)?)
} else {
None
};
Ok(EcmascriptModuleContent {
inner_code: bytes.into(),
source_map,
is_esm: eval_context.is_esm(specified_module_type),
CodeGenResult {
program: Program::Script(Script {
span: DUMMY_SP,
body,
shebang: None,
}),
source_map: Arc::new(SourceMap::default()),
comments: Either::Left(Default::default()),
is_esm: false,
generate_source_map: false,
original_source_map: None,
}
.cell())
}
ParseResult::Unparseable { messages } => Ok(EcmascriptModuleContent {
inner_code: format!(
"const e = new Error(`Could not parse module \
'{path}'\n{error_messages}`);\ne.code = 'MODULE_UNPARSEABLE';\nthrow e;",
path = ident.path().to_string().await?,
error_messages = messages
.as_ref()
.and_then(|m| { m.first().map(|f| format!("\n{}", f)) })
.unwrap_or("".into())
)
.into(),
source_map: None,
is_esm: false,
ParseResult::NotFound => {
let path = ident.path().to_string().await?;
let msg = format!("Could not parse module '{path}'");
let body = vec![
quote!(
"const e = new Error($msg);" as Stmt,
msg: Expr = Expr::Lit(msg.into()),
),
quote!("e.code = 'MODULE_UNPARSEABLE';" as Stmt),
quote!("throw e;" as Stmt),
];
CodeGenResult {
program: Program::Script(Script {
span: DUMMY_SP,
body,
shebang: None,
}),
source_map: Arc::new(SourceMap::default()),
comments: Either::Left(Default::default()),
is_esm: false,
generate_source_map: false,
original_source_map: None,
}
}
.cell()),
ParseResult::NotFound => Ok(EcmascriptModuleContent {
inner_code: format!(
"const e = new Error(\"Could not parse module '{path}'\");\ne.code = \
'MODULE_UNPARSEABLE';\nthrow e;",
path = ident.path().to_string().await?
)
.into(),
source_map: None,
is_esm: false,
})
}
async fn emit_content(content: CodeGenResult) -> Result<Vc<EcmascriptModuleContent>> {
let CodeGenResult {
program,
source_map,
comments,
is_esm,
generate_source_map,
original_source_map,
} = content;
let mut bytes: Vec<u8> = vec![];
// TODO: Insert this as a sourceless segment so that sourcemaps aren't affected.
// = format!("/* {} */\n", self.module.path().to_string().await?).into_bytes();
let mut mappings = vec![];
{
let comments = match comments {
Either::Left(comments) => Either::Left(comments.into_consumable()),
Either::Right(ref comments) => Either::Right(comments.consumable()),
};
let comments: &dyn Comments = match &comments {
Either::Left(comments) => comments,
Either::Right(comments) => comments,
};
let mut emitter = Emitter {
cfg: swc_core::ecma::codegen::Config::default(),
cm: source_map.clone(),
comments: Some(&comments),
wr: JsWriter::new(
source_map.clone(),
"\n",
&mut bytes,
generate_source_map.then_some(&mut mappings),
),
};
emitter.emit_program(&program)?;
}
let source_map = if generate_source_map {
if let Some(original_source_map) = original_source_map {
Some(generate_js_source_map(
source_map.clone(),
mappings,
original_source_map.await?.as_ref(),
)?)
} else {
Some(generate_js_source_map(source_map.clone(), mappings, None)?)
}
.cell()),
} else {
None
};
Ok(EcmascriptModuleContent {
inner_code: bytes.into(),
source_map,
is_esm,
}
.cell())
}
fn process_content_with_code_gens<'a>(
fn process_content_with_code_gens(
program: &mut Program,
globals: &Globals,
top_level_mark: Option<Mark>,
code_gens: impl IntoIterator<Item = &'a CodeGeneration>,
code_gens: Vec<CodeGeneration>,
) {
let mut visitors = Vec::new();
let mut root_visitors = Vec::new();
let mut early_hoisted_stmts = FxIndexMap::default();
let mut hoisted_stmts = FxIndexMap::default();
for code_gen in code_gens {
for code_gen in &code_gens {
for CodeGenerationHoistedStmt { key, stmt } in &code_gen.hoisted_stmts {
hoisted_stmts.entry(key.clone()).or_insert(stmt.clone());
}
......
......@@ -204,32 +204,31 @@ impl EcmascriptAnalyzable for EcmascriptModuleFacadeModule {
}
#[turbo_tasks::function]
async fn module_content(
async fn module_content_options(
self: Vc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContent>> {
) -> Result<Vc<EcmascriptModuleContentOptions>> {
let (esm_references, part_references) = self.await?.specific_references().await?;
Ok(EcmascriptModuleContent::new(
EcmascriptModuleContentOptions {
parsed: ParseResult::empty().to_resolved().await?,
ident: self.ident().to_resolved().await?,
specified_module_type: SpecifiedModuleType::EcmaScript,
module_graph,
chunking_context,
references: self.references().to_resolved().await?,
esm_references,
part_references,
code_generation: CodeGens::empty().to_resolved().await?,
async_module: ResolvedVc::cell(Some(self.async_module().to_resolved().await?)),
generate_source_map: false,
original_source_map: OptionStringifiedSourceMap::none().to_resolved().await?,
exports: self.get_exports().to_resolved().await?,
async_module_info,
},
))
Ok(EcmascriptModuleContentOptions {
parsed: ParseResult::empty().to_resolved().await?,
ident: self.ident().to_resolved().await?,
specified_module_type: SpecifiedModuleType::EcmaScript,
module_graph,
chunking_context,
references: self.references().to_resolved().await?,
esm_references,
part_references,
code_generation: CodeGens::empty().to_resolved().await?,
async_module: ResolvedVc::cell(Some(self.async_module().to_resolved().await?)),
generate_source_map: false,
original_source_map: OptionStringifiedSourceMap::none().to_resolved().await?,
exports: self.get_exports().to_resolved().await?,
async_module_info,
}
.cell())
}
}
......
......@@ -92,12 +92,12 @@ impl EcmascriptAnalyzable for EcmascriptModuleLocalsModule {
}
#[turbo_tasks::function]
async fn module_content(
async fn module_content_options(
self: Vc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContent>> {
) -> Result<Vc<EcmascriptModuleContentOptions>> {
let exports = self.get_exports().to_resolved().await?;
let original_module = self.await?.module;
let parsed = original_module.parse().to_resolved().await?;
......@@ -110,24 +110,23 @@ impl EcmascriptAnalyzable for EcmascriptModuleLocalsModule {
.reference_module_source_maps(Vc::upcast(self))
.await?;
Ok(EcmascriptModuleContent::new(
EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.local_references().to_resolved().await?,
esm_references: analyze_result.esm_local_references,
part_references: vec![],
code_generation: analyze_result.code_generation,
async_module: analyze_result.async_module,
generate_source_map,
original_source_map: analyze_result.source_map,
exports,
async_module_info,
},
))
Ok(EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.local_references().to_resolved().await?,
esm_references: analyze_result.esm_local_references,
part_references: vec![],
code_generation: analyze_result.code_generation,
async_module: analyze_result.async_module,
generate_source_map,
original_source_map: analyze_result.source_map,
exports,
async_module_info,
}
.cell())
}
}
......
......@@ -75,12 +75,12 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset {
}
#[turbo_tasks::function]
async fn module_content(
async fn module_content_options(
self: Vc<Self>,
module_graph: ResolvedVc<ModuleGraph>,
chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
async_module_info: Option<ResolvedVc<AsyncModuleInfo>>,
) -> Result<Vc<EcmascriptModuleContent>> {
) -> Result<Vc<EcmascriptModuleContentOptions>> {
let module = self.await?;
let split_data = split_module(*module.full_module);
......@@ -96,24 +96,23 @@ impl EcmascriptAnalyzable for EcmascriptModulePartAsset {
.reference_module_source_maps(Vc::upcast(self))
.await?;
Ok(EcmascriptModuleContent::new(
EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.references().to_resolved().await?,
esm_references: analyze_ref.esm_references,
part_references: vec![],
code_generation: analyze_ref.code_generation,
async_module: analyze_ref.async_module,
generate_source_map,
original_source_map: analyze_ref.source_map,
exports: analyze_ref.exports,
async_module_info,
},
))
Ok(EcmascriptModuleContentOptions {
parsed,
ident: self.ident().to_resolved().await?,
specified_module_type: module_type_result.module_type,
module_graph,
chunking_context,
references: analyze.references().to_resolved().await?,
esm_references: analyze_ref.esm_references,
part_references: vec![],
code_generation: analyze_ref.code_generation,
async_module: analyze_ref.async_module,
generate_source_map,
original_source_map: analyze_ref.source_map,
exports: analyze_ref.exports,
async_module_info,
}
.cell())
}
}
......
......@@ -6,5 +6,6 @@ var { g: global, __dirname, m: module, e: exports } = __turbopack_context__;
{
const e = new Error("Could not parse module '[project]/turbopack/crates/turbopack-tests/tests/snapshot/comptime/unreachable/input/index.js'");
e.code = 'MODULE_UNPARSEABLE';
throw e;}}),
throw e;
}}),
}]);
\ No newline at end of file
......@@ -6,5 +6,6 @@ var { g: global, __dirname, m: module, e: exports } = __turbopack_context__;
{
const e = new Error("Could not parse module '[project]/turbopack/crates/turbopack-tests/tests/snapshot/mdx/error/input/index.js'");
e.code = 'MODULE_UNPARSEABLE';
throw e;}}),
throw e;
}}),
}]);
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment