Unverified Commit 9b994bf8 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: rename reverse topological to postorder topological, fix async…

Turbopack: rename reverse topological to postorder topological, fix async referenced modules (#76429)

### What?

* move iter_neighbors into shared file, rename to iter_neighbors_rev
* fix async referenced modules by filtering by chunking type
* rename reverse topological to postorder topological
parent e6f3782c
...@@ -1678,7 +1678,7 @@ async fn any_output_changed( ...@@ -1678,7 +1678,7 @@ async fn any_output_changed(
.await .await
.completed()? .completed()?
.into_inner() .into_inner()
.into_reverse_topological() .into_postorder_topological()
.map(|m| async move { .map(|m| async move {
let asset_path = m.path().await?; let asset_path = m.path().await?;
if !asset_path.path.ends_with(".map") if !asset_path.path.ends_with(".map")
......
...@@ -112,7 +112,7 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out ...@@ -112,7 +112,7 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out
.await .await
.completed()? .completed()?
.into_inner() .into_inner()
.into_reverse_topological() .into_postorder_topological()
.collect::<FxHashSet<_>>() .collect::<FxHashSet<_>>()
.into_iter() .into_iter()
.collect(), .collect(),
......
...@@ -174,7 +174,7 @@ pub async fn find_server_entries(entry: ResolvedVc<Box<dyn Module>>) -> Result<V ...@@ -174,7 +174,7 @@ pub async fn find_server_entries(entry: ResolvedVc<Box<dyn Module>>) -> Result<V
let mut server_component_entries = vec![]; let mut server_component_entries = vec![];
let mut server_utils = vec![]; let mut server_utils = vec![];
for node in graph.reverse_topological() { for node in graph.postorder_topological() {
match &node.ty { match &node.ty {
VisitClientReferenceNodeType::ServerUtilEntry(server_util, _) => { VisitClientReferenceNodeType::ServerUtilEntry(server_util, _) => {
server_utils.push(*server_util); server_utils.push(*server_util);
......
...@@ -88,10 +88,10 @@ impl<T> AdjacencyMap<T> ...@@ -88,10 +88,10 @@ impl<T> AdjacencyMap<T>
where where
T: Eq + Hash + Clone, T: Eq + Hash + Clone,
{ {
/// Returns an owned iterator over the nodes in reverse topological order, /// Returns an owned iterator over the nodes in postorder topological order,
/// starting from the roots. /// starting from the roots.
pub fn into_reverse_topological(self) -> IntoReverseTopologicalIter<T> { pub fn into_postorder_topological(self) -> IntoPostorderTopologicalIter<T> {
IntoReverseTopologicalIter { IntoPostorderTopologicalIter {
adjacency_map: self.adjacency_map, adjacency_map: self.adjacency_map,
stack: self stack: self
.roots .roots
...@@ -108,7 +108,7 @@ where ...@@ -108,7 +108,7 @@ where
pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges<T> { pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges<T> {
IntoBreadthFirstEdges { IntoBreadthFirstEdges {
adjacency_map: self.adjacency_map, adjacency_map: self.adjacency_map,
stack: self queue: self
.roots .roots
.into_iter() .into_iter()
.rev() .rev()
...@@ -118,10 +118,10 @@ where ...@@ -118,10 +118,10 @@ where
} }
} }
/// Returns an iterator over the nodes in reverse topological order, /// Returns an iterator over the nodes in postorder topological order,
/// starting from the roots. /// starting from the roots.
pub fn reverse_topological(&self) -> ReverseTopologicalIter<T> { pub fn postorder_topological(&self) -> PostorderTopologicalIter<T> {
ReverseTopologicalIter { PostorderTopologicalIter {
adjacency_map: &self.adjacency_map, adjacency_map: &self.adjacency_map,
stack: self stack: self
.roots .roots
...@@ -133,13 +133,13 @@ where ...@@ -133,13 +133,13 @@ where
} }
} }
/// Returns an iterator over the nodes in reverse topological order, /// Returns an iterator over the nodes in postorder topological order,
/// starting from the given node. /// starting from the given node.
pub fn reverse_topological_from_node<'graph>( pub fn postorder_topological_from_node<'graph>(
&'graph self, &'graph self,
node: &'graph T, node: &'graph T,
) -> ReverseTopologicalIter<'graph, T> { ) -> PostorderTopologicalIter<'graph, T> {
ReverseTopologicalIter { PostorderTopologicalIter {
adjacency_map: &self.adjacency_map, adjacency_map: &self.adjacency_map,
stack: vec![(ReverseTopologicalPass::Pre, node)], stack: vec![(ReverseTopologicalPass::Pre, node)],
visited: FxHashSet::default(), visited: FxHashSet::default(),
...@@ -153,9 +153,9 @@ enum ReverseTopologicalPass { ...@@ -153,9 +153,9 @@ enum ReverseTopologicalPass {
Post, Post,
} }
/// An iterator over the nodes of a graph in reverse topological order, starting /// An iterator over the nodes of a graph in postorder topological order, starting
/// from the roots. /// from the roots.
pub struct IntoReverseTopologicalIter<T> pub struct IntoPostorderTopologicalIter<T>
where where
T: Eq + Hash + Clone, T: Eq + Hash + Clone,
{ {
...@@ -164,7 +164,7 @@ where ...@@ -164,7 +164,7 @@ where
visited: FxHashSet<T>, visited: FxHashSet<T>,
} }
impl<T> Iterator for IntoReverseTopologicalIter<T> impl<T> Iterator for IntoPostorderTopologicalIter<T>
where where
T: Eq + Hash + Clone, T: Eq + Hash + Clone,
{ {
...@@ -209,7 +209,7 @@ where ...@@ -209,7 +209,7 @@ where
T: Eq + std::hash::Hash + Clone, T: Eq + std::hash::Hash + Clone,
{ {
adjacency_map: FxHashMap<T, Vec<T>>, adjacency_map: FxHashMap<T, Vec<T>>,
stack: VecDeque<(Option<T>, T)>, queue: VecDeque<(Option<T>, T)>,
expanded: FxHashSet<T>, expanded: FxHashSet<T>,
} }
...@@ -220,14 +220,14 @@ where ...@@ -220,14 +220,14 @@ where
type Item = (Option<T>, T); type Item = (Option<T>, T);
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let (parent, current) = self.stack.pop_front()?; let (parent, current) = self.queue.pop_front()?;
let Some(neighbors) = self.adjacency_map.get(&current) else { let Some(neighbors) = self.adjacency_map.get(&current) else {
return Some((parent, current)); return Some((parent, current));
}; };
if self.expanded.insert(current.clone()) { if self.expanded.insert(current.clone()) {
self.stack.extend( self.queue.extend(
neighbors neighbors
.iter() .iter()
.map(|neighbor| (Some(current.clone()), neighbor.clone())), .map(|neighbor| (Some(current.clone()), neighbor.clone())),
...@@ -238,9 +238,9 @@ where ...@@ -238,9 +238,9 @@ where
} }
} }
/// An iterator over the nodes of a graph in reverse topological order, starting /// An iterator over the nodes of a graph in postorder topological order, starting
/// from the roots. /// from the roots.
pub struct ReverseTopologicalIter<'graph, T> pub struct PostorderTopologicalIter<'graph, T>
where where
T: Eq + Hash + Clone, T: Eq + Hash + Clone,
{ {
...@@ -249,7 +249,7 @@ where ...@@ -249,7 +249,7 @@ where
visited: FxHashSet<&'graph T>, visited: FxHashSet<&'graph T>,
} }
impl<'graph, T> Iterator for ReverseTopologicalIter<'graph, T> impl<'graph, T> Iterator for PostorderTopologicalIter<'graph, T>
where where
T: Eq + Hash + Clone, T: Eq + Hash + Clone,
{ {
......
...@@ -191,6 +191,19 @@ pub enum ChunkingType { ...@@ -191,6 +191,19 @@ pub enum ChunkingType {
Traced, Traced,
} }
impl ChunkingType {
pub fn is_inherit_async(&self) -> bool {
match self {
ChunkingType::Parallel => false,
ChunkingType::ParallelInheritAsync => true,
ChunkingType::Async => false,
ChunkingType::Isolated { .. } => false,
ChunkingType::Shared { inherit_async, .. } => *inherit_async,
ChunkingType::Traced => false,
}
}
}
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct ChunkingTypeOption(Option<ChunkingType>); pub struct ChunkingTypeOption(Option<ChunkingType>);
......
...@@ -7,7 +7,6 @@ use std::{ ...@@ -7,7 +7,6 @@ use std::{
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use either::Either; use either::Either;
use indexmap::map::Entry; use indexmap::map::Entry;
use petgraph::graph::{DiGraph, EdgeIndex, NodeIndex};
use roaring::RoaringBitmap; use roaring::RoaringBitmap;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
...@@ -22,8 +21,8 @@ use crate::{ ...@@ -22,8 +21,8 @@ use crate::{
chunk::{ChunkGroupType, ChunkingType}, chunk::{ChunkGroupType, ChunkingType},
module::Module, module::Module,
module_graph::{ module_graph::{
get_node, get_node_idx, GraphNodeIndex, GraphTraversalAction, ModuleGraph, get_node, get_node_idx, traced_di_graph::iter_neighbors_rev, GraphNodeIndex,
SingleModuleGraphModuleNode, SingleModuleGraphNode, GraphTraversalAction, ModuleGraph, SingleModuleGraphModuleNode, SingleModuleGraphNode,
}, },
}; };
...@@ -178,14 +177,6 @@ impl Deref for ChunkGroupId { ...@@ -178,14 +177,6 @@ impl Deref for ChunkGroupId {
} }
} }
fn iter_neighbors<N, E>(
graph: &DiGraph<N, E>,
node: NodeIndex,
) -> impl Iterator<Item = (EdgeIndex, NodeIndex)> + '_ {
let mut walker = graph.neighbors(node).detach();
std::iter::from_fn(move || walker.next(graph))
}
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
struct NodeWithPriority { struct NodeWithPriority {
depth: usize, depth: usize,
...@@ -448,7 +439,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro ...@@ -448,7 +439,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
queue_set.remove(&node); queue_set.remove(&node);
let (node_weight, node) = get_node_idx!(graphs, node)?; let (node_weight, node) = get_node_idx!(graphs, node)?;
let graph = &graphs[node.graph_idx].graph; let graph = &graphs[node.graph_idx].graph;
let neighbors = iter_neighbors(graph, node.node_idx); let neighbors = iter_neighbors_rev(graph, node.node_idx);
visit_count += 1; visit_count += 1;
......
...@@ -26,7 +26,7 @@ use crate::{ ...@@ -26,7 +26,7 @@ use crate::{
module_graph::{ module_graph::{
async_module_info::{compute_async_module_info, AsyncModulesInfo}, async_module_info::{compute_async_module_info, AsyncModulesInfo},
chunk_group_info::{compute_chunk_group_info, ChunkGroupInfo}, chunk_group_info::{compute_chunk_group_info, ChunkGroupInfo},
traced_di_graph::TracedDiGraph, traced_di_graph::{iter_neighbors_rev, TracedDiGraph},
}, },
reference::primary_chunkable_referenced_modules, reference::primary_chunkable_referenced_modules,
}; };
...@@ -528,18 +528,14 @@ impl SingleModuleGraph { ...@@ -528,18 +528,14 @@ impl SingleModuleGraph {
let graph = &self.graph; let graph = &self.graph;
let entries = entries.into_iter().map(|e| self.get_module(e).unwrap()); let entries = entries.into_iter().map(|e| self.get_module(e).unwrap());
enum ReverseTopologicalPass { enum TopologicalPass {
Visit, Visit,
ExpandAndVisit, ExpandAndVisit,
} }
#[allow(clippy::type_complexity)] // This is a temporary internal structure #[allow(clippy::type_complexity)] // This is a temporary internal structure
let mut stack: Vec<( let mut stack: Vec<(TopologicalPass, Option<(NodeIndex, EdgeIndex)>, NodeIndex)> = entries
ReverseTopologicalPass, .map(|e| (TopologicalPass::ExpandAndVisit, None, e))
Option<(NodeIndex, EdgeIndex)>,
NodeIndex,
)> = entries
.map(|e| (ReverseTopologicalPass::ExpandAndVisit, None, e))
.collect(); .collect();
let mut expanded = HashSet::new(); let mut expanded = HashSet::new();
while let Some((pass, parent, current)) = stack.pop() { while let Some((pass, parent, current)) = stack.pop() {
...@@ -555,36 +551,33 @@ impl SingleModuleGraph { ...@@ -555,36 +551,33 @@ impl SingleModuleGraph {
) )
}); });
match pass { match pass {
ReverseTopologicalPass::Visit => { TopologicalPass::Visit => {
visit_postorder(parent_arg, graph.node_weight(current).unwrap(), state); visit_postorder(parent_arg, graph.node_weight(current).unwrap(), state);
} }
ReverseTopologicalPass::ExpandAndVisit => { TopologicalPass::ExpandAndVisit => match graph.node_weight(current).unwrap() {
match graph.node_weight(current).unwrap() { current_node @ SingleModuleGraphNode::Module(_) => {
current_node @ SingleModuleGraphNode::Module(_) => { let action = visit_preorder(parent_arg, current_node, state)?;
let action = visit_preorder(parent_arg, current_node, state)?; if action == GraphTraversalAction::Exclude {
if action == GraphTraversalAction::Exclude { continue;
continue;
}
stack.push((ReverseTopologicalPass::Visit, parent, current));
if action == GraphTraversalAction::Continue && expanded.insert(current)
{
stack.extend(iter_neighbors(graph, current).map(
|(edge, child)| {
(
ReverseTopologicalPass::ExpandAndVisit,
Some((current, edge)),
child,
)
},
));
}
} }
current_node @ SingleModuleGraphNode::VisitedModule { .. } => { stack.push((TopologicalPass::Visit, parent, current));
visit_preorder(parent_arg, current_node, state)?; if action == GraphTraversalAction::Continue && expanded.insert(current) {
visit_postorder(parent_arg, current_node, state); stack.extend(iter_neighbors_rev(graph, current).map(
|(edge, child)| {
(
TopologicalPass::ExpandAndVisit,
Some((current, edge)),
child,
)
},
));
} }
} }
} current_node @ SingleModuleGraphNode::VisitedModule { .. } => {
visit_preorder(parent_arg, current_node, state)?;
visit_postorder(parent_arg, current_node, state);
}
},
} }
} }
...@@ -655,7 +648,14 @@ impl ModuleGraph { ...@@ -655,7 +648,14 @@ impl ModuleGraph {
let async_modules_info = self.async_module_info().await?; let async_modules_info = self.async_module_info().await?;
let entry = ModuleGraph::get_entry(&graphs, module).await?; let entry = ModuleGraph::get_entry(&graphs, module).await?;
let referenced_modules = iter_neighbors(&graphs[entry.graph_idx].graph, entry.node_idx) let referenced_modules = iter_neighbors_rev(&graphs[entry.graph_idx].graph, entry.node_idx)
.filter(|(edge_idx, _)| {
let ty = graphs[entry.graph_idx]
.graph
.edge_weight(*edge_idx)
.unwrap();
ty.is_inherit_async()
})
.map(|(_, child_idx)| { .map(|(_, child_idx)| {
anyhow::Ok( anyhow::Ok(
get_node!( get_node!(
...@@ -670,6 +670,7 @@ impl ModuleGraph { ...@@ -670,6 +670,7 @@ impl ModuleGraph {
}) })
.collect::<Result<Vec<_>>>()? .collect::<Result<Vec<_>>>()?
.into_iter() .into_iter()
.rev()
.filter(|m| async_modules_info.contains(m)) .filter(|m| async_modules_info.contains(m))
.map(|m| *m) .map(|m| *m)
.collect(); .collect();
...@@ -808,7 +809,7 @@ impl ModuleGraph { ...@@ -808,7 +809,7 @@ impl ModuleGraph {
let graph = &graphs[node.graph_idx].graph; let graph = &graphs[node.graph_idx].graph;
let node_weight = get_node!(graphs, node)?; let node_weight = get_node!(graphs, node)?;
if visited.insert(node) { if visited.insert(node) {
let neighbors = iter_neighbors(graph, node.node_idx); let neighbors = iter_neighbors_rev(graph, node.node_idx);
for (edge, succ) in neighbors { for (edge, succ) in neighbors {
let succ = GraphNodeIndex { let succ = GraphNodeIndex {
...@@ -861,7 +862,7 @@ impl ModuleGraph { ...@@ -861,7 +862,7 @@ impl ModuleGraph {
let graph = &graphs[node.graph_idx].graph; let graph = &graphs[node.graph_idx].graph;
let node_weight = get_node!(graphs, node)?; let node_weight = get_node!(graphs, node)?;
if visited.insert(node) { if visited.insert(node) {
let neighbors = iter_neighbors(graph, node.node_idx); let neighbors = iter_neighbors_rev(graph, node.node_idx);
for (edge, succ) in neighbors { for (edge, succ) in neighbors {
let succ = GraphNodeIndex { let succ = GraphNodeIndex {
...@@ -953,19 +954,19 @@ impl ModuleGraph { ...@@ -953,19 +954,19 @@ impl ModuleGraph {
let entries = entries.into_iter().collect::<Vec<_>>(); let entries = entries.into_iter().collect::<Vec<_>>();
enum ReverseTopologicalPass { enum TopologicalPass {
Visit, Visit,
ExpandAndVisit, ExpandAndVisit,
} }
#[allow(clippy::type_complexity)] // This is a temporary internal structure #[allow(clippy::type_complexity)] // This is a temporary internal structure
let mut stack: Vec<( let mut stack: Vec<(
ReverseTopologicalPass, TopologicalPass,
Option<(GraphNodeIndex, EdgeIndex)>, Option<(GraphNodeIndex, EdgeIndex)>,
GraphNodeIndex, GraphNodeIndex,
)> = Vec::with_capacity(entries.len()); )> = Vec::with_capacity(entries.len());
for entry in entries.into_iter().rev() { for entry in entries.into_iter().rev() {
stack.push(( stack.push((
ReverseTopologicalPass::ExpandAndVisit, TopologicalPass::ExpandAndVisit,
None, None,
ModuleGraph::get_entry(&graphs, entry).await?, ModuleGraph::get_entry(&graphs, entry).await?,
)); ));
...@@ -984,31 +985,31 @@ impl ModuleGraph { ...@@ -984,31 +985,31 @@ impl ModuleGraph {
}; };
let current_node = get_node!(graphs, current)?; let current_node = get_node!(graphs, current)?;
match pass { match pass {
ReverseTopologicalPass::Visit => { TopologicalPass::Visit => {
visit_postorder(parent_arg, current_node, state); visit_postorder(parent_arg, current_node, state);
} }
ReverseTopologicalPass::ExpandAndVisit => { TopologicalPass::ExpandAndVisit => {
let action = visit_preorder(parent_arg, current_node, state)?; let action = visit_preorder(parent_arg, current_node, state)?;
if action == GraphTraversalAction::Exclude { if action == GraphTraversalAction::Exclude {
continue; continue;
} }
stack.push((ReverseTopologicalPass::Visit, parent, current)); stack.push((TopologicalPass::Visit, parent, current));
if action == GraphTraversalAction::Continue && expanded.insert(current) { if action == GraphTraversalAction::Continue && expanded.insert(current) {
let graph = &graphs[current.graph_idx].graph; let graph = &graphs[current.graph_idx].graph;
let (neighbors, current) = let (neighbors_rev, current) =
match graph.node_weight(current.node_idx).unwrap() { match graph.node_weight(current.node_idx).unwrap() {
SingleModuleGraphNode::Module(_) => { SingleModuleGraphNode::Module(_) => {
(iter_neighbors(graph, current.node_idx), current) (iter_neighbors_rev(graph, current.node_idx), current)
} }
SingleModuleGraphNode::VisitedModule { idx, .. } => ( SingleModuleGraphNode::VisitedModule { idx, .. } => (
// We switch graphs // We switch graphs
iter_neighbors(&graphs[idx.graph_idx].graph, idx.node_idx), iter_neighbors_rev(&graphs[idx.graph_idx].graph, idx.node_idx),
*idx, *idx,
), ),
}; };
stack.extend(neighbors.map(|(edge, child)| { stack.extend(neighbors_rev.map(|(edge, child)| {
( (
ReverseTopologicalPass::ExpandAndVisit, TopologicalPass::ExpandAndVisit,
Some((current, edge)), Some((current, edge)),
GraphNodeIndex { GraphNodeIndex {
graph_idx: current.graph_idx, graph_idx: current.graph_idx,
...@@ -1277,11 +1278,3 @@ impl Visit<SingleModuleGraphBuilderNode> for SingleModuleGraphBuilder<'_> { ...@@ -1277,11 +1278,3 @@ impl Visit<SingleModuleGraphBuilderNode> for SingleModuleGraphBuilder<'_> {
} }
} }
} }
fn iter_neighbors<N, E>(
graph: &DiGraph<N, E>,
node: NodeIndex,
) -> impl Iterator<Item = (EdgeIndex, NodeIndex)> + '_ {
let mut walker = graph.neighbors(node).detach();
std::iter::from_fn(move || walker.next(graph))
}
use std::ops::Deref; use std::ops::Deref;
use petgraph::graph::DiGraph; use petgraph::graph::{DiGraph, EdgeIndex, NodeIndex};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_tasks::{ use turbo_tasks::{
debug::ValueDebugFormat, debug::ValueDebugFormat,
...@@ -50,3 +50,12 @@ where ...@@ -50,3 +50,12 @@ where
E: NonLocalValue, E: NonLocalValue,
{ {
} }
/// Iterate the edges of a node REVERSED!
pub fn iter_neighbors_rev<N, E>(
graph: &DiGraph<N, E>,
node: NodeIndex,
) -> impl Iterator<Item = (EdgeIndex, NodeIndex)> + '_ {
let mut walker = graph.neighbors(node).detach();
std::iter::from_fn(move || walker.next(graph))
}
...@@ -323,7 +323,7 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out ...@@ -323,7 +323,7 @@ pub async fn all_assets_from_entries(entries: Vc<OutputAssets>) -> Result<Vc<Out
.await .await
.completed()? .completed()?
.into_inner() .into_inner()
.into_reverse_topological() .into_postorder_topological()
.collect(), .collect(),
)) ))
} }
......
...@@ -166,7 +166,7 @@ async fn separate_assets_operation( ...@@ -166,7 +166,7 @@ async fn separate_assets_operation(
let mut internal_assets = FxIndexSet::default(); let mut internal_assets = FxIndexSet::default();
let mut external_asset_entrypoints = FxIndexSet::default(); let mut external_asset_entrypoints = FxIndexSet::default();
for item in graph.into_reverse_topological() { for item in graph.into_postorder_topological() {
match item { match item {
Type::Internal(asset) => { Type::Internal(asset) => {
internal_assets.insert(asset); internal_assets.insert(asset);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment