Unverified Commit 92f72373 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Turbopack: refactor graph traveral and fix module graph determinism (#87312)

### What?

Simplifies the graph traversal.
Allows the GraphStore to store edges and nodes.
This no longer forces us to abuse the node type to store edge info, which actually has lead to a race condition problem which made the module graph non-deterministic.

Removes `skip_duplicated` and made the AdjacencyMap always skip duplicates. That simplifies the implementation. We never not skipped duplicates and that rarely makes sense anyway since any cycle would make it traverse forever...

Also removed a bunch of unused stuff...
parent 37521a02
......@@ -6,7 +6,7 @@ use tracing::{Instrument, Level, Span};
use turbo_rcstr::RcStr;
use turbo_tasks::{
FxIndexMap, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc,
graph::{AdjacencyMap, GraphTraversal, Visit, VisitControlFlow},
graph::{AdjacencyMap, GraphTraversal, Visit},
};
use turbo_tasks_fs::{
DirectoryEntry, File, FileContent, FileSystem, FileSystemPath,
......@@ -404,7 +404,6 @@ pub async fn all_assets_from_entries_filtered(
let emit_spans = tracing::enabled!(Level::INFO);
Ok(Vc::cell(
AdjacencyMap::new()
.skip_duplicates()
.visit(
entries
.await?
......@@ -431,7 +430,6 @@ pub async fn all_assets_from_entries_filtered(
)
.await
.completed()?
.into_inner()
.into_postorder_topological()
.map(|n| n.0)
.collect(),
......@@ -446,14 +444,12 @@ struct OutputAssetFilteredVisit {
impl Visit<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>
for OutputAssetFilteredVisit
{
type Edge = (ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>);
type EdgesIntoIter = Vec<Self::Edge>;
type EdgesIntoIter = Vec<(
(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
(),
)>;
type EdgesFuture = impl Future<Output = Result<Self::EdgesIntoIter>>;
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<Self::Edge> {
VisitControlFlow::Continue(edge)
}
fn edges(
&mut self,
node: &(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
......@@ -466,6 +462,7 @@ impl Visit<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>
fn span(
&mut self,
node: &(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
_edge: Option<&()>,
) -> tracing::Span {
if let Some(ident) = &node.1 {
tracing::trace_span!("asset", name = display(ident))
......@@ -482,7 +479,12 @@ async fn get_referenced_server_assets(
asset: ResolvedVc<Box<dyn OutputAsset>>,
client_root: Option<FileSystemPath>,
exclude_glob: Option<ReadRef<Glob>>,
) -> Result<Vec<(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>)>> {
) -> Result<
Vec<(
(ResolvedVc<Box<dyn OutputAsset>>, Option<ReadRef<RcStr>>),
(),
)>,
> {
let refs = asset.references().all_assets().await?;
refs.iter()
......@@ -503,14 +505,17 @@ async fn get_referenced_server_assets(
}
Ok(Some((
*asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate the list of assets when the span
// name changes
Some(asset.path_string().untracked().await?)
} else {
None
},
(
*asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate the list of assets when the
// span name changes
Some(asset.path_string().untracked().await?)
} else {
None
},
),
(),
)))
})
.try_flat_join()
......
......@@ -86,7 +86,6 @@ pub async fn find_server_entries(
async move {
let emit_spans = tracing::enabled!(Level::INFO);
let graph = AdjacencyMap::new()
.skip_duplicates()
.visit(
vec![FindServerEntriesNode::Internal(
entry,
......@@ -104,8 +103,7 @@ pub async fn find_server_entries(
},
)
.await
.completed()?
.into_inner();
.completed()?;
let mut server_component_entries = vec![];
let mut server_utils = vec![];
......@@ -151,16 +149,15 @@ enum FindServerEntriesNode {
}
impl Visit<FindServerEntriesNode> for FindServerEntries {
type Edge = FindServerEntriesNode;
type EdgesIntoIter = Vec<Self::Edge>;
type EdgesIntoIter = Vec<(FindServerEntriesNode, ())>;
type EdgesFuture = impl Future<Output = Result<Self::EdgesIntoIter>>;
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<FindServerEntriesNode> {
match edge {
FindServerEntriesNode::Internal(..) => VisitControlFlow::Continue(edge),
fn visit(&mut self, node: &FindServerEntriesNode, _edge: Option<&()>) -> VisitControlFlow {
match node {
FindServerEntriesNode::Internal(..) => VisitControlFlow::Continue,
FindServerEntriesNode::ClientReference
| FindServerEntriesNode::ServerUtilEntry(..)
| FindServerEntriesNode::ServerComponentEntry(..) => VisitControlFlow::Skip(edge),
| FindServerEntriesNode::ServerComponentEntry(..) => VisitControlFlow::Skip,
}
}
......@@ -202,29 +199,48 @@ impl Visit<FindServerEntriesNode> for FindServerEntries {
|| ResolvedVc::try_downcast_type::<CssClientReferenceModule>(*module)
.is_some()
{
return Ok(FindServerEntriesNode::ClientReference);
return Ok((FindServerEntriesNode::ClientReference, ()));
}
if let Some(server_component_asset) =
ResolvedVc::try_downcast_type::<NextServerComponentModule>(*module)
{
return Ok(FindServerEntriesNode::ServerComponentEntry(
server_component_asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name
// changes
Some(server_component_asset.ident_string().untracked().await?)
} else {
None
},
return Ok((
FindServerEntriesNode::ServerComponentEntry(
server_component_asset,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name
// changes
Some(server_component_asset.ident_string().untracked().await?)
} else {
None
},
),
(),
));
}
if let Some(server_util_module) =
ResolvedVc::try_downcast_type::<NextServerUtilityModule>(*module)
{
return Ok(FindServerEntriesNode::ServerUtilEntry(
server_util_module,
return Ok((
FindServerEntriesNode::ServerUtilEntry(
server_util_module,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name
// changes
Some(module.ident_string().untracked().await?)
} else {
None
},
),
(),
));
}
Ok((
FindServerEntriesNode::Internal(
*module,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name
// changes
......@@ -232,17 +248,8 @@ impl Visit<FindServerEntriesNode> for FindServerEntries {
} else {
None
},
));
}
Ok(FindServerEntriesNode::Internal(
*module,
if emit_spans {
// INVALIDATION: we don't need to invalidate when the span name changes
Some(module.ident_string().untracked().await?)
} else {
None
},
),
(),
))
});
......@@ -252,9 +259,9 @@ impl Visit<FindServerEntriesNode> for FindServerEntries {
}
}
fn span(&mut self, node: &FindServerEntriesNode) -> tracing::Span {
fn span(&mut self, node: &FindServerEntriesNode, _edge: Option<&()>) -> tracing::Span {
if !self.emit_spans {
return Span::current();
return Span::none();
}
match node {
FindServerEntriesNode::ClientReference => {
......
use std::{collections::VecDeque, hash::Hash};
use std::{
collections::{VecDeque, hash_map::Entry},
hash::Hash,
};
use rustc_hash::{FxHashMap, FxHashSet};
use turbo_tasks_macros::{TraceRawVcs, ValueDebugFormat};
use crate::{
self as turbo_tasks, NonLocalValue,
graph::graph_store::{GraphNode, GraphStore},
};
use crate::{self as turbo_tasks, NonLocalValue, graph::graph_store::GraphStore};
/// A graph traversal that builds an adjacency map
#[derive(Debug, Clone, TraceRawVcs, ValueDebugFormat)]
pub struct AdjacencyMap<T> {
adjacency_map: FxHashMap<T, Vec<T>>,
pub struct AdjacencyMap<T, E> {
adjacency_map: FxHashMap<T, Vec<(T, E)>>,
roots: Vec<T>,
}
unsafe impl<T> NonLocalValue for AdjacencyMap<T> where T: NonLocalValue {}
impl<T> PartialEq for AdjacencyMap<T>
unsafe impl<T, E> NonLocalValue for AdjacencyMap<T, E>
where
T: NonLocalValue,
E: NonLocalValue,
{
}
impl<T, E> PartialEq for AdjacencyMap<T, E>
where
T: Eq + Hash,
E: Eq,
{
fn eq(&self, other: &Self) -> bool {
self.adjacency_map == other.adjacency_map && self.roots == other.roots
}
}
impl<T> Eq for AdjacencyMap<T> where T: Eq + Hash {}
impl<T, E> Eq for AdjacencyMap<T, E>
where
T: Eq + Hash,
E: Eq,
{
}
impl<T> Default for AdjacencyMap<T>
impl<T, E> Default for AdjacencyMap<T, E>
where
T: Eq + Hash + Clone,
{
......@@ -37,7 +47,7 @@ where
}
}
impl<T> AdjacencyMap<T>
impl<T, E> AdjacencyMap<T, E>
where
T: Eq + Hash + Clone,
{
......@@ -55,39 +65,60 @@ where
}
/// Returns an iterator over the children of the given node
pub fn get(&self, node: &T) -> Option<impl Iterator<Item = &T>> {
pub fn get(&self, node: &T) -> Option<impl Iterator<Item = &(T, E)>> {
self.adjacency_map.get(node).map(|vec| vec.iter())
}
/// Returns the number of nodes in the graph
pub fn len(&self) -> usize {
self.adjacency_map.len()
}
/// Returns true if the graph is empty
pub fn is_empty(&self) -> bool {
self.adjacency_map.is_empty()
}
}
impl<T> GraphStore for AdjacencyMap<T>
impl<T, E> GraphStore for AdjacencyMap<T, E>
where
T: Eq + Hash + Clone + Send,
E: Send,
{
type Node = T;
type Edge = E;
type Handle = T;
fn insert(&mut self, from_handle: Option<T>, node: GraphNode<T>) -> Option<(Self::Handle, &T)> {
let vec = if let Some(from_handle) = from_handle {
self.adjacency_map
.entry(from_handle)
.or_insert_with(|| Vec::with_capacity(1))
fn insert(&mut self, from: Option<(&T, E)>, node: T) {
if let Some((from_node, edge)) = from {
let vec = self
.adjacency_map
.entry(from_node.clone())
.or_insert_with(|| Vec::with_capacity(1));
vec.push((node, edge));
} else {
&mut self.roots
self.roots.push(node);
};
}
vec.push(node.node().clone());
Some((node.into_node(), vec.last().unwrap()))
fn try_enter(&mut self, node: &T) -> Option<T> {
match self.adjacency_map.entry(node.clone()) {
Entry::Occupied(_) => None,
Entry::Vacant(e) => {
e.insert(Vec::new());
Some(node.clone())
}
}
}
}
impl<T> AdjacencyMap<T>
impl<T, E> AdjacencyMap<T, E>
where
T: Eq + Hash + Clone,
{
/// Returns an owned iterator over the nodes in postorder topological order,
/// starting from the roots.
pub fn into_postorder_topological(self) -> IntoPostorderTopologicalIter<T> {
pub fn into_postorder_topological(self) -> IntoPostorderTopologicalIter<T, E> {
IntoPostorderTopologicalIter {
adjacency_map: self.adjacency_map,
stack: self
......@@ -102,7 +133,7 @@ where
/// Returns an owned iterator over all edges (node pairs) in reverse breadth first order,
/// starting from the roots.
pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges<T> {
pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges<T, E> {
IntoBreadthFirstEdges {
adjacency_map: self.adjacency_map,
queue: self
......@@ -117,7 +148,7 @@ where
/// Returns an iterator over the nodes in postorder topological order,
/// starting from the roots.
pub fn postorder_topological(&self) -> PostorderTopologicalIter<'_, T> {
pub fn postorder_topological(&self) -> PostorderTopologicalIter<'_, T, E> {
PostorderTopologicalIter {
adjacency_map: &self.adjacency_map,
stack: self
......@@ -135,7 +166,7 @@ where
pub fn postorder_topological_from_node<'graph>(
&'graph self,
node: &'graph T,
) -> PostorderTopologicalIter<'graph, T> {
) -> PostorderTopologicalIter<'graph, T, E> {
PostorderTopologicalIter {
adjacency_map: &self.adjacency_map,
stack: vec![(ReverseTopologicalPass::Pre, node)],
......@@ -152,18 +183,19 @@ enum ReverseTopologicalPass {
/// An iterator over the nodes of a graph in postorder topological order, starting
/// from the roots.
pub struct IntoPostorderTopologicalIter<T>
pub struct IntoPostorderTopologicalIter<T, E>
where
T: Eq + Hash + Clone,
{
adjacency_map: FxHashMap<T, Vec<T>>,
adjacency_map: FxHashMap<T, Vec<(T, E)>>,
stack: Vec<(ReverseTopologicalPass, T)>,
visited: FxHashSet<T>,
}
impl<T> Iterator for IntoPostorderTopologicalIter<T>
impl<T, E> Iterator for IntoPostorderTopologicalIter<T, E>
where
T: Eq + Hash + Clone,
E: Clone,
{
type Item = T;
......@@ -191,7 +223,7 @@ where
neighbors
.iter()
.rev()
.map(|neighbor| (ReverseTopologicalPass::Pre, neighbor.clone())),
.map(|(neighbor, _)| (ReverseTopologicalPass::Pre, neighbor.clone())),
);
}
}
......@@ -201,20 +233,21 @@ where
}
}
pub struct IntoBreadthFirstEdges<T>
pub struct IntoBreadthFirstEdges<T, E>
where
T: Eq + std::hash::Hash + Clone,
{
adjacency_map: FxHashMap<T, Vec<T>>,
queue: VecDeque<(Option<T>, T)>,
adjacency_map: FxHashMap<T, Vec<(T, E)>>,
queue: VecDeque<(Option<(T, E)>, T)>,
expanded: FxHashSet<T>,
}
impl<T> Iterator for IntoBreadthFirstEdges<T>
impl<T, E> Iterator for IntoBreadthFirstEdges<T, E>
where
T: Eq + std::hash::Hash + Clone,
E: Clone,
{
type Item = (Option<T>, T);
type Item = (Option<(T, E)>, T);
fn next(&mut self) -> Option<Self::Item> {
let (parent, current) = self.queue.pop_front()?;
......@@ -225,9 +258,9 @@ where
if self.expanded.insert(current.clone()) {
self.queue.extend(
neighbors
.iter()
.map(|neighbor| (Some(current.clone()), neighbor.clone())),
neighbors.iter().map(|(neighbor, edge)| {
(Some((current.clone(), edge.clone())), neighbor.clone())
}),
);
}
......@@ -237,16 +270,16 @@ where
/// An iterator over the nodes of a graph in postorder topological order, starting
/// from the roots.
pub struct PostorderTopologicalIter<'graph, T>
pub struct PostorderTopologicalIter<'graph, T, E>
where
T: Eq + Hash + Clone,
{
adjacency_map: &'graph FxHashMap<T, Vec<T>>,
adjacency_map: &'graph FxHashMap<T, Vec<(T, E)>>,
stack: Vec<(ReverseTopologicalPass, &'graph T)>,
visited: FxHashSet<&'graph T>,
}
impl<'graph, T> Iterator for PostorderTopologicalIter<'graph, T>
impl<'graph, T, E> Iterator for PostorderTopologicalIter<'graph, T, E>
where
T: Eq + Hash + Clone,
{
......@@ -276,7 +309,7 @@ where
neighbors
.iter()
.rev()
.map(|neighbor| (ReverseTopologicalPass::Pre, neighbor)),
.map(|(neighbor, _)| (ReverseTopologicalPass::Pre, neighbor)),
);
}
}
......
/// The control flow of visiting an edge during a graph traversal.
pub enum VisitControlFlow<Node, Abort = !> {
/// The traversal should continue on the outgoing edges of the given node.
Continue(Node),
/// The traversal should skip visiting the edges the given node.
Skip(Node),
pub enum VisitControlFlow {
/// The edge is included, and the traversal should continue on the outgoing edges of the given
/// node.
Continue,
/// The edge is included, but the traversal should skip visiting the edges the given node.
Skip,
/// The edge is excluded, and the traversal should not continue on the outgoing edges of the
/// given node.
Exclude,
/// The traversal should abort and return immediately.
Abort(Abort),
}
impl<Node, Abort> VisitControlFlow<Node, Abort> {
/// Map the continue and skip values of this control flow.
pub fn map_node<Map, Mapped>(self, mut map: Map) -> VisitControlFlow<Mapped, Abort>
where
Map: FnMut(Node) -> Mapped,
{
match self {
VisitControlFlow::Continue(node) => VisitControlFlow::Continue(map(node)),
VisitControlFlow::Skip(node) => VisitControlFlow::Skip(map(node)),
VisitControlFlow::Abort(abort) => VisitControlFlow::Abort(abort),
}
}
/// Map the abort value of this control flow.
pub fn map_abort<Map, Mapped>(self, mut map: Map) -> VisitControlFlow<Node, Mapped>
where
Map: FnMut(Abort) -> Mapped,
{
match self {
VisitControlFlow::Continue(node) => VisitControlFlow::Continue(node),
VisitControlFlow::Skip(node) => VisitControlFlow::Skip(node),
VisitControlFlow::Abort(abort) => VisitControlFlow::Abort(map(abort)),
}
}
Abort,
}
use std::hash::Hash;
use rustc_hash::FxHashSet;
use super::VisitedNodes;
/// A graph store is a data structure that will be built up during a graph
/// traversal. It is used to store the results of the traversal.
pub trait GraphStore: Send {
type Node: Send;
type Handle: Clone + Send;
type Edge: Send;
type Handle: Send;
// TODO(alexkirsz) An `entry(from_handle) -> Entry` API would be more
// efficient, as right now we're getting the same key multiple times.
/// Inserts a node into the graph store, and returns a handle to it.
///
/// If this method returns `None`, the node edges will not be visited.
fn insert(
&mut self,
from_handle: Option<Self::Handle>,
node: GraphNode<Self::Node>,
) -> Option<(Self::Handle, &Self::Node)>;
}
/// Utility type to ensure that GraphStore::insert can only ever be called from
/// within this module, as a GraphNode can't be constructed outside of it.
#[derive(Clone, Copy, Eq, PartialEq, Debug, Hash, Ord, PartialOrd)]
pub struct GraphNode<Node>(pub(super) Node);
impl<Node> GraphNode<Node> {
/// Consumes this `GraphNode` and returns the underlying node.
pub fn into_node(self) -> Node {
self.0
}
/// Returns a reference the underlying node.
pub fn node(&self) -> &Node {
&self.0
}
/// Returns a mutable reference the underlying node.
pub fn node_mut(&mut self) -> &mut Node {
&mut self.0
}
}
/// A [`GraphStore`] wrapper that skips nodes that have already been
/// visited.
///
/// This is necessary to avoid repeated work when traversing non-tree
/// graphs (i.e. where a node can have more than one incoming edge).
#[derive(Debug)]
pub struct SkipDuplicates<StoreImpl>
where
StoreImpl: GraphStore,
{
store: StoreImpl,
visited: FxHashSet<StoreImpl::Node>,
}
impl<StoreImpl> SkipDuplicates<StoreImpl>
where
StoreImpl: GraphStore,
{
pub fn new(store: StoreImpl) -> Self {
Self {
store,
visited: FxHashSet::default(),
}
}
pub fn new_with_visited_nodes(store: StoreImpl, visited: FxHashSet<StoreImpl::Node>) -> Self {
Self { store, visited }
}
}
impl<StoreImpl> GraphStore for SkipDuplicates<StoreImpl>
where
StoreImpl: GraphStore,
StoreImpl::Node: Eq + std::hash::Hash + Clone,
{
type Node = StoreImpl::Node;
type Handle = StoreImpl::Handle;
fn insert(
&mut self,
from_handle: Option<Self::Handle>,
node: GraphNode<StoreImpl::Node>,
) -> Option<(Self::Handle, &StoreImpl::Node)> {
if !self.visited.contains(node.node()) {
self.visited.insert(node.node().clone());
self.store.insert(from_handle, node)
} else {
// Always insert the node into the store, even if we've already
// visited it. This is necessary to ensure that the store sees all
// edges.
self.store.insert(from_handle, node);
None
}
}
}
impl<StoreImpl> SkipDuplicates<StoreImpl>
where
StoreImpl: GraphStore,
{
/// Consumes the wrapper and returns the underlying store.
pub fn into_inner(self) -> StoreImpl {
self.store
}
/// Consumes the wrapper and returns the underlying store along with the visited nodes.
pub fn into_inner_with_visited(self) -> (StoreImpl, VisitedNodes<StoreImpl::Node>) {
(self.store, VisitedNodes(self.visited))
}
}
/// A [`GraphStore`] wrapper that skips nodes that have already been
/// visited, based on a key extracted from the node.
///
/// This is necessary to avoid repeated work when traversing non-tree
/// graphs (i.e. where a node can have more than one incoming edge).
#[derive(Debug)]
pub struct SkipDuplicatesWithKey<StoreImpl, Key, KeyExtractor>
where
StoreImpl: GraphStore,
Key: Send + Eq + Hash,
KeyExtractor: Send + Fn(&StoreImpl::Node) -> &Key,
{
store: StoreImpl,
visited: FxHashSet<Key>,
key_extractor: KeyExtractor,
}
impl<StoreImpl, Key, KeyExtractor> SkipDuplicatesWithKey<StoreImpl, Key, KeyExtractor>
where
StoreImpl: GraphStore,
Key: Send + Eq + std::hash::Hash + Clone,
KeyExtractor: Send + Fn(&StoreImpl::Node) -> &Key,
{
pub fn new(store: StoreImpl, key_extractor: KeyExtractor) -> Self {
Self {
store,
visited: FxHashSet::default(),
key_extractor,
}
}
}
impl<StoreImpl, Key, KeyExtractor> GraphStore
for SkipDuplicatesWithKey<StoreImpl, Key, KeyExtractor>
where
StoreImpl: GraphStore,
StoreImpl::Node: Eq + std::hash::Hash + Clone,
Key: Send + Eq + std::hash::Hash + Clone,
KeyExtractor: Send + Fn(&StoreImpl::Node) -> &Key,
{
type Node = StoreImpl::Node;
type Handle = StoreImpl::Handle;
fn insert(
&mut self,
from_handle: Option<Self::Handle>,
node: GraphNode<StoreImpl::Node>,
) -> Option<(Self::Handle, &StoreImpl::Node)> {
let key = (self.key_extractor)(node.node());
if !self.visited.contains(key) {
self.visited.insert(key.clone());
self.store.insert(from_handle, node)
} else {
// Always insert the node into the store, even if we've already
// visited it. This is necessary to ensure that the store sees all
// edges.
self.store.insert(from_handle, node);
None
}
}
}
impl<StoreImpl, Key, KeyExtractor> SkipDuplicatesWithKey<StoreImpl, Key, KeyExtractor>
where
StoreImpl: GraphStore,
Key: Send + Eq + std::hash::Hash + Clone,
KeyExtractor: Send + Fn(&StoreImpl::Node) -> &Key,
{
/// Consumes the wrapper and returns the underlying store.
pub fn into_inner(self) -> StoreImpl {
self.store
}
fn insert(&mut self, from: Option<(&Self::Handle, Self::Edge)>, node: Self::Node);
/// Consumes the wrapper and returns the underlying store along with the visited nodes.
pub fn into_inner_with_visited(self) -> (StoreImpl, VisitedNodes<Key>) {
(self.store, VisitedNodes(self.visited))
}
/// Tries to enter a node during traversal for visiting its edges.
/// Returns `true` if the node edges should be visited.
/// Returns `false` if the node has already been visited and should not be explored again.
fn try_enter(&mut self, node: &Self::Node) -> Option<Self::Handle>;
}
......@@ -4,11 +4,7 @@ use anyhow::Result;
use futures::{StreamExt, stream::FuturesUnordered};
use rustc_hash::FxHashSet;
use super::{
SkipDuplicates, Visit, VisitControlFlow,
graph_store::{GraphNode, GraphStore, SkipDuplicatesWithKey},
with_future::With,
};
use super::{Visit, VisitControlFlow, graph_store::GraphStore, with_future::With};
/// A list of modules that were already visited and should be skipped (including their subgraphs).
#[derive(Clone, Default, Debug)]
......@@ -20,29 +16,14 @@ pub struct VisitedNodes<T>(pub FxHashSet<T>);
/// The traversal is done in parallel, and the order of the nodes in the traversal
/// result is determined by the [`GraphStore`] parameter.
pub trait GraphTraversal: GraphStore + Sized {
fn visit<VisitImpl, Abort, Impl>(
fn visit<VisitImpl, Impl>(
self,
root_edges: impl IntoIterator<Item = VisitImpl::Edge>,
root_nodes: impl IntoIterator<Item = Self::Node>,
visit: VisitImpl,
) -> impl Future<Output = GraphTraversalResult<Result<Self>, Abort>> + Send
) -> impl Future<Output = GraphTraversalResult<Result<Self>>> + Send
where
VisitImpl: Visit<Self::Node, Abort, Impl> + Send,
Abort: Send,
VisitImpl: Visit<Self::Node, Self::Edge, Impl> + Send,
Impl: Send;
fn skip_duplicates(self) -> SkipDuplicates<Self>;
fn skip_duplicates_with_visited_nodes(
self,
visited: VisitedNodes<Self::Node>,
) -> SkipDuplicates<Self>;
fn skip_duplicates_with_key<
Key: Send + Eq + std::hash::Hash + Clone,
KeyExtractor: Send + Fn(&Self::Node) -> &Key,
>(
self,
key_extractor: KeyExtractor,
) -> SkipDuplicatesWithKey<Self, Key, KeyExtractor>;
}
impl<Store> GraphTraversal for Store
......@@ -51,68 +32,70 @@ where
{
/// Visits the graph starting from the given `roots`, and returns a future
/// that will resolve to the traversal result.
fn visit<VisitImpl, Abort, Impl>(
fn visit<VisitImpl, Impl>(
mut self,
root_edges: impl IntoIterator<Item = VisitImpl::Edge>,
root_nodes: impl IntoIterator<Item = Self::Node>,
mut visit: VisitImpl,
) -> impl Future<Output = GraphTraversalResult<Result<Self>, Abort>> + Send
) -> impl Future<Output = GraphTraversalResult<Result<Self>>> + Send
where
VisitImpl: Visit<Self::Node, Abort, Impl> + Send,
Abort: Send,
VisitImpl: Visit<Self::Node, Self::Edge, Impl> + Send,
Impl: Send,
{
let mut futures = FuturesUnordered::new();
let mut root_abort = None;
let mut is_abort = false;
// Populate `futures` with all the roots, `root_edges` isn't required to be `Send`, so this
// has to happen outside of the future. We could require `root_edges` to be `Send` in the
// Populate `futures` with all the roots, `root_nodes` isn't required to be `Send`, so this
// has to happen outside of the future. We could require `root_nodes` to be `Send` in the
// future.
for edge in root_edges {
match visit.visit(edge) {
VisitControlFlow::Continue(node) => {
if let Some((parent_handle, node_ref)) = self.insert(None, GraphNode(node)) {
let span = visit.span(node_ref);
futures.push(With::new(visit.edges(node_ref), span, parent_handle));
for node in root_nodes {
match visit.visit(&node, None) {
VisitControlFlow::Continue => {
if let Some(handle) = self.try_enter(&node) {
let span = visit.span(&node, None);
futures.push(With::new(visit.edges(&node), span, handle));
}
self.insert(None, node);
}
VisitControlFlow::Skip(node) => {
self.insert(None, GraphNode(node));
VisitControlFlow::Skip => {
self.insert(None, node);
}
VisitControlFlow::Abort(abort) => {
VisitControlFlow::Exclude => {
// do nothing
}
VisitControlFlow::Abort => {
// this must be returned inside the `async` block below so that it's part of the
// returned future
root_abort = Some(abort)
is_abort = true;
}
}
}
async move {
if let Some(abort) = root_abort {
return GraphTraversalResult::Aborted(abort);
if is_abort {
return GraphTraversalResult::Aborted;
}
loop {
match futures.next().await {
Some((parent_handle, span, Ok(edges))) => {
Some((parent_node, span, Ok(edges))) => {
let _guard = span.enter();
for edge in edges {
match visit.visit(edge) {
VisitControlFlow::Continue(node) => {
if let Some((node_handle, node_ref)) =
self.insert(Some(parent_handle.clone()), GraphNode(node))
{
let span = visit.span(node_ref);
futures.push(With::new(
visit.edges(node_ref),
span,
node_handle,
));
for (node, edge) in edges {
match visit.visit(&node, Some(&edge)) {
VisitControlFlow::Continue => {
if let Some(handle) = self.try_enter(&node) {
let span = visit.span(&node, Some(&edge));
let edges_future = visit.edges(&node);
futures.push(With::new(edges_future, span, handle));
}
self.insert(Some((&parent_node, edge)), node);
}
VisitControlFlow::Skip => {
self.insert(Some((&parent_node, edge)), node);
}
VisitControlFlow::Skip(node) => {
self.insert(Some(parent_handle.clone()), GraphNode(node));
VisitControlFlow::Exclude => {
// do nothing
}
VisitControlFlow::Abort(abort) => {
return GraphTraversalResult::Aborted(abort);
VisitControlFlow::Abort => {
return GraphTraversalResult::Aborted;
}
}
}
......@@ -127,39 +110,18 @@ where
}
}
}
fn skip_duplicates(self) -> SkipDuplicates<Self> {
SkipDuplicates::new(self)
}
fn skip_duplicates_with_visited_nodes(
self,
visited: VisitedNodes<Store::Node>,
) -> SkipDuplicates<Self> {
SkipDuplicates::new_with_visited_nodes(self, visited.0)
}
fn skip_duplicates_with_key<
Key: Send + Eq + std::hash::Hash + Clone,
KeyExtractor: Send + Fn(&Self::Node) -> &Key,
>(
self,
key_extractor: KeyExtractor,
) -> SkipDuplicatesWithKey<Self, Key, KeyExtractor> {
SkipDuplicatesWithKey::new(self, key_extractor)
}
}
pub enum GraphTraversalResult<Completed, Aborted> {
pub enum GraphTraversalResult<Completed> {
Completed(Completed),
Aborted(Aborted),
Aborted,
}
impl<Completed> GraphTraversalResult<Completed, !> {
impl<Completed> GraphTraversalResult<Completed> {
pub fn completed(self) -> Completed {
match self {
GraphTraversalResult::Completed(completed) => completed,
GraphTraversalResult::Aborted(_) => unreachable!("the type parameter `Aborted` is `!`"),
GraphTraversalResult::Aborted => panic!("Graph traversal was aborted"),
}
}
}
......@@ -8,7 +8,7 @@ mod with_future;
pub use adjacency_map::AdjacencyMap;
pub use control_flow::VisitControlFlow;
pub use graph_store::{GraphStore, SkipDuplicates};
pub use graph_store::GraphStore;
pub use graph_traversal::{GraphTraversal, GraphTraversalResult, VisitedNodes};
pub use non_deterministic::NonDeterministic;
pub use visit::Visit;
use super::graph_store::{GraphNode, GraphStore};
use std::hash::Hash;
use rustc_hash::FxHashSet;
use super::graph_store::GraphStore;
/// A graph traversal that does not guarantee any particular order, and may not
/// return the same order every time it is run.
pub struct NonDeterministic<T> {
pub struct NonDeterministic<T, E> {
output: Vec<T>,
visited: FxHashSet<T>,
phantom: std::marker::PhantomData<E>,
}
impl<T> Default for NonDeterministic<T> {
impl<T, E> Default for NonDeterministic<T, E> {
fn default() -> Self {
Self::new()
}
}
impl<T> NonDeterministic<T> {
impl<T, E> NonDeterministic<T, E> {
pub fn new() -> Self {
Self { output: Vec::new() }
Self {
output: Vec::new(),
visited: FxHashSet::default(),
phantom: std::marker::PhantomData,
}
}
}
impl<T> GraphStore for NonDeterministic<T>
impl<T, E> GraphStore for NonDeterministic<T, E>
where
T: Send,
T: Send + Hash + Eq + Clone,
E: Send,
{
type Node = T;
type Edge = E;
type Handle = ();
fn insert(
&mut self,
_from_handle: Option<Self::Handle>,
node: GraphNode<T>,
) -> Option<(Self::Handle, &T)> {
self.output.push(node.into_node());
Some(((), self.output.last().unwrap()))
fn insert(&mut self, _from: Option<(&(), E)>, node: T) {
self.output.push(node);
}
fn try_enter(&mut self, node: &T) -> Option<()> {
if self.visited.insert(node.clone()) {
Some(())
} else {
None
}
}
}
impl<T> IntoIterator for NonDeterministic<T> {
impl<T, E> IntoIterator for NonDeterministic<T, E> {
type Item = T;
type IntoIter = <Vec<T> as IntoIterator>::IntoIter;
......
use std::future::Future;
use std::{future::Future, iter::Map};
use anyhow::Result;
use tracing::Span;
......@@ -7,17 +7,20 @@ use super::VisitControlFlow;
/// A trait that allows a graph traversal to visit the edges of a node
/// transitively.
pub trait Visit<Node, Abort = !, Impl = ()> {
type Edge;
type EdgesIntoIter: IntoIterator<Item = Self::Edge>;
pub trait Visit<Node, Edge = (), Impl = ()> {
type EdgesIntoIter: IntoIterator<Item = (Node, Edge)>;
type EdgesFuture: Future<Output = Result<Self::EdgesIntoIter>> + Send;
/// Visits an edge to get to the neighbor node. Should return a
/// Visits an edge. Should return a
/// [`VisitControlFlow`] that indicates whether to:
/// * continue visiting the neighbor node edges;
/// * skip visiting the neighbor node's edges;
/// * abort the traversal entirely.
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<Node, Abort>;
fn visit(&mut self, node: &Node, edge: Option<&Edge>) -> VisitControlFlow {
let _ = node;
let _ = edge;
VisitControlFlow::Continue
}
/// Returns a future that resolves to the outgoing edges of the given `node`.
///
......@@ -28,11 +31,9 @@ pub trait Visit<Node, Abort = !, Impl = ()> {
/// returns a node reference that's only valid for the lifetime of its `&mut self` reference.
fn edges(&mut self, node: &Node) -> Self::EdgesFuture;
/// Returns a [Span] for the given `node`, under which all edges are
/// processed.
fn span(&mut self, node: &Node) -> Span {
let _ = node;
Span::current()
/// Returns a [Span] for the given `node`, under which all edges are processed.
fn span(&mut self, _node: &Node, _edge: Option<&Edge>) -> Span {
Span::none()
}
}
......@@ -41,45 +42,94 @@ pub trait Visit<Node, Abort = !, Impl = ()> {
// kinds of `FnMut`.
// See https://users.rust-lang.org/t/conflicting-implementation-when-implementing-traits-for-fn/53359/3
pub struct ImplRef;
pub struct ImplWithEdgeRef;
impl<Node, VisitFn, NeighFut, NeighIt> Visit<Node, !, ImplRef> for VisitFn
impl<Node, Edge, VisitFn, NeighFut, NeighIt> Visit<Node, Edge, ImplWithEdgeRef> for VisitFn
where
VisitFn: FnMut(&Node) -> NeighFut,
NeighFut: Future<Output = Result<NeighIt>> + Send,
NeighIt: IntoIterator<Item = Node>,
NeighIt: IntoIterator<Item = (Node, Edge)>,
{
type Edge = Node;
type EdgesIntoIter = NeighIt;
type EdgesFuture = NeighFut;
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<Node> {
VisitControlFlow::Continue(edge)
}
fn edges(&mut self, node: &Node) -> Self::EdgesFuture {
(self)(node)
}
}
pub struct ImplValue;
pub struct ImplWithEdgeValue;
impl<Node, VisitFn, NeighFut, NeighIt> Visit<Node, !, ImplValue> for VisitFn
impl<Node, Edge, VisitFn, NeighFut, NeighIt> Visit<Node, Edge, ImplWithEdgeValue> for VisitFn
where
Node: Clone,
VisitFn: FnMut(Node) -> NeighFut,
NeighFut: Future<Output = Result<NeighIt>> + Send,
NeighIt: IntoIterator<Item = Node>,
NeighIt: IntoIterator<Item = (Node, Edge)>,
{
type Edge = Node;
type EdgesIntoIter = NeighIt;
type EdgesFuture = NeighFut;
fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow<Node> {
VisitControlFlow::Continue(edge)
fn edges(&mut self, node: &Node) -> Self::EdgesFuture {
(self)(node.clone())
}
}
pub struct ImplRef;
impl<Node, VisitFn, NeighFut, NeighIt> Visit<Node, (), ImplRef> for VisitFn
where
VisitFn: FnMut(&Node) -> NeighFut,
NeighFut: Future<Output = Result<NeighIt>> + Send,
NeighIt: IntoIterator<Item = Node>,
{
type EdgesIntoIter = Map<NeighIt::IntoIter, fn(Node) -> (Node, ())>;
type EdgesFuture = NoEdgeFuture<NeighFut>;
fn edges(&mut self, node: &Node) -> Self::EdgesFuture {
(self)(node.clone())
NoEdgeFuture((self)(node))
}
}
pub struct ImplValue;
impl<Node, VisitFn, NeighFut, NeighIt> Visit<Node, (), ImplValue> for VisitFn
where
Node: Clone,
VisitFn: FnMut(Node) -> NeighFut,
NeighFut: Future<Output = Result<NeighIt>> + Send,
NeighIt: IntoIterator<Item = Node>,
{
type EdgesIntoIter = Map<NeighIt::IntoIter, fn(Node) -> (Node, ())>;
type EdgesFuture = NoEdgeFuture<NeighFut>;
fn edges(&mut self, node: &Node) -> Self::EdgesFuture {
NoEdgeFuture((self)(node.clone()))
}
}
pub struct NoEdgeFuture<F>(F);
impl<F, Node, NeighIt> Future for NoEdgeFuture<F>
where
F: Future<Output = Result<NeighIt>> + Send,
NeighIt: IntoIterator<Item = Node>,
{
type Output = Result<Map<NeighIt::IntoIter, fn(Node) -> (Node, ())>>;
fn poll(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Self::Output> {
let future = unsafe { self.map_unchecked_mut(|s| &mut s.0) };
future.poll(cx).map(|res| {
res.map(|it| {
fn map_fn<Node>(node: Node) -> (Node, ()) {
(node, ())
}
let f: fn(Node) -> (Node, ()) = map_fn;
it.into_iter().map(f)
})
})
}
}
......@@ -11,7 +11,7 @@ pin_project! {
#[pin]
future: T,
span: Span,
handle: Option<H>,
data: Option<H>,
}
}
......@@ -19,11 +19,11 @@ impl<T, H> With<T, H>
where
T: Future,
{
pub fn new(future: T, span: Span, handle: H) -> Self {
pub fn new(future: T, span: Span, data: H) -> Self {
Self {
future,
span,
handle: Some(handle),
data: Some(data),
}
}
}
......@@ -44,7 +44,7 @@ where
std::task::Poll::Ready(result) => {
drop(guard);
std::task::Poll::Ready((
this.handle.take().expect("polled after completion"),
this.data.take().expect("polled after completion"),
replace(this.span, Span::none()),
result,
))
......
......@@ -27,11 +27,9 @@ pub async fn any_content_changed_of_module(
root: ResolvedVc<Box<dyn Module>>,
) -> Result<Vc<Completion>> {
let completions = AdjacencyMap::new()
.skip_duplicates()
.visit([root], get_referenced_modules)
.await
.completed()?
.into_inner()
.into_postorder_topological()
.map(|m| content_changed(*ResolvedVc::upcast(m)))
.map(|v| v.to_resolved())
......
......@@ -266,13 +266,11 @@ pub async fn expand_output_assets(
inner_output_assets: bool,
) -> Result<Vec<ResolvedVc<Box<dyn OutputAsset>>>> {
let edges = AdjacencyMap::new()
.skip_duplicates()
.visit(inputs, async |input| {
get_referenced_assets(inner_output_assets, input).await
})
.await
.completed()?
.into_inner()
.into_postorder_topological();
let mut assets = Vec::new();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment