Unverified Commit 74bb0d9a authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: fixup cache handler tracing (#94477)

https://github.com/vercel/next.js/pull/94197 had a bug in the Turbopack
case, where it didn't properly trace all necessary files referenced from
`cacheHandler` and `cacheHandlers`.

If you imported `lodash` in the cache handler, then
- `node_modules/.pnpm/lodash@4.18.1/node_modules/lodash/lodash.js` was
correctly traced
but missing were
- `node_modules/.pnpm/lodash@4.18.1/node_modules/lodash/package.json`
and
- `node_modules/lodash` (the symlink)

The problem was that it wasn't including affecting sources of these
entrypoints, because it was treating those entrypoints as bundled:

```
SingleModuleGraph::new_inner(
        entries: Vec<ChunkGroupEntry>,
        visited_modules: &FxIndexMap<ResolvedVc<Box<dyn Module>>, GraphNodeIndex>,
        include_traced: bool,
        include_binding_usage: bool,
        entries_are_traced: bool, // <------ problematic
    ) -> Result<Vc<Self>>
```

should really have been 

```
SingleModuleGraph::new_inner(
        entries: &GraphEntries, // contains Vec<ChunkGroupEntry> and traced_modules: Vec<Modules
        visited_modules: &FxIndexMap<ResolvedVc<Box<dyn Module>>, GraphNodeIndex>,
        include_traced: bool,
        include_binding_usage: bool,
    ) -> Result<Vc<Self>> {
```
because the single `entries_are_traced: bool` can be different per
entry.
parent ef63481d
......@@ -508,7 +508,7 @@ pub async fn analyze_module_graphs(module_graph: Vc<ModuleGraph>) -> Result<Vc<F
let module_graph = module_graph.await?;
module_graph.traverse_edges_dfs(
module_graph.graphs.iter().flat_map(|g| g.entry_modules()),
module_graph.all_entry_modules(),
&mut (),
|parent, node, _| {
all_modules.insert(node);
......
......@@ -910,7 +910,7 @@ impl AppProject {
// SEGMENT: client_shared_entries and server utils shared by the layout segments
// and the page
let graph = SingleModuleGraph::new_with_entries_visited_intern(
vec![
GraphEntries::from_chunk_groups(vec![
ChunkGroupEntry::Entry(client_shared_entries),
ChunkGroupEntry::SharedMultiple(
server_utils
......@@ -919,7 +919,7 @@ impl AppProject {
.try_join()
.await?,
),
],
]),
visited_modules,
should_trace,
should_read_binding_usage,
......@@ -936,7 +936,9 @@ impl AppProject {
{
// SEGMENT: layout segment
let graph = SingleModuleGraph::new_with_entries_visited_intern(
vec![ChunkGroupEntry::Shared(ResolvedVc::upcast(*module))],
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Shared(
ResolvedVc::upcast(*module),
)]),
visited_modules,
should_trace,
should_read_binding_usage,
......@@ -959,7 +961,7 @@ impl AppProject {
// SEGMENT: rsc entry chunk group
let graph = SingleModuleGraph::new_with_entries_visited_intern(
vec![rsc_entry_chunk_group],
GraphEntries::from_chunk_groups(vec![rsc_entry_chunk_group]),
visited_modules,
should_trace,
should_read_binding_usage,
......@@ -2151,7 +2153,7 @@ impl Endpoint for AppEndpoint {
#[turbo_tasks::function]
async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
let this = self.await?;
Ok(Vc::cell(vec![
Ok(GraphEntries::from_chunk_groups(vec![
ChunkGroupEntry::Entry(vec![self.app_endpoint_entry().await?.rsc_entry]),
ChunkGroupEntry::Entry(
this.app_project
......@@ -2162,7 +2164,8 @@ impl Endpoint for AppEndpoint {
.map(ResolvedVc::upcast)
.collect(),
),
]))
])
.cell())
}
#[turbo_tasks::function]
......@@ -2201,9 +2204,10 @@ impl Endpoint for AppEndpoint {
.await?,
);
Ok(Vc::cell(vec![ChunkGroupEntry::Shared(
server_actions_loader,
)]))
Ok(
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Shared(server_actions_loader)])
.cell(),
)
}
#[turbo_tasks::function]
......
......@@ -256,7 +256,10 @@ impl Endpoint for InstrumentationEndpoint {
#[turbo_tasks::function]
async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
let entry_module = self.entry_module().to_resolved().await?;
Ok(Vc::cell(vec![ChunkGroupEntry::Entry(vec![entry_module])]))
Ok(
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(vec![entry_module])])
.cell(),
)
}
#[turbo_tasks::function]
......
......@@ -406,9 +406,12 @@ impl Endpoint for MiddlewareEndpoint {
#[turbo_tasks::function]
async fn entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
Ok(Vc::cell(vec![ChunkGroupEntry::Entry(vec![
self.entry_module().to_resolved().await?,
])]))
Ok(
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(vec![
self.entry_module().to_resolved().await?,
])])
.cell(),
)
}
#[turbo_tasks::function]
......
......@@ -170,7 +170,7 @@ impl NextDynamicGraph {
}
Either::Left(std::iter::once(entry))
} else {
Either::Right(graph.graphs.first().unwrap().entry_modules())
Either::Right(graph.graphs.first().unwrap().chunk_group_modules())
};
let mut result = vec![];
......@@ -557,7 +557,7 @@ impl ClientReferencesGraph {
}
Either::Left(std::iter::once(entry))
} else {
Either::Right(graph.graphs.first().unwrap().entry_modules())
Either::Right(graph.graphs.first().unwrap().chunk_group_modules())
};
// Because we care about 'evaluation order' we need to collect client references in the
......@@ -784,7 +784,7 @@ async fn validate_pages_css_imports_individual(
}
Either::Left(std::iter::once(entry))
} else {
Either::Right(graph.graphs.first().unwrap().entry_modules())
Either::Right(graph.graphs.first().unwrap().chunk_group_modules())
};
let mut candidates = vec![];
......
......@@ -14,7 +14,7 @@ use turbopack::externals_tracing_module_context;
use turbopack_core::{
asset::{Asset, AssetContent},
module::{Module, Modules},
module_graph::{ModuleGraph, SingleModuleGraph, chunk_group_info::ChunkGroupEntry},
module_graph::{GraphEntries, ModuleGraph, SingleModuleGraph},
output::{OutputAsset, OutputAssets, OutputAssetsReference},
reference_type::CommonJsReferenceSubType,
resolve::{ResolveErrorMode, origin::PlainResolveOrigin, parse::Request},
......@@ -106,8 +106,8 @@ impl Asset for ServerNftJsonAsset {
.join(&this.project.node_root().await?.path)?;
let module_graph = ModuleGraph::from_graphs(
vec![SingleModuleGraph::new_with_traced_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(self.entries().owned().await?)]),
vec![SingleModuleGraph::new_with_entries(
GraphEntries::new(vec![], self.entries().owned().await?).resolved_cell(),
true,
false,
)],
......
......@@ -381,7 +381,7 @@ pub async fn traced_module_data_for_graph(
// This function is very similar to traced_modules_for_entries, but doesn't apply the glob and
// is executed only once for the whole graph.
let module_graph = module_graph.await?;
let entries = module_graph.graphs.iter().flat_map(|g| g.entry_modules());
let entries = module_graph.all_entry_modules();
let traced_entries = traced_entries.await?.into_iter().collect::<FxHashSet<_>>();
......
......@@ -735,7 +735,7 @@ impl PageEndpoint {
.flatten()
{
let graph = SingleModuleGraph::new_with_entries_visited_intern(
vec![ChunkGroupEntry::Shared(module)],
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Shared(module)]),
visited_modules,
should_trace,
should_read_binding_usage,
......@@ -745,7 +745,9 @@ impl PageEndpoint {
}
let graph = SingleModuleGraph::new_with_entries_visited_intern(
vec![ChunkGroupEntry::Entry(vec![ssr_chunk_module.ssr_module])],
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(vec![
ssr_chunk_module.ssr_module,
])]),
visited_modules,
should_trace,
should_read_binding_usage,
......@@ -1732,7 +1734,7 @@ impl Endpoint for PageEndpoint {
})
.collect::<Vec<_>>();
Ok(Vc::cell(modules))
Ok(GraphEntries::from_chunk_groups(modules).cell())
}
#[turbo_tasks::function]
......
......@@ -1421,22 +1421,25 @@ impl Project {
#[turbo_tasks::function]
pub async fn get_all_entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
let mut modules = self
let endpoint_entries = self
.get_all_endpoints(false)
.await?
.iter()
.map(async |endpoint| Ok(endpoint.entries().owned().await?))
.try_flat_join()
.map(|endpoint| endpoint.entries().owned())
.try_join()
.await?;
modules.extend(self.client_main_modules().await?.iter().cloned());
modules.extend(
self.additional_traced_modules()
.await?
.iter()
.cloned()
.map(|m| ChunkGroupEntry::Entry(vec![m])),
let result = GraphEntries::concatenate(
endpoint_entries
.into_iter()
.chain(std::iter::once(self.client_main_modules().owned().await?))
.chain(std::iter::once(GraphEntries::new(
vec![],
self.additional_traced_modules().owned().await?,
))),
);
Ok(Vc::cell(modules))
Ok(result.cell())
}
#[turbo_tasks::function]
......@@ -1444,14 +1447,15 @@ impl Project {
self: Vc<Self>,
graphs: Vc<ModuleGraph>,
) -> Result<Vc<GraphEntries>> {
let modules = self
.get_all_endpoints(false)
.await?
.iter()
.map(async |endpoint| Ok(endpoint.additional_entries(graphs).owned().await?))
.try_flat_join()
.await?;
Ok(Vc::cell(modules))
let result = GraphEntries::concatenate(
self.get_all_endpoints(false)
.await?
.iter()
.map(|endpoint| endpoint.additional_entries(graphs).owned())
.try_join()
.await?,
);
Ok(result.cell())
}
#[turbo_tasks::function]
......@@ -1490,7 +1494,8 @@ impl Project {
.collect();
ModuleGraph::from_graphs(
vec![SingleModuleGraph::new_with_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(entries)]),
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(entries)])
.resolved_cell(),
is_production,
is_production,
)],
......@@ -2506,17 +2511,17 @@ impl Project {
#[turbo_tasks::function]
pub async fn client_main_modules(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
let pages_project = self.pages_project();
let mut modules = vec![ChunkGroupEntry::Entry(vec![
let mut chunk_groups = vec![ChunkGroupEntry::Entry(vec![
pages_project.client_main_module().to_resolved().await?,
])];
if let Some(app_project) = *self.app_project().await? {
modules.push(ChunkGroupEntry::Entry(vec![
chunk_groups.push(ChunkGroupEntry::Entry(vec![
app_project.client_main_module().to_resolved().await?,
]));
}
Ok(Vc::cell(modules))
Ok(GraphEntries::from_chunk_groups(chunk_groups).cell())
}
/// Gets the module id strategy for the project.
......
......@@ -88,9 +88,8 @@ pub async fn endpoint_entry_modules(
let entries = endpoint.entries().await?;
let additional_entries = endpoint.additional_entries(base_module_graph).await?;
let modules = entries
.iter()
.chain(additional_entries.iter())
.flat_map(|e| e.entries())
.chunk_group_modules()
.chain(additional_entries.chunk_group_modules())
.collect::<FxIndexSet<_>>();
Ok(Vc::cell(modules.into_iter().collect()))
}
......@@ -114,9 +113,8 @@ pub async fn endpoints_entry_modules(
.iter()
.flat_map(|(entries, additional_entries)| {
entries
.iter()
.chain(additional_entries.iter())
.flat_map(|e| e.entries())
.chunk_group_modules()
.chain(additional_entries.chunk_group_modules())
})
.collect::<FxIndexSet<_>>();
Ok(Vc::cell(modules.into_iter().collect()))
......
......@@ -7,7 +7,7 @@ import { join } from 'path'
const { next } = nextTestSetup({
files: __dirname,
dependencies: {
lodash: 'latest',
lodash: '4.18.1',
'webpack-bundle-analyzer': 'latest',
},
})
......
const cache = new Map()
const isEven = require('is-even')
module.exports = class CacheHandler {
constructor() {
console.log('initialized custom cache-handler')
console.log('initialized custom cache-handler: ', isEven(3))
}
async get(key, ctx) {
......
......@@ -30,6 +30,9 @@ describe('required server files app router', () => {
cacheMaxMemorySize: 0,
output: 'standalone',
},
dependencies: {
'is-even': '1.0.0',
},
skipStart: true,
})
......
......@@ -32,6 +32,9 @@ describe('required server files i18n', () => {
build: 'next build',
},
},
dependencies: {
'is-even': '1.0.0',
},
installCommand: 'pnpm i',
buildCommand: 'pnpm build',
nextConfig: {
......
......@@ -57,6 +57,9 @@ describe.skip('required server files app router', () => {
cacheComponents: true,
output: 'standalone',
},
dependencies: {
'is-even': '1.0.0',
},
})
beforeAll(async () => {
......
......@@ -72,6 +72,9 @@ describe('required server files', () => {
}
},
},
dependencies: {
'is-even': '1.0.0',
},
skipStart: true,
})
......
......@@ -5,7 +5,7 @@ describe('app fetch build cache', () => {
files: __dirname,
dependencies: {
'@aws-sdk/client-s3': 'latest',
lodash: 'latest',
lodash: '4.18.1',
},
})
......
......@@ -33,7 +33,7 @@ use turbopack_core::{
issue::{IssueReporter, IssueSeverity, handle_issues},
module::Module,
module_graph::{
ModuleGraph, SingleModuleGraph,
GraphEntries, ModuleGraph, SingleModuleGraph,
binding_usage_info::compute_binding_usage_info,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
......@@ -312,7 +312,8 @@ async fn build_internal(
.await?;
let single_graph = SingleModuleGraph::new_with_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(entries.clone())]),
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(entries.clone())])
.resolved_cell(),
false,
true,
);
......
......@@ -12,7 +12,9 @@ use turbopack_core::{
environment::Environment,
file_source::FileSource,
module::Module,
module_graph::{ModuleGraph, SingleModuleGraph, chunk_group_info::ChunkGroupEntry},
module_graph::{
GraphEntries, ModuleGraph, SingleModuleGraph, chunk_group_info::ChunkGroupEntry,
},
reference_type::{EntryReferenceSubType, ReferenceType},
resolve::{
origin::{PlainResolveOrigin, ResolveOrigin, ResolveOriginExt},
......@@ -160,7 +162,8 @@ pub async fn create_web_entry_source(
.collect::<Vec<ResolvedVc<Box<dyn Module>>>>();
let module_graph = ModuleGraph::from_graphs(
vec![SingleModuleGraph::new_with_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(all_modules)]),
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(all_modules)])
.resolved_cell(),
false,
false,
)],
......
......@@ -144,7 +144,7 @@ pub async fn compute_binding_usage_info(
None
};
let entries = graph_ref.graphs.iter().flat_map(|g| g.entry_modules());
let entries = graph_ref.all_chunk_group_entry_modules();
let visit_count = graph_ref.traverse_edges_fixed_point_with_priority(
entries.map(|m| (m, 0)),
......
......@@ -428,11 +428,7 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
span.record("module_count", module_count);
// use all entries from all graphs
let entries = graph
.graphs
.iter()
.flat_map(|g| g.entries.iter())
.collect::<Vec<_>>();
let entries = graph.all_chunk_group_entries().collect::<Vec<_>>();
// First, compute the depth for each module in the graph
let module_depth: FxHashMap<ResolvedVc<Box<dyn Module>>, usize> = {
......@@ -511,11 +507,9 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
}
}
let entry_chunk_group_keys = graph
.graphs
let entry_chunk_group_keys = entries
.iter()
.flat_map(|g| g.entries.iter())
.flat_map(|chunk_group| {
.flat_map(|&chunk_group| {
let chunk_group_key =
entry_to_chunk_group_id(chunk_group.clone(), &mut chunk_groups_map);
chunk_group
......
......@@ -96,10 +96,8 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc<
span.record("module_count", module_count);
// Use all entries from all graphs
let entries = graphs
.iter()
.flat_map(|g| g.entries.iter())
.flat_map(|g| g.entries())
let entries = module_graph
.all_chunk_group_entry_modules()
.collect::<Vec<_>>();
// First, compute the depth for each module in the graph
......
......@@ -421,14 +421,15 @@ pub struct EvaluateEntries {
impl EvaluateEntries {
#[turbo_tasks::function]
pub async fn graph_entries(self: Vc<Self>) -> Result<Vc<GraphEntries>> {
Ok(Vc::cell(vec![ChunkGroupEntry::Entry(
Ok(GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(
self.await?
.entries
.iter()
.cloned()
.map(ResolvedVc::upcast)
.collect(),
)]))
)])
.cell())
}
}
......
......@@ -45,7 +45,7 @@ use turbopack_core::{
issue::{CollectibleIssuesExt, IssueFilter, IssueSeverity},
module::Module,
module_graph::{
ModuleGraph, SingleModuleGraph,
GraphEntries, ModuleGraph, SingleModuleGraph,
binding_usage_info::compute_binding_usage_info,
chunk_group_info::{ChunkGroup, ChunkGroupEntry},
},
......@@ -456,7 +456,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.collect();
let single_graph = SingleModuleGraph::new_with_entries(
ResolvedVc::cell(vec![ChunkGroupEntry::Entry(entry_modules.clone())]),
GraphEntries::from_chunk_groups(vec![ChunkGroupEntry::Entry(entry_modules.clone())])
.resolved_cell(),
false,
true,
);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment