Unverified Commit 7d91feca authored by Benjamin Woodruff ‮'s avatar Benjamin Woodruff Committed by GitHub

Turbopack: Auto-detect pnpm GVS and configure an additional root for it (#99758)

This implements auto-configuration (behind a flag for now) for pnpm 11+.

pnpm 10 supports GVS, but `node_modules/.modules.yaml` is actually YAML
on 10. 11 changes it to a JSON file, which lets us avoid pulling in a
YAML parser. IMO, this is fine as GVS on pnpm 10 is very experimental
(it seems like it doesn't play nicely with `packageExtensions`).

TODO in future PRs:
- bun, nub, aube support
- figure out how to support `NODE_PATH` so that hoisting works for
broken packages with phantom dependencies (users can work around this
with `packageExtensions`, pnpm documents this problem in the GVS docs)
- Switch on GVS for (most of) the Next.js e2e tests so that we have some
good coverage of this feature.
https://github.com/vercel/next.js/pull/100337
parent d054fe4a
...@@ -9984,11 +9984,14 @@ dependencies = [ ...@@ -9984,11 +9984,14 @@ dependencies = [
"anyhow", "anyhow",
"bincode", "bincode",
"either", "either",
"fs-err",
"indoc", "indoc",
"serde", "serde",
"serde_json", "serde_json",
"serde_qs", "serde_qs",
"smallvec", "smallvec",
"tempfile",
"thiserror 1.0.69",
"tracing", "tracing",
"turbo-bincode", "turbo-bincode",
"turbo-rcstr", "turbo-rcstr",
......
This diff is collapsed.
...@@ -103,9 +103,11 @@ use turbopack_node::execution_context::ExecutionContext; ...@@ -103,9 +103,11 @@ use turbopack_node::execution_context::ExecutionContext;
use turbopack_node::worker_threads_backend; use turbopack_node::worker_threads_backend;
use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher}; use turbopack_nodejs::{NodeJsChunkingContext, fs::NodeModulesPathMatcher};
pub use crate::additional_roots::AdditionalRootConfig; pub use crate::additional_roots::{AdditionalRootConfig, AdditionalRootName, BuiltinRootKind};
use crate::{ use crate::{
additional_roots::{AdditionalDiskFileSystem, create_additional_root_file_systems}, additional_roots::{
AdditionalDiskFileSystem, BuiltinRootsConfig, create_additional_root_file_systems,
},
aggregate_hmr::ServerHmrChunkLists, aggregate_hmr::ServerHmrChunkLists,
app::{AppProject, OptionAppProject}, app::{AppProject, OptionAppProject},
empty::EmptyEndpoint, empty::EmptyEndpoint,
...@@ -477,6 +479,13 @@ async fn prepare_project_container_state( ...@@ -477,6 +479,13 @@ async fn prepare_project_container_state(
.and_then(|value| value.as_str()) .and_then(|value| value.as_str())
.unwrap_or(".next"); .unwrap_or(".next");
let builtin_roots = BuiltinRootsConfig {
detect_global_virtual_store: config_json
.pointer("/experimental/turbopackDetectGlobalVirtualStore")
.and_then(|value| value.as_bool())
.unwrap_or(false),
};
let watcher_config = DiskWatcherConfig { let watcher_config = DiskWatcherConfig {
poll_interval: options.watch.poll_interval, poll_interval: options.watch.poll_interval,
report_invalidation_reason: true, report_invalidation_reason: true,
...@@ -547,7 +556,9 @@ async fn prepare_project_container_state( ...@@ -547,7 +556,9 @@ async fn prepare_project_container_state(
let additional_roots = create_additional_root_file_systems( let additional_roots = create_additional_root_file_systems(
container_vc, container_vc,
configured_additional_roots, configured_additional_roots,
builtin_roots,
&project_root, &project_root,
&project_path,
watcher_config, watcher_config,
map, map,
config_path, config_path,
......
...@@ -29,8 +29,9 @@ use next_api::{ ...@@ -29,8 +29,9 @@ use next_api::{
RouteOperation, RouteOperation,
}, },
project::{ project::{
AdditionalRootConfig, DebugBuildPaths, DefineEnv, DraftModeOptions, PartialProjectOptions, AdditionalRootConfig, AdditionalRootName, DebugBuildPaths, DefineEnv, DraftModeOptions,
Project, ProjectContainer, ProjectOptions, WatchOptions, activate_lazy_chunk_operation, PartialProjectOptions, Project, ProjectContainer, ProjectOptions, WatchOptions,
activate_lazy_chunk_operation,
}, },
project_asset_hashes_manifest::immutable_hashes_manifest_asset_if_enabled, project_asset_hashes_manifest::immutable_hashes_manifest_asset_if_enabled,
route::{Endpoint, EndpointGroupKey, Route}, route::{Endpoint, EndpointGroupKey, Route},
...@@ -327,7 +328,7 @@ impl NapiProjectOptions { ...@@ -327,7 +328,7 @@ impl NapiProjectOptions {
additional_roots: additional_roots additional_roots: additional_roots
.into_iter() .into_iter()
.map(|root| AdditionalRootConfig { .map(|root| AdditionalRootConfig {
key: root.key, name: AdditionalRootName::Configured(root.key),
path: root.path, path: root.path,
ignore_if_missing: root.ignore_if_missing.unwrap_or(false), ignore_if_missing: root.ignore_if_missing.unwrap_or(false),
}) })
......
...@@ -26,9 +26,9 @@ Turbopack `.nft.json` files include `fileHashes` and `symlinks` extensions: ...@@ -26,9 +26,9 @@ Turbopack `.nft.json` files include `fileHashes` and `symlinks` extensions:
- `fileHashes` is a parallel array with the same length and ordering as `files`. For example, `fileHashes[0]` is the hash for `files[0]`. A consumer can use the hash as a content identifier for caching or deduplicating traced assets. - `fileHashes` is a parallel array with the same length and ordering as `files`. For example, `fileHashes[0]` is the hash for `files[0]`. A consumer can use the hash as a content identifier for caching or deduplicating traced assets.
- `symlinks` maps indexes in the sibling `files` array to their immediate targets. When `symlinks` is present, files without a corresponding entry can be assumed to be normal files or directories. - `symlinks` maps indexes in the sibling `files` array to their immediate targets. When `symlinks` is present, files without a corresponding entry can be assumed to be normal files or directories.
Traces that use [`experimental.turbopackAdditionalRoots`](/docs/app/api-reference/config/next-config-js/turbopack#additional-roots-experimental) also include the `additionalRoots` extension: Traces that use [additional roots](/docs/app/api-reference/config/next-config-js/turbopack#additional-roots-experimental) also include the `additionalRoots` extension:
- `additionalRoots` is an ordered list of named source roots. Each root has its own `files`, `fileHashes`, and `symlinks` lists. Its `path` is relative to the directory containing the `.nft.json` file and identifies the source location on the build machine. `additionalRoots` names match those provided in `next.config.js` and are guaranteed to be valid directory names on most operating systems and filesystems. - `additionalRoots` is an ordered list of named source roots. Each root has its own `files`, `fileHashes`, and `symlinks` lists. Its `path` is relative to the directory containing the `.nft.json` file and identifies the source location on the build machine. Names are guaranteed to be valid directory names.
## Automatically Copying Traced Files ## Automatically Copying Traced Files
......
...@@ -159,7 +159,21 @@ To avoid ambiguous references to files, additional roots may not overlap `turbop ...@@ -159,7 +159,21 @@ To avoid ambiguous references to files, additional roots may not overlap `turbop
Additional roots change Next.js's [`trace format`](/docs/app/api-reference/config/next-config-js/output#turbopack-nft-extensions). Additional roots change Next.js's [`trace format`](/docs/app/api-reference/config/next-config-js/output#turbopack-nft-extensions).
When using [`output: "standalone"`](/docs/app/api-reference/config/next-config-js/output#automatically-copying-traced-files), unbundled files stored in these additional roots will be copied to a subdirectory of the output directory containing the configured name of the root. When using [`output: "standalone"`](/docs/app/api-reference/config/next-config-js/output#automatically-copying-traced-files), unbundled files stored in these additional roots will be copied to a subdirectory of the output directory named after the root. For roots configured in `next.config.js`, this is the configured name with an `@` prefix (e.g. `next_additional_roots/@linkedPackages`).
#### Global virtual store
Some package managers can install packages into a global virtual store outside of the project. Set `experimental.turbopackDetectGlobalVirtualStore` to `true` to detect the store and add it as an additional root named `gvs`. Currently, only pnpm 11 or later with the [global virtual store](https://pnpm.io/global-virtual-store) enabled is supported.
```js filename="next.config.js"
module.exports = {
experimental: {
turbopackDetectGlobalVirtualStore: true,
},
}
```
Next.js reads the `virtualStoreDir` in `node_modules/.modules.yaml`, searching from the project directory up to `turbopack.root`. The root is only added when the store is outside of `turbopack.root`.
### Configuring webpack loaders ### Configuring webpack loaders
...@@ -438,14 +452,15 @@ The option automatically adds a polyfill for debug IDs to the JavaScript bundle ...@@ -438,14 +452,15 @@ The option automatically adds a polyfill for debug IDs to the JavaScript bundle
## Version History ## Version History
| Version | Changes | | Version | Changes |
| -------- | ---------------------------------------------------- | | -------- | ---------------------------------------------------------- |
| `16.4.0` | `experimental.turbopackAdditionalRoots` was added | | `16.5.0` | `experimental.turbopackDetectGlobalVirtualStore` was added |
| `16.2.0` | `turbopackLoader` import attributes were added. | | `16.4.0` | `experimental.turbopackAdditionalRoots` was added |
| `16.2.0` | `turbopack.rules.*.type` was added. | | `16.2.0` | `turbopackLoader` import attributes were added. |
| `16.2.0` | `turbopack.rules.*.condition.contentType` was added. | | `16.2.0` | `turbopack.rules.*.type` was added. |
| `16.2.0` | `turbopack.rules.*.condition.query` was added. | | `16.2.0` | `turbopack.rules.*.condition.contentType` was added. |
| `16.0.0` | `turbopack.debugIds` was added. | | `16.2.0` | `turbopack.rules.*.condition.query` was added. |
| `16.0.0` | `turbopack.rules.*.condition` was added. | | `16.0.0` | `turbopack.debugIds` was added. |
| `15.3.0` | `experimental.turbo` is changed to `turbopack`. | | `16.0.0` | `turbopack.rules.*.condition` was added. |
| `13.0.0` | `experimental.turbo` introduced. | | `15.3.0` | `experimental.turbo` is changed to `turbopack`. |
| `13.0.0` | `experimental.turbo` introduced. |
...@@ -54,9 +54,8 @@ export interface NftJson extends NftFileList { ...@@ -54,9 +54,8 @@ export interface NftJson extends NftFileList {
/** Turbopack extension: Paths stored with a different base path. */ /** Turbopack extension: Paths stored with a different base path. */
export interface NftAdditionalRoot extends NftFileList { export interface NftAdditionalRoot extends NftFileList {
/** /**
* Stable unique identifier provided in `next.config.js`. This can be used to * Stable unique identifier. This can be used to generate the output path
* generate the output path where these files are copied, such as * where these files are copied, such as `next_additional_roots/${name}`.
* `next_additional_roots/${name}`.
* *
* This uses a character set that is valid on most filesystems, and identifiers * This uses a character set that is valid on most filesystems, and identifiers
* are guaranteed not to overlap on case-insensitive filesystems. * are guaranteed not to overlap on case-insensitive filesystems.
......
...@@ -213,6 +213,7 @@ export const experimentalSchema = { ...@@ -213,6 +213,7 @@ export const experimentalSchema = {
}) })
) )
.optional(), .optional(),
turbopackDetectGlobalVirtualStore: z.boolean().optional(),
outputHashSalt: z.string().optional(), outputHashSalt: z.string().optional(),
useSkewCookie: z.boolean().optional(), useSkewCookie: z.boolean().optional(),
after: z.boolean().optional(), after: z.boolean().optional(),
......
...@@ -529,6 +529,12 @@ export interface ExperimentalConfig { ...@@ -529,6 +529,12 @@ export interface ExperimentalConfig {
string, string,
{ path: string; ignoreIfMissing?: boolean } { path: string; ignoreIfMissing?: boolean }
> >
/**
* Detect pnpm's global virtual store and add it as an additional root. Requires pnpm 11 or later.
*
* @default false
*/
turbopackDetectGlobalVirtualStore?: boolean
/** /**
* @deprecated Use the top-level `outputHashSalt` option instead. * @deprecated Use the top-level `outputHashSalt` option instead.
......
...@@ -146,7 +146,7 @@ describe('turbopack additional roots', () => { ...@@ -146,7 +146,7 @@ describe('turbopack additional roots', () => {
expect(crossRootSymlinks).toMatchInlineSnapshot(` expect(crossRootSymlinks).toMatchInlineSnapshot(`
[ [
{ {
"additionalRoot": "linkedPackages", "additionalRoot": "@linkedPackages",
"file": "../../node_modules/sibling-639f6b1f4617eee0", "file": "../../node_modules/sibling-639f6b1f4617eee0",
"target": "node_modules/sibling", "target": "node_modules/sibling",
}, },
...@@ -159,7 +159,7 @@ describe('turbopack additional roots', () => { ...@@ -159,7 +159,7 @@ describe('turbopack additional roots', () => {
"node_modules/sibling/index.js", "node_modules/sibling/index.js",
"node_modules/sibling/package.json", "node_modules/sibling/package.json",
], ],
"name": "linkedPackages", "name": "@linkedPackages",
"path": "<temporary-root>", "path": "<temporary-root>",
"symlinks": [], "symlinks": [],
}, },
...@@ -194,7 +194,7 @@ describe('turbopack additional roots', () => { ...@@ -194,7 +194,7 @@ describe('turbopack additional roots', () => {
const target = path.join( const target = path.join(
'next_additional_roots', 'next_additional_roots',
'linkedPackages', '@linkedPackages',
'node_modules', 'node_modules',
'sibling' 'sibling'
) )
...@@ -266,7 +266,7 @@ describe('turbopack additional roots', () => { ...@@ -266,7 +266,7 @@ describe('turbopack additional roots', () => {
const stagedRoot = path.join( const stagedRoot = path.join(
standaloneDirectory, standaloneDirectory,
'next_additional_roots', 'next_additional_roots',
'linkedPackages' '@linkedPackages'
) )
expect( expect(
await fs.pathExists( await fs.pathExists(
......
...@@ -22,11 +22,13 @@ workspace = true ...@@ -22,11 +22,13 @@ workspace = true
anyhow = { workspace = true } anyhow = { workspace = true }
bincode = { workspace = true } bincode = { workspace = true }
either = { workspace = true } either = { workspace = true }
fs-err = { workspace = true }
indoc = { workspace = true } indoc = { workspace = true }
serde = { workspace = true } serde = { workspace = true }
serde_json = { workspace = true } serde_json = { workspace = true }
serde_qs = { workspace = true } serde_qs = { workspace = true }
smallvec = { workspace = true } smallvec = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
...@@ -40,3 +42,6 @@ turbopack-core = { workspace = true } ...@@ -40,3 +42,6 @@ turbopack-core = { workspace = true }
turbopack-ecmascript = { workspace = true } turbopack-ecmascript = { workspace = true }
turbopack-ecmascript-runtime = { workspace = true } turbopack-ecmascript-runtime = { workspace = true }
[dev-dependencies]
tempfile = { workspace = true }
use std::{
io::{self, ErrorKind},
path::{Path, PathBuf},
};
use serde::Deserialize;
use thiserror::Error;
/// pnpm records its modules state in this file. Despite the extension, pnpm 11 and later write it
/// as JSON. Older versions write YAML, which is not supported.
const PNPM_MODULES_MANIFEST: &str = ".modules.yaml";
/// Detects pnpm's [global virtual store][gvs], which lives outside of the project. The symlinks
/// pnpm creates in `node_modules` point into it, so they can only be followed if it is an
/// additional root.
///
/// Both arguments are native filesystem paths; `project_path` is relative to `project_root`.
///
/// This reads `virtualStoreDir` from the nearest `node_modules/.modules.yaml`, searching from the
/// project directory up to the project root. The returned path is not canonicalized, and may not
/// exist. A virtual store located inside the project root (the default `node_modules/.pnpm`) is
/// later ignored.
///
/// [gvs]: https://pnpm.io/global-virtual-store
pub fn find_global_virtual_store_untracked(
project_root: &Path,
project_path: &Path,
) -> Result<Option<PathBuf>, GlobalVirtualStoreError> {
let mut dir = project_root.join(project_path);
while dir.starts_with(project_root) {
let modules_dir = dir.join("node_modules");
let manifest_path = modules_dir.join(PNPM_MODULES_MANIFEST);
match fs_err::read(&manifest_path) {
Ok(manifest) => {
return virtual_store_dir_from_manifest(&modules_dir, &manifest).map_err(
|source| GlobalVirtualStoreError::InvalidManifest {
path: manifest_path,
source,
},
);
}
Err(error) if error.kind() == ErrorKind::NotFound => {}
Err(error) => return Err(GlobalVirtualStoreError::Io(error)),
}
if !dir.pop() {
break;
}
}
Ok(None)
}
#[derive(Debug, Error)]
pub enum GlobalVirtualStoreError {
/// The manifest exists, but could not be read. The error from `fs_err` includes the path.
#[error(transparent)]
Io(io::Error),
/// The manifest looks like JSON, but failed to parse.
#[error("failed to parse {}: {source}", path.display())]
InvalidManifest {
path: PathBuf,
source: serde_json::Error,
},
}
fn virtual_store_dir_from_manifest(
modules_dir: &Path,
manifest: &[u8],
) -> Result<Option<PathBuf>, serde_json::Error> {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct PnpmModulesManifest {
virtual_store_dir: Option<String>,
}
// This is a `*.yaml` file, but pnpm >=11.0.0 stores this as JSON (YAML is a superset of JSON).
// We don't want to pull in a YAML parser dependency, so older versions are ignored. They write
// block-style YAML, which never starts with `{`, so anything else must be valid JSON.
if manifest.trim_ascii_start().first() != Some(&b'{') {
return Ok(None);
}
let manifest: PnpmModulesManifest = serde_json::from_slice(manifest)?;
let Some(virtual_store_dir) = manifest.virtual_store_dir.filter(|dir| !dir.is_empty()) else {
return Ok(None);
};
// Relative paths are relative to the `node_modules` directory. `join` keeps absolute paths
// as-is.
Ok(Some(modules_dir.join(virtual_store_dir)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_virtual_store_dir_from_manifest() {
let modules_dir = Path::new("/workspace/project/node_modules");
assert_eq!(
virtual_store_dir_from_manifest(
modules_dir,
br#"{"layoutVersion":5,"virtualStoreDir":"../../../store/v11/links"}"#
)
.unwrap(),
Some(modules_dir.join("../../../store/v11/links"))
);
let absolute = if cfg!(windows) {
r"C:\store\v11\links"
} else {
"/store/v11/links"
};
assert_eq!(
virtual_store_dir_from_manifest(
modules_dir,
serde_json::json!({ "virtualStoreDir": absolute })
.to_string()
.as_bytes()
)
.unwrap(),
Some(PathBuf::from(absolute))
);
// missing, empty, or YAML (written by pnpm 10 and older)
for manifest in [
"",
"{}",
r#"{"virtualStoreDir":""}"#,
"virtualStoreDir: ../../store/v10/links\n",
] {
assert_eq!(
virtual_store_dir_from_manifest(modules_dir, manifest.as_bytes()).unwrap(),
None,
"{manifest}"
);
}
// malformed JSON
assert!(virtual_store_dir_from_manifest(modules_dir, br#" {"virtualStoreDir":"#).is_err());
}
#[test]
fn finds_global_virtual_store() {
let temp = tempfile::tempdir().unwrap();
let temp = fs_err::canonicalize(temp.path()).unwrap();
let root = temp.join("workspace");
let store = temp.join("store/v11/links");
fs_err::create_dir_all(root.join("node_modules")).unwrap();
fs_err::create_dir_all(root.join("apps/web/node_modules")).unwrap();
fs_err::create_dir_all(&store).unwrap();
fs_err::write(
root.join("node_modules").join(PNPM_MODULES_MANIFEST),
r#"{"virtualStoreDir":"../../store/v11/links"}"#,
)
.unwrap();
let find = |project_root: &Path, project_path: &Path| {
find_global_virtual_store_untracked(project_root, project_path)
.unwrap()
.map(|path| fs_err::canonicalize(path).unwrap())
};
// searches upwards from the project directory
assert_eq!(find(&root, Path::new("apps/web")), Some(store.clone()));
assert_eq!(find(&root, Path::new("")), Some(store));
// does not search above the project root
assert_eq!(find(&root.join("apps"), Path::new("web")), None);
}
}
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
pub(crate) mod chunking_context; pub(crate) mod chunking_context;
pub mod ecmascript; pub mod ecmascript;
pub mod fs; pub mod fs;
pub mod global_virtual_store;
pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder}; pub use chunking_context::{NodeJsChunkingContext, NodeJsChunkingContextBuilder};
pub use ecmascript::node::{ pub use ecmascript::node::{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment