Unverified Commit 357b2b3c authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Use pnpm for isolated test installs (#98425)

### What?

Migrate the five local isolated test installs that explicitly used npm
to pnpm. The Nx fixture now has pnpm workspace metadata, while
filesystem-layout-sensitive fixtures use pnpm's hoisted linker with
copied package files.

### Why?

The npm-based Nx install bypassed the repository's centralized
supply-chain protections and could select a package immediately after
publication, including temporarily incomplete multi-package releases.
Using pnpm makes isolated installs inherit the repository's
`minimumReleaseAge`, exclusions, and exotic-subdependency policy.

The other npm installs depended on npm-style real package directories.
On Node versions affected by nodejs/node#65113, hoisted/copy mode
preserves that layout without leaving these fixtures outside the shared
pnpm security configuration; fixed Node releases use normal pnpm
linking.

### How?

- Use normal pnpm workspace resolution for the Nx fixture.
- Use `node-linker=hoisted` and `package-import-method=copy` for
filesystem tests only on affected Node releases; Node 24.21+ and 26.8+
use normal linking. Node 20 CI keeps the workaround because no fixed
Node 20 release exists.
- Validate local `@next/env` tarballs through the lockfile when hoisted
installs do not expose pnpm's virtual-store path marker.
- Keep the deployment-environment npm install unchanged.

### Verification

- `pnpm build-all`
- `pnpm types`
- A 9-version throwaway assertion verified the affected/fixed Node
release matrix
- `pnpm test-dev-turbo test/e2e/app-dir/nx-handling/nx-handling.test.ts
test/e2e/handle-non-hoisted-swc-helpers/index.test.ts
test/e2e/filesystem-cache/filesystem-cache.test.ts
test/e2e/filesystem-cache/warm-restart-task-stats.test.ts
test/e2e/filesystem-cache/evict-after-snapshot.test.ts` — all 25 tests
passed after installing the sandbox's missing Playwright browser
- Production Turbopack: Nx, non-hoisted SWC helper, build-cache-default,
and warm restart passed (9/9)
- `filesystem-cache.test.ts` production baseline: 15/17 passed; the same
two cache-growth bounds fail under both the unchanged npm fixture and
the pnpm fixture at nearly identical percentages, so they are
pre-existing sandbox-specific failures
- Generated-layout inspection: no package symlinks outside expected
`.bin` command shims; package files are copied; `node_modules/.pnpm` is
metadata-only

<!-- NEXT_JS_LLM -->


<!-- fleet 81cd457d-6956-4cf9-b6f6-9ebf9d95f285 -->

---------
Co-authored-by: 's avatarvercel-fleet-prod[bot] <318278635+vercel-fleet-prod[bot]@users.noreply.github.com>
Co-authored-by: 's avatarTobias Koppers <1365881+sokra@users.noreply.github.com>
parent 4bb6fc4b
...@@ -4,14 +4,12 @@ describe('nx-handling', () => { ...@@ -4,14 +4,12 @@ describe('nx-handling', () => {
const { next } = nextTestSetup({ const { next } = nextTestSetup({
skipDeployment: true, skipDeployment: true,
files: __dirname, files: __dirname,
installCommand: 'npm i', buildCommand: 'pnpm run build',
buildCommand: 'npm run build', startCommand: isNextDev ? 'pnpm run dev' : 'pnpm run start',
startCommand: isNextDev ? 'npm run dev' : 'npm run start',
packageJson: { packageJson: {
name: '@nx-next/source', name: '@nx-next/source',
version: '0.0.0', version: '0.0.0',
private: true, private: true,
packageManager: 'npm@10.9.2',
scripts: { scripts: {
build: 'rm -rf dist; nx run next-nx-test:build', build: 'rm -rf dist; nx run next-nx-test:build',
dev: 'nx run next-nx-test:dev', dev: 'nx run next-nx-test:dev',
......
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
"build": "nx run next-nx-test:build", "build": "nx run next-nx-test:build",
"dev": "nx run next-nx-test:dev", "dev": "nx run next-nx-test:dev",
"start": "nx run next-nx-test:serve:production", "start": "nx run next-nx-test:serve:production",
"repro": "rm -rf dist && npm run build && npm run start" "repro": "rm -rf dist && pnpm run build && pnpm run start"
}, },
"private": true, "private": true,
"dependencies": { "dependencies": {
...@@ -27,6 +27,5 @@ ...@@ -27,6 +27,5 @@
}, },
"workspaces": [ "workspaces": [
"apps/*" "apps/*"
], ]
"packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
} }
import { nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { getPnpmRealpathWorkaround } from '../../lib/pnpm-realpath-workaround'
import { retry, waitFor } from 'next-test-utils' import { retry, waitFor } from 'next-test-utils'
// Eviction requires the dev server (HMR) and persistent caching (Turbopack). // Eviction requires the dev server (HMR) and persistent caching (Turbopack).
...@@ -8,25 +9,20 @@ import { retry, waitFor } from 'next-test-utils' ...@@ -8,25 +9,20 @@ import { retry, waitFor } from 'next-test-utils'
// @force-gate !deploy // @force-gate !deploy
// @force-gate dev // @force-gate dev
describe('evict-after-snapshot', () => { describe('evict-after-snapshot', () => {
const envVars = [ const env = {
'ENABLE_CACHING=1', ENABLE_CACHING: '1',
'TURBO_ENGINE_IGNORE_DIRTY=1', TURBO_ENGINE_IGNORE_DIRTY: '1',
'TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000', TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS: '1000',
// Persist even tiny snapshots so the test doesn't depend on the // Persist even tiny snapshots so the test doesn't depend on the
// minimum-compilation-time threshold. // minimum-compilation-time threshold.
'TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS=0', TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS: '0',
'ENABLE_EVICTION=1', ENABLE_EVICTION: '1',
].join(' ') }
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: __dirname, files: __dirname,
packageJson: { overrideFiles: getPnpmRealpathWorkaround(),
scripts: { env,
dev: `${envVars} next dev`,
},
},
installCommand: 'npm i',
startCommand: 'npm run dev',
}) })
async function waitForSnapshotAndEviction() { async function waitForSnapshotAndEviction() {
......
...@@ -5,6 +5,7 @@ import fs from 'fs/promises' ...@@ -5,6 +5,7 @@ import fs from 'fs/promises'
import { existsSync } from 'fs' import { existsSync } from 'fs'
import path from 'path' import path from 'path'
import { parseTraceEvents } from '../../lib/parse-trace-file' import { parseTraceEvents } from '../../lib/parse-trace-file'
import { getPnpmRealpathWorkaround } from '../../lib/pnpm-realpath-workaround'
async function getDirectorySize(dirPath: string): Promise<number> { async function getDirectorySize(dirPath: string): Promise<number> {
try { try {
...@@ -39,32 +40,22 @@ for (const cacheEnabled of [false, true]) { ...@@ -39,32 +40,22 @@ for (const cacheEnabled of [false, true]) {
delete process.env.NEXT_PUBLIC_ENV_VAR delete process.env.NEXT_PUBLIC_ENV_VAR
}) })
let envVars = [ const env = {
`ENABLE_CACHING=${cacheEnabled ? '1' : ''}`, ENABLE_CACHING: cacheEnabled ? '1' : '',
// Make it easier to run in development, test directories are cleared between runs already so this is safe. // Make it easier to run in development, test directories are cleared between runs already so this is safe.
`TURBO_ENGINE_IGNORE_DIRTY=1`, TURBO_ENGINE_IGNORE_DIRTY: '1',
// decrease the idle timeout to make the test more reliable // decrease the idle timeout to make the test more reliable
`TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000`, TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS: '1000',
// persist even tiny snapshots so the test doesn't depend on the // persist even tiny snapshots so the test doesn't depend on the
// minimum-compilation-time threshold // minimum-compilation-time threshold
`TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS=0`, TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS: '0',
].join(' ') }
const { next, isTurbopack } = nextTestSetup({ const { next, isTurbopack } = nextTestSetup({
files: __dirname, files: __dirname,
packageJson: { overrideFiles: getPnpmRealpathWorkaround(),
packageManager: 'npm@10.9.2', // Pass the cache setting through every harness-managed build and restart.
scripts: { env,
build: `${envVars} next build`,
dev: `${envVars} next dev`,
start: 'next start',
},
},
// We need to use npm here as pnpms symlinks trigger a weird bug (kernel bug?)
installCommand: 'npm i',
// Next is always started with caching, but this can disable it for the followup restarts
buildCommand: `npm run build`,
startCommand: isNextDev ? 'npm run dev' : 'npm run start',
}) })
beforeAll(() => { beforeAll(() => {
......
import { nextTestSetup, isNextDev } from 'e2e-utils' import { nextTestSetup, isNextDev } from 'e2e-utils'
import { getPnpmRealpathWorkaround } from '../../lib/pnpm-realpath-workaround'
import { waitFor } from 'next-test-utils' import { waitFor } from 'next-test-utils'
import fs from 'fs/promises' import fs from 'fs/promises'
import path from 'path' import path from 'path'
...@@ -30,31 +31,22 @@ const STATS_RELATIVE_PATH = '.next/warm-restart-task-stats.json' ...@@ -30,31 +31,22 @@ const STATS_RELATIVE_PATH = '.next/warm-restart-task-stats.json'
// @force-gate !deploy // @force-gate !deploy
// @force-gate turbopack // @force-gate turbopack
describe('warm-restart task statistics', () => { describe('warm-restart task statistics', () => {
const env = [ const env = {
'ENABLE_CACHING=1', ENABLE_CACHING: '1',
'TURBO_ENGINE_IGNORE_DIRTY=1', TURBO_ENGINE_IGNORE_DIRTY: '1',
'TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS=1000', TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS: '1000',
// Persist even tiny snapshots so the test doesn't depend on the // Persist even tiny snapshots so the test doesn't depend on the
// minimum-compilation-time threshold. // minimum-compilation-time threshold.
'TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS=0', TURBO_ENGINE_SNAPSHOT_MIN_ACTIVE_TIME_MILLIS: '0',
`NEXT_TURBOPACK_TASK_STATISTICS=${STATS_RELATIVE_PATH}`, NEXT_TURBOPACK_TASK_STATISTICS: STATS_RELATIVE_PATH,
// Wait for turbo-tasks to persist the cache before the dev process exits. // Wait for turbo-tasks to persist the cache before the dev process exits.
'NEXT_DEV_WAIT_FOR_TURBOPACK_SHUTDOWN=1', NEXT_DEV_WAIT_FOR_TURBOPACK_SHUTDOWN: '1',
].join(' ') }
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: __dirname, files: __dirname,
packageJson: { overrideFiles: getPnpmRealpathWorkaround(),
packageManager: 'npm@10.9.2', env,
scripts: {
build: `${env} next build`,
dev: `${env} next dev`,
start: 'next start',
},
},
installCommand: 'npm i',
buildCommand: 'npm run build',
startCommand: isNextDev ? 'npm run dev' : 'npm run start',
}) })
beforeAll(() => { beforeAll(() => {
......
import { isNextDev, nextTestSetup } from 'e2e-utils' import { nextTestSetup } from 'e2e-utils'
import { renderViaHTTP } from 'next-test-utils' import { renderViaHTTP } from 'next-test-utils'
describe('handle-non-hoisted-swc-helpers', () => { describe('handle-non-hoisted-swc-helpers', () => {
const { next } = nextTestSetup({ const { next } = nextTestSetup({
files: { files: {
'.npmrc': `# The helper move below needs real package directories, not pnpm symlinks.
node-linker=hoisted
package-import-method=copy
`,
'pages/index.js': ` 'pages/index.js': `
export default function Page() { export default function Page() {
return <p>hello world</p> return <p>hello world</p>
...@@ -20,18 +24,8 @@ describe('handle-non-hoisted-swc-helpers', () => { ...@@ -20,18 +24,8 @@ describe('handle-non-hoisted-swc-helpers', () => {
} }
`, `,
}, },
packageJson: {
packageManager: 'npm@10.9.2',
scripts: {
build: 'next build',
dev: 'next dev',
start: 'next start',
},
},
installCommand: installCommand:
'npm install; mkdir -p node_modules/next/node_modules/@swc; mv node_modules/@swc/helpers node_modules/next/node_modules/@swc/', 'pnpm install && mkdir -p node_modules/next/node_modules/@swc && mv node_modules/@swc/helpers node_modules/next/node_modules/@swc/',
buildCommand: 'npm run build',
startCommand: isNextDev ? 'npm run dev' : 'npm run start',
dependencies: {}, dependencies: {},
}) })
......
...@@ -139,6 +139,49 @@ async function applyWorkspaceOverrides(installDir, isolationRoot, overrides) { ...@@ -139,6 +139,49 @@ async function applyWorkspaceOverrides(installDir, isolationRoot, overrides) {
await fs.writeFile(workspaceFile, yaml.dump(workspaceConfig)) await fs.writeFile(workspaceFile, yaml.dump(workspaceConfig))
} }
/**
* pnpm's hoisted linker does not expose the `@pkg+name@file` virtual-store
* path used by the default linker. Verify the exact local tarball through the
* lockfile instead.
*
* @param {string} installDir
* @param {string} packageName
* @param {string} expectedTarballPath
* @returns {Promise<boolean>}
*/
async function lockfileResolvesLocalTarball(
installDir,
packageName,
expectedTarballPath
) {
const lockfile = /** @type {Record<string, any>} */ (
yaml.load(
await fs.readFile(path.join(installDir, 'pnpm-lock.yaml'), 'utf8')
)
)
const expectedRealpath = await fs.realpath(expectedTarballPath)
for (const [key, pkg] of Object.entries(lockfile.packages || {})) {
const tarball = pkg?.resolution?.tarball
if (
!key.startsWith(`${packageName}@file:`) ||
typeof tarball !== 'string' ||
!tarball.startsWith('file:')
) {
continue
}
const resolvedTarball = path.resolve(
installDir,
tarball.slice('file:'.length)
)
if ((await fs.realpath(resolvedTarball)) === expectedRealpath) {
return true
}
}
return false
}
/** /**
* @param {import('next/dist/trace').Span} parentSpan * @param {import('next/dist/trace').Span} parentSpan
* @returns {Promise<Map<string, string>>} * @returns {Promise<Map<string, string>>}
...@@ -368,7 +411,9 @@ async function createNextInstall({ ...@@ -368,7 +411,9 @@ async function createNextInstall({
.traceAsyncFn(() => installDependencies(installDir, tmpDir)) .traceAsyncFn(() => installDependencies(installDir, tmpDir))
// `@next/env` is a dependency of `next`, so it only resolves to the // `@next/env` is a dependency of `next`, so it only resolves to the
// local tarball if the overrides were applied. // local tarball if the overrides were applied. Every generic isolated
// install reaches this guard, but the lockfile fallback short-circuits
// off when the default linker exposes its virtual-store path.
if (!combinedDependencies['@next/env']) { if (!combinedDependencies['@next/env']) {
const envDir = await fs.realpath( const envDir = await fs.realpath(
path.join( path.join(
...@@ -376,7 +421,18 @@ async function createNextInstall({ ...@@ -376,7 +421,18 @@ async function createNextInstall({
'../@next/env' '../@next/env'
) )
) )
if (!envDir.includes('@next+env@file')) { const envTarballPath = pkgPaths.get('@next/env')
if (
!envDir.includes('@next+env@file') &&
!(
envTarballPath &&
(await lockfileResolvesLocalTarball(
installDir,
'@next/env',
envTarballPath
))
)
) {
throw new Error( throw new Error(
`@next/env resolved from the npm registry instead of the local tarball (${envDir}), ` + `@next/env resolved from the npm registry instead of the local tarball (${envDir}), ` +
'the workspace overrides were not applied to the install' 'the workspace overrides were not applied to the install'
......
const PNPM_HOISTED_CONFIG = `# Work around the Node.js realpath bug fixed in 24.21.0 and 26.8.0.
# https://github.com/nodejs/node/pull/65113
node-linker=hoisted
package-import-method=copy
`
function hasNodeRealpathFix(nodeVersion: string): boolean {
const [major, minor] = nodeVersion.split('.').map(Number)
return (
(major === 24 && minor >= 21) || major > 26 || (major === 26 && minor >= 8)
)
}
export function getPnpmRealpathWorkaround(
nodeVersion = process.versions.node
): Record<string, string> | undefined {
if (hasNodeRealpathFix(nodeVersion)) {
return undefined
}
return { '.npmrc': PNPM_HOISTED_CONFIG }
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment