Unverified Commit f01543a7 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Fix unsafe caching for isolated tests (#45602)

Noticed while testing across CI environments that the `test-pack` handling is not-concurrent or cache safe so this removes it from being used by default and moves it behind a flag to allow further investigating later. 

In CI if `test-pack` is called at the same time due to multiple concurrency it can cause the resulted archive to be corrupted so may require a form of lock to resolve in CI although locally re-using the same archive name/path isn't safe with pnpm as it won't bust the store cache and continue to leverage previous cache. 
parent 7cc1f33f
...@@ -55,27 +55,131 @@ module.exports = (actionInfo) => { ...@@ -55,27 +55,131 @@ module.exports = (actionInfo) => {
} }
}, },
async linkPackages({ repoDir, nextSwcVersion }) { async linkPackages({ repoDir, nextSwcVersion }) {
execa.sync('pnpm', ['turbo', 'run', 'test-pack'], { let useTestPack = process.env.NEXT_TEST_PACK
cwd: repoDir,
env: { NEXT_SWC_VERSION: nextSwcVersion }, if (useTestPack) {
}) execa.sync('pnpm', ['turbo', 'run', 'test-pack'], {
cwd: repoDir,
const pkgPaths = new Map() env: { NEXT_SWC_VERSION: nextSwcVersion },
const pkgs = await fs.readdir(path.join(repoDir, 'packages')) })
pkgs.forEach((pkgDirname) => { const pkgPaths = new Map()
const { name } = require(path.join( const pkgs = (await fs.readdir(path.join(repoDir, 'packages'))).filter(
repoDir, (item) => !item.startsWith('.')
'packages',
pkgDirname,
'package.json'
))
pkgPaths.set(
name,
path.join(repoDir, 'packages', pkgDirname, `packed-${pkgDirname}.tgz`)
) )
})
return pkgPaths pkgs.forEach((pkgDirname) => {
const { name } = require(path.join(
repoDir,
'packages',
pkgDirname,
'package.json'
))
pkgPaths.set(
name,
path.join(
repoDir,
'packages',
pkgDirname,
`packed-${pkgDirname}.tgz`
)
)
})
return pkgPaths
} else {
// TODO: remove after next stable release (current v13.1.2)
const pkgPaths = new Map()
const pkgDatas = new Map()
let pkgs
try {
pkgs = await fs.readdir(path.join(repoDir, 'packages'))
} catch (err) {
if (err.code === 'ENOENT') {
require('console').log('no packages to link')
return pkgPaths
}
throw err
}
for (const pkg of pkgs) {
const pkgPath = path.join(repoDir, 'packages', pkg)
const packedPkgPath = path.join(pkgPath, `${pkg}-packed.tgz`)
const pkgDataPath = path.join(pkgPath, 'package.json')
if (!fs.existsSync(pkgDataPath)) {
require('console').log(`Skipping ${pkgDataPath}`)
continue
}
const pkgData = require(pkgDataPath)
const { name } = pkgData
pkgDatas.set(name, {
pkgDataPath,
pkg,
pkgPath,
pkgData,
packedPkgPath,
})
pkgPaths.set(name, packedPkgPath)
}
for (const pkg of pkgDatas.keys()) {
const { pkgDataPath, pkgData } = pkgDatas.get(pkg)
for (const pkg of pkgDatas.keys()) {
const { packedPkgPath } = pkgDatas.get(pkg)
if (!pkgData.dependencies || !pkgData.dependencies[pkg]) continue
pkgData.dependencies[pkg] = packedPkgPath
}
// make sure native binaries are included in local linking
if (pkg === '@next/swc') {
if (!pkgData.files) {
pkgData.files = []
}
pkgData.files.push('native')
require('console').log(
'using swc binaries: ',
await exec(`ls ${path.join(path.dirname(pkgDataPath), 'native')}`)
)
}
if (pkg === 'next') {
if (nextSwcVersion) {
Object.assign(pkgData.dependencies, {
'@next/swc-linux-x64-gnu': nextSwcVersion,
})
} else {
if (pkgDatas.get('@next/swc')) {
pkgData.dependencies['@next/swc'] =
pkgDatas.get('@next/swc').packedPkgPath
} else {
pkgData.files.push('native')
}
}
}
await fs.writeFile(
pkgDataPath,
JSON.stringify(pkgData, null, 2),
'utf8'
)
}
// wait to pack packages until after dependency paths have been updated
// to the correct versions
await Promise.all(
Array.from(pkgDatas.keys()).map(async (pkgName) => {
const { pkg, pkgPath } = pkgDatas.get(pkgName)
await exec(
`cd ${pkgPath} && yarn pack -f '${pkg}-packed.tgz'`,
true
)
})
)
return pkgPaths
}
}, },
} }
} }
...@@ -19,6 +19,11 @@ async function createNextInstall({ ...@@ -19,6 +19,11 @@ async function createNextInstall({
.traceAsyncFn(async (rootSpan) => { .traceAsyncFn(async (rootSpan) => {
const tmpDir = await fs.realpath(process.env.NEXT_TEST_DIR || os.tmpdir()) const tmpDir = await fs.realpath(process.env.NEXT_TEST_DIR || os.tmpdir())
const origRepoDir = path.join(__dirname, '../../') const origRepoDir = path.join(__dirname, '../../')
const tmpRepoDir = path.join(
tmpDir,
`next-repo-${randomBytes(32).toString('hex')}${dirSuffix}`
)
const installDir = path.join( const installDir = path.join(
tmpDir, tmpDir,
`next-install-${randomBytes(32).toString('hex')}${dirSuffix}` `next-install-${randomBytes(32).toString('hex')}${dirSuffix}`
...@@ -58,6 +63,28 @@ async function createNextInstall({ ...@@ -58,6 +63,28 @@ async function createNextInstall({
} }
}) })
for (const item of ['package.json', 'packages']) {
await rootSpan
.traceChild(`copy ${item} to temp dir`)
.traceAsyncFn(async () => {
await fs.copy(
path.join(origRepoDir, item),
path.join(tmpRepoDir, item),
{
filter: (item) => {
return (
!item.includes('node_modules') &&
!item.includes('.DS_Store') &&
// Exclude Rust compilation files
!/next[\\/]build[\\/]swc[\\/]target/.test(item) &&
!/next-swc[\\/]target/.test(item)
)
},
}
)
})
}
let combinedDependencies = dependencies let combinedDependencies = dependencies
if (!(packageJson && packageJson.nextPrivateSkipLocalDeps)) { if (!(packageJson && packageJson.nextPrivateSkipLocalDeps)) {
...@@ -65,7 +92,7 @@ async function createNextInstall({ ...@@ -65,7 +92,7 @@ async function createNextInstall({
.traceChild('linkPackages') .traceChild('linkPackages')
.traceAsyncFn(() => .traceAsyncFn(() =>
linkPackages({ linkPackages({
repoDir: origRepoDir, repoDir: tmpRepoDir,
}) })
) )
combinedDependencies = { combinedDependencies = {
...@@ -109,7 +136,11 @@ async function createNextInstall({ ...@@ -109,7 +136,11 @@ async function createNextInstall({
await rootSpan await rootSpan
.traceChild('run generic install command') .traceChild('run generic install command')
.traceAsyncFn(async () => { .traceAsyncFn(async () => {
const args = ['install', '--strict-peer-dependencies=false'] const args = [
'install',
'--strict-peer-dependencies=false',
'--no-frozen-lockfile',
]
if (process.env.NEXT_TEST_PREFER_OFFLINE === '1') { if (process.env.NEXT_TEST_PREFER_OFFLINE === '1') {
args.push('--prefer-offline') args.push('--prefer-offline')
...@@ -123,6 +154,7 @@ async function createNextInstall({ ...@@ -123,6 +154,7 @@ async function createNextInstall({
}) })
} }
await fs.remove(tmpRepoDir)
return installDir return installDir
}) })
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment