Unverified Commit ea9c5aac authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Update caching for swc turbo builds (#42929)

Updates to leverage turbo remote caching for the swc builds and
normalize the package versions to prevent repo version bumps from
un-necessarily invalidating the cache.

Full cache hits can be seen in this test run
https://github.com/vercel/next.js/actions/runs/3469932131/jobs/5797559609
parent a0e51781
This diff is collapsed.
......@@ -5,7 +5,8 @@
"scripts": {
"build-native": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --features plugin --js false native",
"build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --js false native",
"build-wasm": "wasm-pack build crates/wasm --scope=next"
"build-wasm": "wasm-pack build crates/wasm --scope=next",
"cache-build-native": "echo $(ls native)"
},
"napi": {
"name": "next-swc",
......
#!/usr/bin/env node
// @ts-check
/*
This prevents busting the turbo cache un-necessarily due
to bumping the version in the repo's package.json files
*/
const path = require('path')
const fs = require('fs/promises')
const cwd = process.cwd()
const NORMALIZED_VERSION = '0.0.0'
const readJson = async (filePath) =>
JSON.parse(await fs.readFile(filePath, 'utf8'))
const writeJson = async (filePath, data) =>
fs.writeFile(filePath, JSON.stringify(data, null, 2) + '\n')
;(async function () {
const packages = await fs.readdir(path.join(cwd, 'packages'))
const pkgJsonData = new Map()
const pkgNames = []
await Promise.all(
packages.map(async (pkgDir) => {
const data = await readJson(
path.join(cwd, 'packages', pkgDir, 'package.json')
)
pkgNames.push(data.name)
pkgJsonData.set(pkgDir, data)
})
)
const normalizeVersions = async (filePath, data) => {
data = data || (await readJson(filePath))
const version = data.version
if (version) {
data.version = NORMALIZED_VERSION
const normalizeEntry = (type, key) => {
const pkgVersion = data[type][key]
if (pkgNames.includes(key) && pkgVersion === version) {
data[type][key] = NORMALIZED_VERSION
}
}
for (const key of Object.keys(data.dependencies || {})) {
normalizeEntry('dependencies', key)
}
for (const key of Object.keys(data.devDependencies || {})) {
normalizeEntry('devDependencies', key)
}
await writeJson(filePath, data)
}
}
await Promise.all(
packages.map((pkgDir) =>
normalizeVersions(
path.join(cwd, 'packages', pkgDir, 'package.json'),
pkgJsonData.get(pkgDir)
)
)
)
await normalizeVersions(path.join(cwd, 'lerna.json'))
await fs.unlink(path.join(cwd, 'pnpm-lock.yaml'))
await fs.writeFile(path.join(cwd, 'pnpm-lock.yaml'), '')
})()
#!/usr/bin/env node
// @ts-check
const { execSync } = require('child_process')
;(async function () {
const turboResult = execSync(
`pnpm turbo run cache-build-native --dry=json -- --platform --release --target x86_64-unknown-freebsd`
).toString()
const turboData = JSON.parse(turboResult)
const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')
if (!task) {
console.warn(`Failed to find related turbo task`, turboResult)
return
}
// pull cache if it was available
if (task.cacheState.local || task.cacheState.remote) {
const pullResult = execSync(
`pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd`
).toString()
console.log(pullResult)
} else {
console.warn(`No turbo cache was available, continuing...`)
console.warn(task)
}
})()
......@@ -13,6 +13,10 @@
"dependsOn": ["^build-wasm"],
"outputs": ["crates/wasm/pkg/*"]
},
"cache-build-native": {
"dependsOn": ["^cache-build-native"],
"outputs": ["native/*.node"]
},
"build": {
"dependsOn": ["^build"],
"outputs": ["dist/**"]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment