Unverified Commit 43c1c60b authored by Sebastian "Sebbie" Silbermann's avatar Sebastian "Sebbie" Silbermann Committed by GitHub

[16.3] [ci] Use OIDC tokens to read private preview builds (#97258)

Backports https://github.com/vercel/next.js/pull/96919

🤖 Generated with [Claude Code](https://claude.com/claude-code)
parent 9dbb357d
...@@ -741,7 +741,12 @@ jobs: ...@@ -741,7 +741,12 @@ jobs:
# Outer bound only. The script gives up first, so a tarball that never # Outer bound only. The script gives up first, so a tarball that never
# arrives is reported by it rather than by the runner killing the job. # arrives is reported by it rather than by the runner killing the job.
timeout-minutes: 35 timeout-minutes: 35
permissions: {} permissions:
# contents: read is needed to check out in private repos (the mirror).
contents: read
# Minting GitHub OIDC tokens to read private preview builds. Dormant in
# this repo since builds here are public.
id-token: write
steps: steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with: with:
...@@ -755,7 +760,7 @@ jobs: ...@@ -755,7 +760,7 @@ jobs:
- name: Wait for preview tarball - name: Wait for preview tarball
env: env:
PREVIEW_BUILDS_BASE_URL: ${{ vars.PREVIEW_BUILDS_BASE_URL }} PREVIEW_BUILDS_BASE_URL: ${{ vars.PREVIEW_BUILDS_BASE_URL }}
PREVIEW_BUILDS_READ_TOKEN: ${{ secrets.PREVIEW_BUILDS_READ_TOKEN }} PREVIEW_BUILDS_ACCESS: ${{ vars.PREVIEW_BUILDS_ACCESS }}
run: | run: |
set -euo pipefail set -euo pipefail
node scripts/wait-for-preview-tarball.mjs \ node scripts/wait-for-preview-tarball.mjs \
...@@ -774,6 +779,11 @@ jobs: ...@@ -774,6 +779,11 @@ jobs:
matrix: matrix:
group: [1/5, 2/5, 3/5, 4/5, 5/5] group: [1/5, 2/5, 3/5, 4/5, 5/5]
permissions:
contents: read
# Minting GitHub OIDC tokens to read private preview builds. Dormant in
# this repo since builds here are public.
id-token: write
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
# Keep Next.js related env variables in sync with additionalEnv in next-deploy.ts # Keep Next.js related env variables in sync with additionalEnv in next-deploy.ts
...@@ -803,6 +813,11 @@ jobs: ...@@ -803,6 +813,11 @@ jobs:
matrix: matrix:
group: [1/5, 2/5, 3/5, 4/5, 5/5] group: [1/5, 2/5, 3/5, 4/5, 5/5]
permissions:
contents: read
# Minting GitHub OIDC tokens to read private preview builds. Dormant in
# this repo since builds here are public.
id-token: write
uses: ./.github/workflows/build_reusable.yml uses: ./.github/workflows/build_reusable.yml
with: with:
afterBuild: | afterBuild: |
......
...@@ -142,9 +142,7 @@ env: ...@@ -142,9 +142,7 @@ env:
VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }} VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
# Read token for the auth-protected preview-builds npm mirror, used by deploy PREVIEW_BUILDS_ACCESS: ${{ vars.PREVIEW_BUILDS_ACCESS }}
# tests to install Next.js artifacts during the remote build.
PREVIEW_BUILDS_READ_TOKEN: ${{ secrets.PREVIEW_BUILDS_READ_TOKEN }}
NEXT_TEST_PREFER_OFFLINE: 1 NEXT_TEST_PREFER_OFFLINE: 1
NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }} NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }}
NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }} NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }}
......
...@@ -4,6 +4,7 @@ import yargs from 'yargs' ...@@ -4,6 +4,7 @@ import yargs from 'yargs'
import getChangedTests from './get-changed-tests.mjs' import getChangedTests from './get-changed-tests.mjs'
import { import {
assertPreviewTarballPublished, assertPreviewTarballPublished,
createPreviewBuildsReadTokenGetter,
previewTarballUrl, previewTarballUrl,
} from './wait-for-preview-tarball.mjs' } from './wait-for-preview-tarball.mjs'
...@@ -111,7 +112,7 @@ async function main() { ...@@ -111,7 +112,7 @@ async function main() {
await assertPreviewTarballPublished({ await assertPreviewTarballPublished({
commitSha, commitSha,
previewBuildsBaseUrl, previewBuildsBaseUrl,
readToken: process.env.PREVIEW_BUILDS_READ_TOKEN, getReadToken: createPreviewBuildsReadTokenGetter(),
}) })
} }
......
...@@ -12,6 +12,76 @@ const DEFAULT_TIMEOUT_MINUTES = 30 ...@@ -12,6 +12,76 @@ const DEFAULT_TIMEOUT_MINUTES = 30
const POLL_INTERVAL_MS = 15_000 const POLL_INTERVAL_MS = 15_000
const PROGRESS_LOG_INTERVAL_MS = 60_000 const PROGRESS_LOG_INTERVAL_MS = 60_000
/**
* Mints a GitHub Actions OIDC token for the given audience.
* Returns null outside of GitHub Actions.
*
* @param {string} audience
* @returns {Promise<string | null>}
*/
async function mintGitHubActionsOidcToken(audience) {
const requestUrl = process.env.ACTIONS_ID_TOKEN_REQUEST_URL
const requestToken = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN
if (!requestUrl || !requestToken) {
return null
}
const url = new URL(requestUrl)
url.searchParams.set('audience', audience)
const response = await fetch(url, {
headers: { Authorization: `Bearer ${requestToken}` },
})
if (!response.ok) {
throw new Error(
`Failed to mint GitHub OIDC token: ${response.status} ${await response.text()}`
)
}
const { value } = await response.json()
return value
}
/**
* Reads a preview-builds credential, minting a fresh one when needed.
* vercel-packages accepts GitHub Actions OIDC tokens minted for its audience.
* Public preview builds need no credentials and return null. Private builds
* mint a GitHub Actions OIDC token, which expires about five minutes after
* issuance, so the token is cached and re-minted before it expires rather
* than minted once up front.
*
* @returns {() => Promise<string | null>}
*/
export function createPreviewBuildsReadTokenGetter() {
if (process.env.PREVIEW_BUILDS_ACCESS !== 'private') {
return async () => null
}
let cachedToken = null
let cachedTokenExpiresAt = 0
return async () => {
if (cachedToken !== null && cachedTokenExpiresAt - 60_000 > Date.now()) {
return cachedToken
}
const token = await mintGitHubActionsOidcToken(
'https://vercel-packages.vercel.app'
)
if (token === null) {
throw new Error(
'Preview builds are private (PREVIEW_BUILDS_ACCESS=private) ' +
'but no GitHub Actions OIDC token can be minted. ' +
'Grant the job the `id-token: write` permission.'
)
}
const payload = JSON.parse(
Buffer.from(token.split('.')[1], 'base64url').toString()
)
cachedToken = token
cachedTokenExpiresAt = payload.exp * 1000
return token
}
}
/** /**
* URL of the `next` preview tarball for a commit. `vercel-packages` answers * URL of the `next` preview tarball for a commit. `vercel-packages` answers
* with a redirect to Vercel Blob, which only serves the tarball once * with a redirect to Vercel Blob, which only serves the tarball once
...@@ -66,29 +136,37 @@ function commitChecksUrl(commitSha) { ...@@ -66,29 +136,37 @@ function commitChecksUrl(commitSha) {
* *
* @param {string} url * @param {string} url
* @param {Record<string, string> | undefined} headers * @param {Record<string, string> | undefined} headers
* @returns {Promise<{ published: boolean, lastResponse: string }>} * @returns {Promise<{ published: boolean, status: number | null, lastResponse: string, responseHeaders: string | null }>}
*/ */
async function probeTarball(url, headers) { async function probeTarball(url, headers) {
try { try {
const response = await fetch(url, { method: 'HEAD', headers }) const response = await fetch(url, { method: 'HEAD', headers })
return { return {
published: response.ok, published: response.ok,
status: response.status,
lastResponse: lastResponse:
response.status === 404 ? 'not published yet' : `${response.status}`, response.status === 404 ? 'not published yet' : `${response.status}`,
// Response headers carry request IDs that help debug failures.
responseHeaders: response.ok
? null
: JSON.stringify(Object.fromEntries(response.headers)),
} }
} catch (error) { } catch (error) {
return { return {
published: false, published: false,
status: null,
lastResponse: `request failed (${error instanceof Error ? error.message : error})`, lastResponse: `request failed (${error instanceof Error ? error.message : error})`,
responseHeaders: null,
} }
} }
} }
/** /**
* @param {string | undefined} readToken * @param {() => Promise<string | null>} getReadToken
* @returns {Record<string, string> | undefined} * @returns {Promise<Record<string, string> | undefined>}
*/ */
function requestHeaders(readToken) { async function requestHeaders(getReadToken) {
const readToken = await getReadToken()
return readToken ? { Authorization: `Bearer ${readToken}` } : undefined return readToken ? { Authorization: `Bearer ${readToken}` } : undefined
} }
...@@ -96,10 +174,16 @@ function requestHeaders(readToken) { ...@@ -96,10 +174,16 @@ function requestHeaders(readToken) {
* @param {object} options * @param {object} options
* @param {string} options.commitSha * @param {string} options.commitSha
* @param {string} options.lastResponse * @param {string} options.lastResponse
* @param {string | null} [options.responseHeaders]
* @param {number} [options.timeoutMs] Omitted when nothing was waited out. * @param {number} [options.timeoutMs] Omitted when nothing was waited out.
* @returns {Error} * @returns {Error}
*/ */
function notPublishedError({ commitSha, lastResponse, timeoutMs }) { function notPublishedError({
commitSha,
lastResponse,
responseHeaders,
timeoutMs,
}) {
const checksUrl = commitChecksUrl(commitSha) const checksUrl = commitChecksUrl(commitSha)
return new Error( return new Error(
`Preview tarball for commit ${commitSha} was not published` + `Preview tarball for commit ${commitSha} was not published` +
...@@ -108,7 +192,8 @@ function notPublishedError({ commitSha, lastResponse, timeoutMs }) { ...@@ -108,7 +192,8 @@ function notPublishedError({ commitSha, lastResponse, timeoutMs }) {
`The tarball is published by the "upload-preview-tarballs" workflow ` + `The tarball is published by the "upload-preview-tarballs" workflow ` +
`once "build-and-deploy" has completed for this commit, so check ` + `once "build-and-deploy" has completed for this commit, so check ` +
`whether that run failed or is still in progress.` + `whether that run failed or is still in progress.` +
(checksUrl ? ` See ${checksUrl}` : '') (checksUrl ? ` See ${checksUrl}` : '') +
(responseHeaders ? ` Response headers: ${responseHeaders}` : '')
) )
} }
...@@ -120,22 +205,22 @@ function notPublishedError({ commitSha, lastResponse, timeoutMs }) { ...@@ -120,22 +205,22 @@ function notPublishedError({ commitSha, lastResponse, timeoutMs }) {
* @param {object} options * @param {object} options
* @param {string} options.commitSha * @param {string} options.commitSha
* @param {string} [options.previewBuildsBaseUrl] * @param {string} [options.previewBuildsBaseUrl]
* @param {string} [options.readToken] * @param {() => Promise<string | null>} [options.getReadToken]
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
export async function assertPreviewTarballPublished({ export async function assertPreviewTarballPublished({
commitSha, commitSha,
previewBuildsBaseUrl, previewBuildsBaseUrl,
readToken, getReadToken,
}) { }) {
const url = previewTarballUrl(previewBuildsBaseUrl, commitSha) const url = previewTarballUrl(previewBuildsBaseUrl, commitSha)
const { published, lastResponse } = await probeTarball( const { published, lastResponse, responseHeaders } = await probeTarball(
url, url,
requestHeaders(readToken) await requestHeaders(getReadToken ?? (async () => null))
) )
if (!published) { if (!published) {
throw notPublishedError({ commitSha, lastResponse }) throw notPublishedError({ commitSha, lastResponse, responseHeaders })
} }
console.info(`Preview tarball for commit ${commitSha} is available at ${url}`) console.info(`Preview tarball for commit ${commitSha} is available at ${url}`)
...@@ -151,7 +236,7 @@ export async function assertPreviewTarballPublished({ ...@@ -151,7 +236,7 @@ export async function assertPreviewTarballPublished({
* @param {string} options.commitSha * @param {string} options.commitSha
* @param {string} [options.previewBuildsBaseUrl] * @param {string} [options.previewBuildsBaseUrl]
* @param {number} options.timeoutMs * @param {number} options.timeoutMs
* @param {string} [options.readToken] * @param {() => Promise<string | null>} [options.getReadToken]
* @param {number} [options.pollIntervalMs] * @param {number} [options.pollIntervalMs]
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
...@@ -159,11 +244,10 @@ export async function waitForPreviewTarball({ ...@@ -159,11 +244,10 @@ export async function waitForPreviewTarball({
commitSha, commitSha,
previewBuildsBaseUrl, previewBuildsBaseUrl,
timeoutMs, timeoutMs,
readToken, getReadToken = async () => null,
pollIntervalMs = POLL_INTERVAL_MS, pollIntervalMs = POLL_INTERVAL_MS,
}) { }) {
const url = previewTarballUrl(previewBuildsBaseUrl, commitSha) const url = previewTarballUrl(previewBuildsBaseUrl, commitSha)
const headers = requestHeaders(readToken)
const startedAt = Date.now() const startedAt = Date.now()
const deadline = startedAt + timeoutMs const deadline = startedAt + timeoutMs
let lastProgressLogAt = startedAt let lastProgressLogAt = startedAt
...@@ -173,7 +257,10 @@ export async function waitForPreviewTarball({ ...@@ -173,7 +257,10 @@ export async function waitForPreviewTarball({
) )
for (;;) { for (;;) {
const { published, lastResponse } = await probeTarball(url, headers) // A fresh token per probe: the OIDC token expires after five minutes,
// well before the overall timeout.
const { published, status, lastResponse, responseHeaders } =
await probeTarball(url, await requestHeaders(getReadToken))
const now = Date.now() const now = Date.now()
if (published) { if (published) {
...@@ -183,8 +270,21 @@ export async function waitForPreviewTarball({ ...@@ -183,8 +270,21 @@ export async function waitForPreviewTarball({
return return
} }
if (status === 401 || status === 403) {
// Retrying won't change the authorization outcome.
throw new Error(
`Not authorized to access the preview tarball at ${url} (last response: ${lastResponse}). ` +
(responseHeaders ? `Response headers: ${responseHeaders}` : '')
)
}
if (now >= deadline) { if (now >= deadline) {
throw notPublishedError({ commitSha, lastResponse, timeoutMs }) throw notPublishedError({
commitSha,
lastResponse,
responseHeaders,
timeoutMs,
})
} }
if (now - lastProgressLogAt >= PROGRESS_LOG_INTERVAL_MS) { if (now - lastProgressLogAt >= PROGRESS_LOG_INTERVAL_MS) {
...@@ -231,7 +331,7 @@ async function main() { ...@@ -231,7 +331,7 @@ async function main() {
commitSha, commitSha,
previewBuildsBaseUrl: values['preview-builds-base-url'], previewBuildsBaseUrl: values['preview-builds-base-url'],
timeoutMs: timeoutMinutes * 60_000, timeoutMs: timeoutMinutes * 60_000,
readToken: process.env.PREVIEW_BUILDS_READ_TOKEN, getReadToken: createPreviewBuildsReadTokenGetter(),
}) })
} }
......
...@@ -510,18 +510,26 @@ export class NextDeployInstance extends NextInstance { ...@@ -510,18 +510,26 @@ export class NextDeployInstance extends NextInstance {
this.parseIdsFromCliOuput() this.parseIdsFromCliOuput()
} }
// When the preview-builds npm mirror is auth-protected, the deploy build // When preview builds are private, the deploy build installs Next.js
// installs Next.js artifacts from it and needs credentials. We write an // artifacts from an auth-protected route and needs credentials. The build
// `.npmrc` with a read token (provided as a CI secret) so the remote install // authenticates with the Vercel OIDC token that Vercel automatically
// can authenticate. Only written when the token is set, so unprotected and // provides to builds (vercel-packages accepts it for allowlisted teams), so
// local deploy runs are unaffected. // we write an `.npmrc` referencing it. Referencing the environment variable
// instead of inlining a token keeps credentials out of the uploaded
// deployment source. Only written for private preview builds since public
// ones need no credentials and pnpm fails when an `.npmrc` references an
// unset environment variable.
// TODO: pnpm >= 10.34.2 no longer expands environment variables in
// repository .npmrc files (GHSA-3qhv-2rgh-x77r). An install command writing
// to the user-level pnpm config (like vercel/front does) did not work with
// `vercel deploy` and needs more investigation.
private async writeMirrorNpmrcIfNecessary(): Promise<void> { private async writeMirrorNpmrcIfNecessary(): Promise<void> {
const token = process.env.PREVIEW_BUILDS_READ_TOKEN
const baseUrlRaw = process.env.NEXT_TEST_PREVIEW_BUILDS_BASE_URL const baseUrlRaw = process.env.NEXT_TEST_PREVIEW_BUILDS_BASE_URL
const access = process.env.PREVIEW_BUILDS_ACCESS
if (!token || !baseUrlRaw) { if (!baseUrlRaw || access !== 'private') {
require('console').log( require('console').log(
`Skipping .npmrc write for preview-builds mirror: missing token or base URL` `Skipping .npmrc write for preview-builds mirror: missing base URL or preview builds are public`
) )
return return
} }
...@@ -536,7 +544,7 @@ export class NextDeployInstance extends NextInstance { ...@@ -536,7 +544,7 @@ export class NextDeployInstance extends NextInstance {
) )
await fs.writeFile( await fs.writeFile(
path.join(this.testDir, '.npmrc'), path.join(this.testDir, '.npmrc'),
`${registryKey}:_authToken=${token}\n` `${registryKey}:_authToken=\${VERCEL_OIDC_TOKEN}\n`
) )
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment