Unverified Commit 8330e4c4 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

test: Improve test cache handler implementations (#98098)

- Some of them were not forwarding `getExpiration` or `softTags` to
`defaultCacheHandler`
- use-cache-cross-deployment was ignoring softTags and expiration. (This
patch was written by Sol)

---------
Co-authored-by: 's avatarHendrik Liebau <mail@hendrik-liebau.de>
parent 00b2275c
......@@ -8,6 +8,7 @@ const crypto = require('node:crypto')
const cacheDir = path.join(__dirname, '.file-system-cache')
/** @param {string} cacheKey */
function filePathForKey(cacheKey) {
const hash = crypto.createHash('sha256').update(cacheKey).digest('hex')
return path.join(cacheDir, `${hash}.json`)
......
......@@ -35,9 +35,7 @@ const cacheHandler = {
async getExpiration(tags) {
console.log('ModernCustomCacheHandler::getExpiration', JSON.stringify(tags))
// Expecting soft tags in `get` to be used by the cache handler for checking
// the expiration of a cache entry, instead of letting Next.js handle it.
return Infinity
return defaultCacheHandler.getExpiration(tags)
},
async updateTags(tags) {
......
......@@ -24,7 +24,7 @@ const cacheHandler = {
async getExpiration(tags) {
console.log('WiringModernCacheHandler::getExpiration', JSON.stringify(tags))
return Infinity
return defaultCacheHandler.getExpiration(tags)
},
async updateTags(tags) {
......
......@@ -3,117 +3,209 @@
const fs = require('fs')
const path = require('path')
const dataFilePath = path.join(__dirname, 'handler-remote-data.json')
/**
* @type {Record<string, string>}
* @type {Record<string, { value: string, expiresAt?: number }>}
*/
let data = {}
try {
data = JSON.parse(
fs.readFileSync(path.join(__dirname, 'handler-remote-data.json'), 'utf8')
)
data = JSON.parse(fs.readFileSync(dataFilePath, 'utf8'))
} catch (_e) {}
function persistData() {
fs.writeFileSync(dataFilePath, JSON.stringify(data, null, 2))
}
// Date.now is considered sync IO by cache components
const now = () => performance.timeOrigin + performance.now()
// This is a Redis-like interface.
const client = {
/**
* @param {string} key
* @returns unknown
* @returns {Promise<string | undefined>}
*/
async get(key) {
return data[key]
const stored = data[key]
if (!stored) return undefined
if (stored.expiresAt !== undefined && stored.expiresAt <= now()) {
delete data[key]
persistData()
return undefined
}
return stored.value
},
/**
*
* @param {string} key
* @param {string} value
* @param {{expire?: number} | undefined} options
*/
async set(key, value, options) {
// options.expire is ignored in this impl
data[key] = value
fs.writeFileSync(
path.join(__dirname, 'handler-remote-data.json'),
JSON.stringify(data, null, 2)
)
if (options?.expire !== undefined && options.expire <= 0) {
delete data[key]
} else {
data[key] = {
value,
expiresAt:
options?.expire === undefined
? undefined
: now() + options.expire * 1000,
}
}
persistData()
},
}
/** @type {Map<string, Promise<void>>} */
const pendingSets = new Map()
/** @param {string} cacheKey */
const entryKey = (cacheKey) => `entry:${cacheKey}`
/** @param {string} tag */
const tagKey = (tag) => `tag:${tag}`
/**
* @typedef {{ stale?: number, expired?: number }} TagManifestEntry
*/
/**
* @param {string} tag
* @returns {Promise<TagManifestEntry>}
*/
async function getTagManifestEntry(tag) {
const stored = await client.get(tagKey(tag))
return stored ? JSON.parse(stored) : {}
}
/**
* @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler}
*/
module.exports = {
async get(cacheKey, softTags) {
console.log('CustomCacheHandler::get', cacheKey, JSON.stringify([softTags]))
// Retrieve from Redis
const stored = await client.get(cacheKey)
const pendingSet = pendingSets.get(cacheKey)
if (pendingSet) {
await pendingSet
}
const stored = await client.get(entryKey(cacheKey))
if (!stored) return undefined
// Deserialize the entry
const data = JSON.parse(stored)
const entry = JSON.parse(stored)
let revalidate = entry.revalidate
for (const tag of entry.tags) {
const tagManifestEntry = await getTagManifestEntry(tag)
if (
tagManifestEntry.expired !== undefined &&
tagManifestEntry.expired <= now() &&
tagManifestEntry.expired > entry.timestamp
) {
return undefined
}
if (
tagManifestEntry.stale !== undefined &&
tagManifestEntry.stale > entry.timestamp
) {
revalidate = -1
}
}
// Reconstruct the ReadableStream from stored data
return {
value: new ReadableStream({
start(controller) {
controller.enqueue(Buffer.from(data.value, 'base64'))
controller.enqueue(Buffer.from(entry.value, 'base64'))
controller.close()
},
}),
tags: data.tags,
stale: data.stale,
timestamp: data.timestamp,
expire: data.expire,
revalidate: data.revalidate,
tags: entry.tags,
stale: entry.stale,
timestamp: entry.timestamp,
expire: entry.expire,
revalidate,
}
},
async set(cacheKey, pendingEntry) {
console.log('CustomCacheHandler::set', cacheKey)
const entry = await pendingEntry
// Read the stream to get the data
const reader = entry.value.getReader()
const chunks = []
/** @type {() => void} */
let resolvePending = () => {}
/** @type {Promise<void>} */
const pendingSet = new Promise((resolve) => {
resolvePending = /** @type {() => void} */ (resolve)
})
pendingSets.set(cacheKey, pendingSet)
try {
while (true) {
const { done, value } = await reader.read()
if (done) break
chunks.push(value)
const entry = await pendingEntry
// Read the stream to get the data
const reader = entry.value.getReader()
/** @type {Uint8Array[]} */
const chunks = []
try {
while (true) {
const { done, value } = await reader.read()
if (done) break
chunks.push(value)
}
} finally {
reader.releaseLock()
}
// Combine chunks and serialize
const value = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)))
await client.set(
entryKey(cacheKey),
JSON.stringify({
value: value.toString('base64'),
tags: entry.tags,
stale: entry.stale,
timestamp: entry.timestamp,
expire: entry.expire,
revalidate: entry.revalidate,
}),
{ expire: entry.expire }
)
} finally {
reader.releaseLock()
resolvePending()
pendingSets.delete(cacheKey)
}
// Combine chunks and serialize for Redis storage
const data = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)))
await client.set(
cacheKey,
JSON.stringify({
value: data.toString('base64'),
tags: entry.tags,
stale: entry.stale,
timestamp: entry.timestamp,
expire: entry.expire,
revalidate: entry.revalidate,
}),
{ expire: entry.expire }
)
},
async refreshTags() {
// No-op for basic Redis implementation
// Could sync with external tag service if needed
// Tags are read directly from the remote store, so there is nothing to sync.
},
async getExpiration(tags) {
// Return 0 to indicate no tags have been revalidated
// Could query Redis for tag expiration timestamps if tracking them
return 0
const entries = await Promise.all(tags.map(getTagManifestEntry))
return Math.max(...entries.map((entry) => entry.expired || 0), 0)
},
async updateTags(tags, durations) {
// Implement tag-based invalidation if needed
// Could iterate over keys with matching tags and delete them
const currentTime = now()
await Promise.all(
tags.map(async (tag) => {
const entry = await getTagManifestEntry(tag)
if (durations) {
entry.stale = currentTime
if (durations.expire !== undefined) {
entry.expired = currentTime + durations.expire * 1000
}
} else {
entry.expired = currentTime
}
await client.set(tagKey(tag), JSON.stringify(entry), undefined)
})
)
},
}
......@@ -22,9 +22,7 @@ const cacheHandler = {
},
async getExpiration(tags) {
// Expecting soft tags in `get` to be used by the cache handler for checking
// the expiration of a cache entry, instead of letting Next.js handle it.
return Infinity
return defaultCacheHandler.getExpiration(tags)
},
async updateTags(tags) {
......
......@@ -9,8 +9,8 @@ const defaultCacheHandler =
* @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler}
*/
const cacheHandler = {
async get(cacheKey) {
return defaultCacheHandler.get(cacheKey)
async get(cacheKey, softTags) {
return defaultCacheHandler.get(cacheKey, softTags)
},
async set(cacheKey, pendingEntry) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment