Unverified Commit 8330e4c4 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

test: Improve test cache handler implementations (#98098)

- Some of them were not forwarding `getExpiration` or `softTags` to
`defaultCacheHandler`
- use-cache-cross-deployment was ignoring softTags and expiration. (This
patch was written by Sol)

---------
Co-authored-by: 's avatarHendrik Liebau <mail@hendrik-liebau.de>
parent 00b2275c
...@@ -8,6 +8,7 @@ const crypto = require('node:crypto') ...@@ -8,6 +8,7 @@ const crypto = require('node:crypto')
const cacheDir = path.join(__dirname, '.file-system-cache') const cacheDir = path.join(__dirname, '.file-system-cache')
/** @param {string} cacheKey */
function filePathForKey(cacheKey) { function filePathForKey(cacheKey) {
const hash = crypto.createHash('sha256').update(cacheKey).digest('hex') const hash = crypto.createHash('sha256').update(cacheKey).digest('hex')
return path.join(cacheDir, `${hash}.json`) return path.join(cacheDir, `${hash}.json`)
......
...@@ -35,9 +35,7 @@ const cacheHandler = { ...@@ -35,9 +35,7 @@ const cacheHandler = {
async getExpiration(tags) { async getExpiration(tags) {
console.log('ModernCustomCacheHandler::getExpiration', JSON.stringify(tags)) console.log('ModernCustomCacheHandler::getExpiration', JSON.stringify(tags))
// Expecting soft tags in `get` to be used by the cache handler for checking return defaultCacheHandler.getExpiration(tags)
// the expiration of a cache entry, instead of letting Next.js handle it.
return Infinity
}, },
async updateTags(tags) { async updateTags(tags) {
......
...@@ -24,7 +24,7 @@ const cacheHandler = { ...@@ -24,7 +24,7 @@ const cacheHandler = {
async getExpiration(tags) { async getExpiration(tags) {
console.log('WiringModernCacheHandler::getExpiration', JSON.stringify(tags)) console.log('WiringModernCacheHandler::getExpiration', JSON.stringify(tags))
return Infinity return defaultCacheHandler.getExpiration(tags)
}, },
async updateTags(tags) { async updateTags(tags) {
......
...@@ -3,117 +3,209 @@ ...@@ -3,117 +3,209 @@
const fs = require('fs') const fs = require('fs')
const path = require('path') const path = require('path')
const dataFilePath = path.join(__dirname, 'handler-remote-data.json')
/** /**
* @type {Record<string, string>} * @type {Record<string, { value: string, expiresAt?: number }>}
*/ */
let data = {} let data = {}
try { try {
data = JSON.parse( data = JSON.parse(fs.readFileSync(dataFilePath, 'utf8'))
fs.readFileSync(path.join(__dirname, 'handler-remote-data.json'), 'utf8')
)
} catch (_e) {} } catch (_e) {}
function persistData() {
fs.writeFileSync(dataFilePath, JSON.stringify(data, null, 2))
}
// Date.now is considered sync IO by cache components
const now = () => performance.timeOrigin + performance.now()
// This is a Redis-like interface.
const client = { const client = {
/** /**
* @param {string} key * @param {string} key
* @returns unknown * @returns {Promise<string | undefined>}
*/ */
async get(key) { async get(key) {
return data[key] const stored = data[key]
if (!stored) return undefined
if (stored.expiresAt !== undefined && stored.expiresAt <= now()) {
delete data[key]
persistData()
return undefined
}
return stored.value
}, },
/** /**
*
* @param {string} key * @param {string} key
* @param {string} value * @param {string} value
* @param {{expire?: number} | undefined} options * @param {{expire?: number} | undefined} options
*/ */
async set(key, value, options) { async set(key, value, options) {
// options.expire is ignored in this impl if (options?.expire !== undefined && options.expire <= 0) {
data[key] = value delete data[key]
fs.writeFileSync( } else {
path.join(__dirname, 'handler-remote-data.json'), data[key] = {
JSON.stringify(data, null, 2) value,
) expiresAt:
options?.expire === undefined
? undefined
: now() + options.expire * 1000,
}
}
persistData()
}, },
} }
/** @type {Map<string, Promise<void>>} */
const pendingSets = new Map()
/** @param {string} cacheKey */
const entryKey = (cacheKey) => `entry:${cacheKey}`
/** @param {string} tag */
const tagKey = (tag) => `tag:${tag}`
/**
* @typedef {{ stale?: number, expired?: number }} TagManifestEntry
*/
/**
* @param {string} tag
* @returns {Promise<TagManifestEntry>}
*/
async function getTagManifestEntry(tag) {
const stored = await client.get(tagKey(tag))
return stored ? JSON.parse(stored) : {}
}
/** /**
* @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler} * @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler}
*/ */
module.exports = { module.exports = {
async get(cacheKey, softTags) { async get(cacheKey, softTags) {
console.log('CustomCacheHandler::get', cacheKey, JSON.stringify([softTags])) console.log('CustomCacheHandler::get', cacheKey, JSON.stringify([softTags]))
// Retrieve from Redis
const stored = await client.get(cacheKey) const pendingSet = pendingSets.get(cacheKey)
if (pendingSet) {
await pendingSet
}
const stored = await client.get(entryKey(cacheKey))
if (!stored) return undefined if (!stored) return undefined
// Deserialize the entry // Deserialize the entry
const data = JSON.parse(stored) const entry = JSON.parse(stored)
let revalidate = entry.revalidate
for (const tag of entry.tags) {
const tagManifestEntry = await getTagManifestEntry(tag)
if (
tagManifestEntry.expired !== undefined &&
tagManifestEntry.expired <= now() &&
tagManifestEntry.expired > entry.timestamp
) {
return undefined
}
if (
tagManifestEntry.stale !== undefined &&
tagManifestEntry.stale > entry.timestamp
) {
revalidate = -1
}
}
// Reconstruct the ReadableStream from stored data // Reconstruct the ReadableStream from stored data
return { return {
value: new ReadableStream({ value: new ReadableStream({
start(controller) { start(controller) {
controller.enqueue(Buffer.from(data.value, 'base64')) controller.enqueue(Buffer.from(entry.value, 'base64'))
controller.close() controller.close()
}, },
}), }),
tags: data.tags, tags: entry.tags,
stale: data.stale, stale: entry.stale,
timestamp: data.timestamp, timestamp: entry.timestamp,
expire: data.expire, expire: entry.expire,
revalidate: data.revalidate, revalidate,
} }
}, },
async set(cacheKey, pendingEntry) { async set(cacheKey, pendingEntry) {
console.log('CustomCacheHandler::set', cacheKey) console.log('CustomCacheHandler::set', cacheKey)
const entry = await pendingEntry /** @type {() => void} */
let resolvePending = () => {}
// Read the stream to get the data /** @type {Promise<void>} */
const reader = entry.value.getReader() const pendingSet = new Promise((resolve) => {
const chunks = [] resolvePending = /** @type {() => void} */ (resolve)
})
pendingSets.set(cacheKey, pendingSet)
try { try {
while (true) { const entry = await pendingEntry
const { done, value } = await reader.read()
if (done) break // Read the stream to get the data
chunks.push(value) const reader = entry.value.getReader()
/** @type {Uint8Array[]} */
const chunks = []
try {
while (true) {
const { done, value } = await reader.read()
if (done) break
chunks.push(value)
}
} finally {
reader.releaseLock()
} }
// Combine chunks and serialize
const value = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)))
await client.set(
entryKey(cacheKey),
JSON.stringify({
value: value.toString('base64'),
tags: entry.tags,
stale: entry.stale,
timestamp: entry.timestamp,
expire: entry.expire,
revalidate: entry.revalidate,
}),
{ expire: entry.expire }
)
} finally { } finally {
reader.releaseLock() resolvePending()
pendingSets.delete(cacheKey)
} }
// Combine chunks and serialize for Redis storage
const data = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)))
await client.set(
cacheKey,
JSON.stringify({
value: data.toString('base64'),
tags: entry.tags,
stale: entry.stale,
timestamp: entry.timestamp,
expire: entry.expire,
revalidate: entry.revalidate,
}),
{ expire: entry.expire }
)
}, },
async refreshTags() { async refreshTags() {
// No-op for basic Redis implementation // Tags are read directly from the remote store, so there is nothing to sync.
// Could sync with external tag service if needed
}, },
async getExpiration(tags) { async getExpiration(tags) {
// Return 0 to indicate no tags have been revalidated const entries = await Promise.all(tags.map(getTagManifestEntry))
// Could query Redis for tag expiration timestamps if tracking them return Math.max(...entries.map((entry) => entry.expired || 0), 0)
return 0
}, },
async updateTags(tags, durations) { async updateTags(tags, durations) {
// Implement tag-based invalidation if needed const currentTime = now()
// Could iterate over keys with matching tags and delete them
await Promise.all(
tags.map(async (tag) => {
const entry = await getTagManifestEntry(tag)
if (durations) {
entry.stale = currentTime
if (durations.expire !== undefined) {
entry.expired = currentTime + durations.expire * 1000
}
} else {
entry.expired = currentTime
}
await client.set(tagKey(tag), JSON.stringify(entry), undefined)
})
)
}, },
} }
...@@ -22,9 +22,7 @@ const cacheHandler = { ...@@ -22,9 +22,7 @@ const cacheHandler = {
}, },
async getExpiration(tags) { async getExpiration(tags) {
// Expecting soft tags in `get` to be used by the cache handler for checking return defaultCacheHandler.getExpiration(tags)
// the expiration of a cache entry, instead of letting Next.js handle it.
return Infinity
}, },
async updateTags(tags) { async updateTags(tags) {
......
...@@ -9,8 +9,8 @@ const defaultCacheHandler = ...@@ -9,8 +9,8 @@ const defaultCacheHandler =
* @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler} * @type {import('next/dist/server/lib/cache-handlers/types').CacheHandler}
*/ */
const cacheHandler = { const cacheHandler = {
async get(cacheKey) { async get(cacheKey, softTags) {
return defaultCacheHandler.get(cacheKey) return defaultCacheHandler.get(cacheKey, softTags)
}, },
async set(cacheKey, pendingEntry) { async set(cacheKey, pendingEntry) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment