Commit 7fb1093d authored by Luke Sandberg's avatar Luke Sandberg

Run the filesystem caching test with and without caching enabled.

parent a70575fc
import { nextTestSetup, isNextDev } from 'e2e-utils' import { nextTestSetup, isNextDev } from 'e2e-utils'
import { waitFor } from 'next-test-utils' import { waitFor } from 'next-test-utils'
describe('persistent-caching', () => { describe('filesystem-caching', () => {
process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' process.env.NEXT_PUBLIC_ENV_VAR = 'hello world'
// Make it easier to run in development, test directories are cleared between runs already so this is safe.
process.env.TURBO_ENGINE_DISABLE_VERSIONING = '1'
// decrease the idle timeout to make the test more reliable
process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000'
const { skipped, next, isTurbopack } = nextTestSetup({ const { skipped, next, isTurbopack } = nextTestSetup({
files: __dirname, files: __dirname,
skipDeployment: true, skipDeployment: true,
...@@ -30,7 +34,7 @@ describe('persistent-caching', () => { ...@@ -30,7 +34,7 @@ describe('persistent-caching', () => {
async function stop() { async function stop() {
if (isNextDev) { if (isNextDev) {
// Give FileSystem Cache time to write to disk // Give FileSystem Cache time to write to disk
// Turbopack has an idle timeout of 2s // Turbopack is conffigured to wait 1s above.
// Webpack has an idle timeout (after large changes) of 1s // Webpack has an idle timeout (after large changes) of 1s
// and we give time a bit more to allow writing to disk // and we give time a bit more to allow writing to disk
await waitFor(3000) await waitFor(3000)
...@@ -43,6 +47,7 @@ describe('persistent-caching', () => { ...@@ -43,6 +47,7 @@ describe('persistent-caching', () => {
} }
it('should filesystem cache loaders', async () => { it('should filesystem cache loaders', async () => {
process.env.ENABLE_CACHING = '1'
let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp
{ {
const browser = await next.browser('/') const browser = await next.browser('/')
...@@ -114,7 +119,13 @@ describe('persistent-caching', () => { ...@@ -114,7 +119,13 @@ describe('persistent-caching', () => {
} }
} }
const POTENTIAL_CHANGES = { interface Change {
checkInitial(): Promise<void>
withChange(previous: () => Promise<void>): Promise<void>
checkChanged(): Promise<void>
fullInvalidation?: boolean
}
const POTENTIAL_CHANGES: Record<string, Change> = {
'RSC change': { 'RSC change': {
checkInitial: makeTextCheck('/', 'hello world'), checkInitial: makeTextCheck('/', 'hello world'),
withChange: makeFileEdit('app/page.tsx'), withChange: makeFileEdit('app/page.tsx'),
...@@ -189,80 +200,84 @@ describe('persistent-caching', () => { ...@@ -189,80 +200,84 @@ describe('persistent-caching', () => {
await textCheck('/env/client', 'hello filesystem cache') await textCheck('/env/client', 'hello filesystem cache')
}, },
}, },
} } as const
const KEYS = Object.keys(POTENTIAL_CHANGES) // Checking only single change and all combined for performance reasons.
for (let bitset = 1; bitset < 1 << KEYS.length; bitset++) { const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [
let combination = [] k,
for (let i = 0; i < KEYS.length; i++) { [v],
if (bitset & (1 << i)) { ]) as Array<[string, Array<Change>]>
combination.push(KEYS[i]) combinations.push([
} Object.keys(POTENTIAL_CHANGES).join(', '),
} Object.values(POTENTIAL_CHANGES),
// Checking only single change and all combined for performance reasons. ])
if (combination.length !== 1 && combination.length !== KEYS.length) continue for (const cacheEnabled of [true, false]) {
process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0'
describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => {
for (const [name, changes] of combinations) {
it(`should allow to change files while stopped (${name})`, async () => {
let fullInvalidation = false
for (const change of changes) {
await change.checkInitial()
if (change.fullInvalidation) {
fullInvalidation = true
}
}
it(`should allow to change files while stopped (${combination.join(', ')})`, async () => { let unchangedTimestamp: string
let fullInvalidation = false if (!fullInvalidation) {
for (const key of combination) { const browser = await next.browser('/unchanged')
await POTENTIAL_CHANGES[key].checkInitial() unchangedTimestamp = await browser.elementByCss('main').text()
if (POTENTIAL_CHANGES[key].fullInvalidation) { expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/)
fullInvalidation = true await browser.close()
} }
}
let unchangedTimestamp async function checkChanged() {
if (!fullInvalidation) { for (const change of changes) {
const browser = await next.browser('/unchanged') await change.checkChanged()
unchangedTimestamp = await browser.elementByCss('main').text() }
expect(unchangedTimestamp).toMatch(/Timestamp = \d+$/)
await browser.close()
}
async function checkChanged() { if (!fullInvalidation) {
for (const key of combination) { const browser = await next.browser('/unchanged')
await POTENTIAL_CHANGES[key].checkChanged() const timestamp = await browser.elementByCss('main').text()
} expect(unchangedTimestamp).toEqual(timestamp)
await browser.close()
}
}
if (!fullInvalidation) { await stop()
const browser = await next.browser('/unchanged')
const timestamp = await browser.elementByCss('main').text()
expect(unchangedTimestamp).toEqual(timestamp)
await browser.close()
}
}
await stop() async function inner() {
await start()
await checkChanged()
// Some no-op change builds
for (let i = 0; i < 2; i++) {
await restartCycle()
await checkChanged()
}
await stop()
}
async function inner() { let current = inner
await start() for (const change of changes) {
await checkChanged() const prev = current
// Some no-op change builds current = () => change.withChange(prev)
for (let i = 0; i < 2; i++) { }
await restartCycle() await current()
await checkChanged()
}
await stop()
}
let current = inner await start()
for (const key of combination) { for (const change of changes) {
const prev = current await change.checkInitial()
current = () => POTENTIAL_CHANGES[key].withChange(prev) }
}
await current()
await start()
for (const key of combination) {
await POTENTIAL_CHANGES[key].checkInitial()
}
if (!fullInvalidation) { if (!fullInvalidation) {
const browser = await next.browser('/unchanged') const browser = await next.browser('/unchanged')
const timestamp = await browser.elementByCss('main').text() const timestamp = await browser.elementByCss('main').text()
expect(unchangedTimestamp).toEqual(timestamp) expect(unchangedTimestamp).toEqual(timestamp)
await browser.close() await browser.close()
}
}, 200000)
} }
}, 200000) })
} }
}) })
const enableCaching = !!process.env.ENABLE_CACHING
/** /**
* @type {import('next').NextConfig} * @type {import('next').NextConfig}
*/ */
...@@ -16,8 +18,8 @@ const nextConfig = { ...@@ -16,8 +18,8 @@ const nextConfig = {
}, },
}, },
experimental: { experimental: {
turbopackFileSystemCacheForDev: true, turbopackFileSystemCacheForDev: enableCaching,
turbopackFileSystemCacheForBuild: true, turbopackFileSystemCacheForBuild: enableCaching,
}, },
env: { env: {
NEXT_PUBLIC_CONFIG_ENV: 'hello world', NEXT_PUBLIC_CONFIG_ENV: 'hello world',
...@@ -31,6 +33,9 @@ const nextConfig = { ...@@ -31,6 +33,9 @@ const nextConfig = {
test: /app\/loader(?:\/client)?\/page\.tsx/, test: /app\/loader(?:\/client)?\/page\.tsx/,
use: ['./my-loader.js'], use: ['./my-loader.js'],
}) })
config.cache = Object.freeze({
type: enableCaching ? 'filesystem' : 'memory',
})
if (dev) { if (dev) {
// Make webpack consider the build as large change which makes it filesystem cache it sooner // Make webpack consider the build as large change which makes it filesystem cache it sooner
config.plugins.push((compiler) => { config.plugins.push((compiler) => {
......
...@@ -12,7 +12,7 @@ use std::{ ...@@ -12,7 +12,7 @@ use std::{
ops::Range, ops::Range,
pin::Pin, pin::Pin,
sync::{ sync::{
Arc, Arc, LazyLock,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
}, },
}; };
...@@ -72,6 +72,16 @@ use crate::{ ...@@ -72,6 +72,16 @@ use crate::{
const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1); const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1);
/// Configurable idle timeout for snapshot persistence.
/// Defaults to 2 seconds if not set or if the value is invalid.
static IDLE_TIMEOUT: LazyLock<Duration> = LazyLock::new(|| {
std::env::var("TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_millis)
.unwrap_or(Duration::from_secs(2))
});
struct SnapshotRequest { struct SnapshotRequest {
snapshot_requested: bool, snapshot_requested: bool,
suspended_operations: FxHashSet<PtrEqArc<AnyOperation>>, suspended_operations: FxHashSet<PtrEqArc<AnyOperation>>,
...@@ -2421,8 +2431,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2421,8 +2431,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
loop { loop {
const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300); const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300);
const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120); const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120);
const IDLE_TIMEOUT: Duration = Duration::from_secs(2); let idle_timeout = *IDLE_TIMEOUT;
let (time, mut reason) = let (time, mut reason) =
if matches!(job, TurboTasksBackendJob::InitialSnapshot) { if matches!(job, TurboTasksBackendJob::InitialSnapshot) {
(FIRST_SNAPSHOT_WAIT, "initial snapshot timeout") (FIRST_SNAPSHOT_WAIT, "initial snapshot timeout")
...@@ -2439,7 +2448,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2439,7 +2448,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let mut idle_start_listener = self.idle_start_event.listen(); let mut idle_start_listener = self.idle_start_event.listen();
let mut idle_end_listener = self.idle_end_event.listen(); let mut idle_end_listener = self.idle_end_event.listen();
let mut idle_time = if turbo_tasks.is_idle() { let mut idle_time = if turbo_tasks.is_idle() {
Instant::now() + IDLE_TIMEOUT Instant::now() + idle_timeout
} else { } else {
far_future() far_future()
}; };
...@@ -2449,11 +2458,11 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2449,11 +2458,11 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
return; return;
}, },
_ = &mut idle_start_listener => { _ = &mut idle_start_listener => {
idle_time = Instant::now() + IDLE_TIMEOUT; idle_time = Instant::now() + idle_timeout;
idle_start_listener = self.idle_start_event.listen() idle_start_listener = self.idle_start_event.listen()
}, },
_ = &mut idle_end_listener => { _ = &mut idle_end_listener => {
idle_time = until + IDLE_TIMEOUT; idle_time = until + idle_timeout;
idle_end_listener = self.idle_end_event.listen() idle_end_listener = self.idle_end_event.listen()
}, },
_ = tokio::time::sleep_until(until) => { _ = tokio::time::sleep_until(until) => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment