Commit 7fb1093d authored by Luke Sandberg's avatar Luke Sandberg

Run the filesystem caching test with and without caching enabled.

parent a70575fc
import { nextTestSetup, isNextDev } from 'e2e-utils' import { nextTestSetup, isNextDev } from 'e2e-utils'
import { waitFor } from 'next-test-utils' import { waitFor } from 'next-test-utils'
describe('persistent-caching', () => { describe('filesystem-caching', () => {
process.env.NEXT_PUBLIC_ENV_VAR = 'hello world' process.env.NEXT_PUBLIC_ENV_VAR = 'hello world'
// Make it easier to run in development, test directories are cleared between runs already so this is safe.
process.env.TURBO_ENGINE_DISABLE_VERSIONING = '1'
// decrease the idle timeout to make the test more reliable
process.env.TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS = '1000'
const { skipped, next, isTurbopack } = nextTestSetup({ const { skipped, next, isTurbopack } = nextTestSetup({
files: __dirname, files: __dirname,
skipDeployment: true, skipDeployment: true,
...@@ -30,7 +34,7 @@ describe('persistent-caching', () => { ...@@ -30,7 +34,7 @@ describe('persistent-caching', () => {
async function stop() { async function stop() {
if (isNextDev) { if (isNextDev) {
// Give FileSystem Cache time to write to disk // Give FileSystem Cache time to write to disk
// Turbopack has an idle timeout of 2s // Turbopack is conffigured to wait 1s above.
// Webpack has an idle timeout (after large changes) of 1s // Webpack has an idle timeout (after large changes) of 1s
// and we give time a bit more to allow writing to disk // and we give time a bit more to allow writing to disk
await waitFor(3000) await waitFor(3000)
...@@ -43,6 +47,7 @@ describe('persistent-caching', () => { ...@@ -43,6 +47,7 @@ describe('persistent-caching', () => {
} }
it('should filesystem cache loaders', async () => { it('should filesystem cache loaders', async () => {
process.env.ENABLE_CACHING = '1'
let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp let appTimestamp, unchangedTimestamp, appClientTimestamp, pagesTimestamp
{ {
const browser = await next.browser('/') const browser = await next.browser('/')
...@@ -114,7 +119,13 @@ describe('persistent-caching', () => { ...@@ -114,7 +119,13 @@ describe('persistent-caching', () => {
} }
} }
const POTENTIAL_CHANGES = { interface Change {
checkInitial(): Promise<void>
withChange(previous: () => Promise<void>): Promise<void>
checkChanged(): Promise<void>
fullInvalidation?: boolean
}
const POTENTIAL_CHANGES: Record<string, Change> = {
'RSC change': { 'RSC change': {
checkInitial: makeTextCheck('/', 'hello world'), checkInitial: makeTextCheck('/', 'hello world'),
withChange: makeFileEdit('app/page.tsx'), withChange: makeFileEdit('app/page.tsx'),
...@@ -189,29 +200,31 @@ describe('persistent-caching', () => { ...@@ -189,29 +200,31 @@ describe('persistent-caching', () => {
await textCheck('/env/client', 'hello filesystem cache') await textCheck('/env/client', 'hello filesystem cache')
}, },
}, },
} } as const
const KEYS = Object.keys(POTENTIAL_CHANGES)
for (let bitset = 1; bitset < 1 << KEYS.length; bitset++) {
let combination = []
for (let i = 0; i < KEYS.length; i++) {
if (bitset & (1 << i)) {
combination.push(KEYS[i])
}
}
// Checking only single change and all combined for performance reasons. // Checking only single change and all combined for performance reasons.
if (combination.length !== 1 && combination.length !== KEYS.length) continue const combinations = Object.entries(POTENTIAL_CHANGES).map(([k, v]) => [
k,
it(`should allow to change files while stopped (${combination.join(', ')})`, async () => { [v],
]) as Array<[string, Array<Change>]>
combinations.push([
Object.keys(POTENTIAL_CHANGES).join(', '),
Object.values(POTENTIAL_CHANGES),
])
for (const cacheEnabled of [true, false]) {
process.env.ENABLE_CACHING = cacheEnabled ? '1' : '0'
describe(`with cache ${cacheEnabled ? 'enabled' : 'disabled'}`, () => {
for (const [name, changes] of combinations) {
it(`should allow to change files while stopped (${name})`, async () => {
let fullInvalidation = false let fullInvalidation = false
for (const key of combination) { for (const change of changes) {
await POTENTIAL_CHANGES[key].checkInitial() await change.checkInitial()
if (POTENTIAL_CHANGES[key].fullInvalidation) { if (change.fullInvalidation) {
fullInvalidation = true fullInvalidation = true
} }
} }
let unchangedTimestamp let unchangedTimestamp: string
if (!fullInvalidation) { if (!fullInvalidation) {
const browser = await next.browser('/unchanged') const browser = await next.browser('/unchanged')
unchangedTimestamp = await browser.elementByCss('main').text() unchangedTimestamp = await browser.elementByCss('main').text()
...@@ -220,8 +233,8 @@ describe('persistent-caching', () => { ...@@ -220,8 +233,8 @@ describe('persistent-caching', () => {
} }
async function checkChanged() { async function checkChanged() {
for (const key of combination) { for (const change of changes) {
await POTENTIAL_CHANGES[key].checkChanged() await change.checkChanged()
} }
if (!fullInvalidation) { if (!fullInvalidation) {
...@@ -246,15 +259,15 @@ describe('persistent-caching', () => { ...@@ -246,15 +259,15 @@ describe('persistent-caching', () => {
} }
let current = inner let current = inner
for (const key of combination) { for (const change of changes) {
const prev = current const prev = current
current = () => POTENTIAL_CHANGES[key].withChange(prev) current = () => change.withChange(prev)
} }
await current() await current()
await start() await start()
for (const key of combination) { for (const change of changes) {
await POTENTIAL_CHANGES[key].checkInitial() await change.checkInitial()
} }
if (!fullInvalidation) { if (!fullInvalidation) {
...@@ -265,4 +278,6 @@ describe('persistent-caching', () => { ...@@ -265,4 +278,6 @@ describe('persistent-caching', () => {
} }
}, 200000) }, 200000)
} }
})
}
}) })
const enableCaching = !!process.env.ENABLE_CACHING
/** /**
* @type {import('next').NextConfig} * @type {import('next').NextConfig}
*/ */
...@@ -16,8 +18,8 @@ const nextConfig = { ...@@ -16,8 +18,8 @@ const nextConfig = {
}, },
}, },
experimental: { experimental: {
turbopackFileSystemCacheForDev: true, turbopackFileSystemCacheForDev: enableCaching,
turbopackFileSystemCacheForBuild: true, turbopackFileSystemCacheForBuild: enableCaching,
}, },
env: { env: {
NEXT_PUBLIC_CONFIG_ENV: 'hello world', NEXT_PUBLIC_CONFIG_ENV: 'hello world',
...@@ -31,6 +33,9 @@ const nextConfig = { ...@@ -31,6 +33,9 @@ const nextConfig = {
test: /app\/loader(?:\/client)?\/page\.tsx/, test: /app\/loader(?:\/client)?\/page\.tsx/,
use: ['./my-loader.js'], use: ['./my-loader.js'],
}) })
config.cache = Object.freeze({
type: enableCaching ? 'filesystem' : 'memory',
})
if (dev) { if (dev) {
// Make webpack consider the build as large change which makes it filesystem cache it sooner // Make webpack consider the build as large change which makes it filesystem cache it sooner
config.plugins.push((compiler) => { config.plugins.push((compiler) => {
......
...@@ -12,7 +12,7 @@ use std::{ ...@@ -12,7 +12,7 @@ use std::{
ops::Range, ops::Range,
pin::Pin, pin::Pin,
sync::{ sync::{
Arc, Arc, LazyLock,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
}, },
}; };
...@@ -72,6 +72,16 @@ use crate::{ ...@@ -72,6 +72,16 @@ use crate::{
const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1); const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1);
/// Configurable idle timeout for snapshot persistence.
/// Defaults to 2 seconds if not set or if the value is invalid.
static IDLE_TIMEOUT: LazyLock<Duration> = LazyLock::new(|| {
std::env::var("TURBO_ENGINE_SNAPSHOT_IDLE_TIMEOUT_MILLIS")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_millis)
.unwrap_or(Duration::from_secs(2))
});
struct SnapshotRequest { struct SnapshotRequest {
snapshot_requested: bool, snapshot_requested: bool,
suspended_operations: FxHashSet<PtrEqArc<AnyOperation>>, suspended_operations: FxHashSet<PtrEqArc<AnyOperation>>,
...@@ -2421,8 +2431,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2421,8 +2431,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
loop { loop {
const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300); const FIRST_SNAPSHOT_WAIT: Duration = Duration::from_secs(300);
const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120); const SNAPSHOT_INTERVAL: Duration = Duration::from_secs(120);
const IDLE_TIMEOUT: Duration = Duration::from_secs(2); let idle_timeout = *IDLE_TIMEOUT;
let (time, mut reason) = let (time, mut reason) =
if matches!(job, TurboTasksBackendJob::InitialSnapshot) { if matches!(job, TurboTasksBackendJob::InitialSnapshot) {
(FIRST_SNAPSHOT_WAIT, "initial snapshot timeout") (FIRST_SNAPSHOT_WAIT, "initial snapshot timeout")
...@@ -2439,7 +2448,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2439,7 +2448,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
let mut idle_start_listener = self.idle_start_event.listen(); let mut idle_start_listener = self.idle_start_event.listen();
let mut idle_end_listener = self.idle_end_event.listen(); let mut idle_end_listener = self.idle_end_event.listen();
let mut idle_time = if turbo_tasks.is_idle() { let mut idle_time = if turbo_tasks.is_idle() {
Instant::now() + IDLE_TIMEOUT Instant::now() + idle_timeout
} else { } else {
far_future() far_future()
}; };
...@@ -2449,11 +2458,11 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2449,11 +2458,11 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
return; return;
}, },
_ = &mut idle_start_listener => { _ = &mut idle_start_listener => {
idle_time = Instant::now() + IDLE_TIMEOUT; idle_time = Instant::now() + idle_timeout;
idle_start_listener = self.idle_start_event.listen() idle_start_listener = self.idle_start_event.listen()
}, },
_ = &mut idle_end_listener => { _ = &mut idle_end_listener => {
idle_time = until + IDLE_TIMEOUT; idle_time = until + idle_timeout;
idle_end_listener = self.idle_end_event.listen() idle_end_listener = self.idle_end_event.listen()
}, },
_ = tokio::time::sleep_until(until) => { _ = tokio::time::sleep_until(until) => {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment