Unverified Commit 6bdd0475 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Merge branch 'canary' into sokra/flaky-scroll-test

parents 07636dfc f7e66622
...@@ -58,10 +58,7 @@ impl NextMode { ...@@ -58,10 +58,7 @@ impl NextMode {
} }
pub fn is_production(&self) -> bool { pub fn is_production(&self) -> bool {
match self { !self.is_development()
NextMode::Development => false,
NextMode::Build => true,
}
} }
pub fn runtime_type(&self) -> RuntimeType { pub fn runtime_type(&self) -> RuntimeType {
......
...@@ -19,8 +19,8 @@ use turbopack_browser::{ ...@@ -19,8 +19,8 @@ use turbopack_browser::{
}; };
use turbopack_core::{ use turbopack_core::{
chunk::{ chunk::{
ChunkingConfig, ChunkingContext, MangleType, MinifyType, SourceMapsType, ChunkingConfig, ChunkingContext, MangleType, MinifyType, SourceMapSourceType,
module_id_strategies::ModuleIdStrategy, SourceMapsType, module_id_strategies::ModuleIdStrategy,
}, },
compile_time_info::{CompileTimeDefines, CompileTimeInfo, FreeVarReference, FreeVarReferences}, compile_time_info::{CompileTimeDefines, CompileTimeInfo, FreeVarReference, FreeVarReferences},
environment::{BrowserEnvironment, Environment, ExecutionEnvironment}, environment::{BrowserEnvironment, Environment, ExecutionEnvironment},
...@@ -489,7 +489,7 @@ pub async fn get_client_chunking_context( ...@@ -489,7 +489,7 @@ pub async fn get_client_chunking_context(
if next_mode.is_development() { if next_mode.is_development() {
builder = builder builder = builder
.hot_module_replacement() .hot_module_replacement()
.use_file_source_map_uris() .source_map_source_type(SourceMapSourceType::AbsoluteFileUri)
.dynamic_chunk_content_loading(true); .dynamic_chunk_content_loading(true);
} else { } else {
builder = builder builder = builder
......
...@@ -16,7 +16,7 @@ use turbopack::{ ...@@ -16,7 +16,7 @@ use turbopack::{
}; };
use turbopack_core::{ use turbopack_core::{
chunk::{ chunk::{
ChunkingConfig, MangleType, MinifyType, SourceMapsType, ChunkingConfig, MangleType, MinifyType, SourceMapSourceType, SourceMapsType,
module_id_strategies::ModuleIdStrategy, module_id_strategies::ModuleIdStrategy,
}, },
compile_time_defines, compile_time_defines,
...@@ -1060,9 +1060,13 @@ pub async fn get_server_chunking_context_with_client_assets( ...@@ -1060,9 +1060,13 @@ pub async fn get_server_chunking_context_with_client_assets(
.file_tracing(next_mode.is_production()) .file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?); .debug_ids(*debug_ids.await?);
if next_mode.is_development() { builder = builder.source_map_source_type(if next_mode.is_development() {
builder = builder.use_file_source_map_uris(); SourceMapSourceType::AbsoluteFileUri
} else { } else {
// TODO(lukesandberg): switch to relative once next is compatible.
SourceMapSourceType::TurbopackUri
});
if next_mode.is_production() {
builder = builder builder = builder
.chunking_config( .chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?, Vc::<EcmascriptChunkType>::default().to_resolved().await?,
...@@ -1142,9 +1146,11 @@ pub async fn get_server_chunking_context( ...@@ -1142,9 +1146,11 @@ pub async fn get_server_chunking_context(
.debug_ids(*debug_ids.await?); .debug_ids(*debug_ids.await?);
if next_mode.is_development() { if next_mode.is_development() {
builder = builder.use_file_source_map_uris() builder = builder.source_map_source_type(SourceMapSourceType::AbsoluteFileUri);
} else { } else {
builder = builder builder = builder
// TODO(lukesandberg): switch to relative once next is compatible.
.source_map_source_type(SourceMapSourceType::TurbopackUri)
.chunking_config( .chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?, Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig { ChunkingConfig {
......
...@@ -112,7 +112,10 @@ export default async function getChangedTests() { ...@@ -112,7 +112,10 @@ export default async function getChangedTests() {
.catch(() => false) .catch(() => false)
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) { if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
if (file.startsWith('test/e2e/')) { if (
file.startsWith('test/e2e/') ||
file.startsWith('test/integration/')
) {
devTests.push(file) devTests.push(file)
prodTests.push(file) prodTests.push(file)
} else if (file.startsWith('test/prod')) { } else if (file.startsWith('test/prod')) {
......
...@@ -58,24 +58,27 @@ describe('nextjs APIs in after()', () => { ...@@ -58,24 +58,27 @@ describe('nextjs APIs in after()', () => {
`[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.` `[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] nested headers(): ok`) expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.` `[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] cookies(): ok`) expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.` `[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
) )
})
await retry(() => {
const logs = getLogs()
expect(logs).not.toContain(`[${path}] nested cookies(): ok`) expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.` `[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] connection(): ok`) expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.` `[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] nested connection(): ok`) expect(logs).not.toContain(`[${path}] nested connection(): ok`)
...@@ -105,24 +108,27 @@ describe('nextjs APIs in after()', () => { ...@@ -105,24 +108,27 @@ describe('nextjs APIs in after()', () => {
`[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.` `[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] nested headers(): ok`) expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.` `[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] cookies(): ok`) expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.` `[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
) )
})
await retry(() => {
const logs = isNextDev ? getLogs() : buildLogs // in `next start` the error was logged at build time
expect(logs).not.toContain(`[${path}] nested cookies(): ok`) expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.` `[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] connection(): ok`) expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).toContain( expect(logs).toContain(
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.` `[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
) )
expect(logs).not.toContain(`[${path}] nested connection(): ok`) expect(logs).not.toContain(`[${path}] nested connection(): ok`)
......
...@@ -18,11 +18,7 @@ describe('New Link Behavior with material-ui', () => { ...@@ -18,11 +18,7 @@ describe('New Link Behavior with material-ui', () => {
'@emotion/styled': 'latest', '@emotion/styled': 'latest',
'@mui/icons-material': 'latest', '@mui/icons-material': 'latest',
'@mui/material': 'latest', '@mui/material': 'latest',
next: 'latest',
'prop-types': 'latest', 'prop-types': 'latest',
// Use minimum peer dep version instead of v9 of eslint to avoid breaking changes
eslint: '8.56.0',
'eslint-config-next': 'latest',
}, },
}) })
......
...@@ -8,7 +8,6 @@ import { ...@@ -8,7 +8,6 @@ import {
launchApp, launchApp,
nextStart, nextStart,
nextBuild, nextBuild,
check,
retry, retry,
} from 'next-test-utils' } from 'next-test-utils'
...@@ -39,17 +38,15 @@ const runTests = () => { ...@@ -39,17 +38,15 @@ const runTests = () => {
await browser.eval(() => window.next.router.push('/another')) await browser.eval(() => window.next.router.push('/another'))
await check( await retry(async () => {
() => browser.eval(() => document.documentElement.innerHTML), const html = await browser.eval(() => document.documentElement.innerHTML)
/hi from another/ expect(html).toMatch(/hi from another/)
) })
await browser.eval(() => (window.didHydrate = false)) await browser.eval(() => (window.didHydrate = false))
await browser.eval(() => window.history.back()) await browser.eval(() => window.history.back())
await check(() => browser.eval(() => window.didHydrate), { await retry(async () => {
test(content) { expect(await browser.eval(() => window.didHydrate)).toBe(true)
return content
},
}) })
await retry(async () => { await retry(async () => {
......
...@@ -19,6 +19,7 @@ no_fast_stale = [] ...@@ -19,6 +19,7 @@ no_fast_stale = []
verify_serialization = [] verify_serialization = []
verify_aggregation_graph = [] verify_aggregation_graph = []
verify_immutable = [] verify_immutable = []
verify_determinism = []
trace_aggregation_update = [] trace_aggregation_update = []
trace_find_and_schedule = [] trace_find_and_schedule = []
trace_task_completion = [] trace_task_completion = []
......
...@@ -31,7 +31,7 @@ use turbo_tasks::{ ...@@ -31,7 +31,7 @@ use turbo_tasks::{
TraitTypeId, TurboTasksBackendApi, ValueTypeId, TraitTypeId, TurboTasksBackendApi, ValueTypeId,
backend::{ backend::{
Backend, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskRoot, Backend, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskRoot,
TransientTaskType, TurboTasksExecutionError, TypedCellContent, TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode,
}, },
event::{Event, EventListener}, event::{Event, EventListener},
message_queue::TimingEvent, message_queue::TimingEvent,
...@@ -423,6 +423,8 @@ struct TaskExecutionCompletePrepareResult { ...@@ -423,6 +423,8 @@ struct TaskExecutionCompletePrepareResult {
pub new_children: FxHashSet<TaskId>, pub new_children: FxHashSet<TaskId>,
pub removed_data: Vec<CachedDataItem>, pub removed_data: Vec<CachedDataItem>,
pub is_now_immutable: bool, pub is_now_immutable: bool,
#[cfg(feature = "verify_determinism")]
pub no_output_set: bool,
pub new_output: Option<OutputValue>, pub new_output: Option<OutputValue>,
pub output_dependent_tasks: SmallVec<[TaskId; 4]>, pub output_dependent_tasks: SmallVec<[TaskId; 4]>,
} }
...@@ -1765,6 +1767,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1765,6 +1767,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children, new_children,
mut removed_data, mut removed_data,
is_now_immutable, is_now_immutable,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output, new_output,
output_dependent_tasks, output_dependent_tasks,
}) = self.task_execution_completed_prepare( }) = self.task_execution_completed_prepare(
...@@ -1809,6 +1813,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1809,6 +1813,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
if self.task_execution_completed_finish( if self.task_execution_completed_finish(
&mut ctx, &mut ctx,
task_id, task_id,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output, new_output,
&mut removed_data, &mut removed_data,
is_now_immutable, is_now_immutable,
...@@ -1843,6 +1849,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -1843,6 +1849,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children: Default::default(), new_children: Default::default(),
removed_data: Default::default(), removed_data: Default::default(),
is_now_immutable: false, is_now_immutable: false,
#[cfg(feature = "verify_determinism")]
no_output_set: false,
new_output: None, new_output: None,
output_dependent_tasks: Default::default(), output_dependent_tasks: Default::default(),
}); });
...@@ -2021,6 +2029,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2021,6 +2029,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// Check if output need to be updated // Check if output need to be updated
let current_output = get!(task, Output); let current_output = get!(task, Output);
#[cfg(feature = "verify_determinism")]
let no_output_set = current_output.is_none();
let new_output = match result { let new_output = match result {
Ok(RawVc::TaskOutput(output_task_id)) => { Ok(RawVc::TaskOutput(output_task_id)) => {
if let Some(OutputValue::Output(current_task_id)) = current_output if let Some(OutputValue::Output(current_task_id)) = current_output
...@@ -2092,6 +2102,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2092,6 +2102,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children, new_children,
removed_data, removed_data,
is_now_immutable, is_now_immutable,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output, new_output,
output_dependent_tasks, output_dependent_tasks,
}) })
...@@ -2232,6 +2244,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2232,6 +2244,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
&self, &self,
ctx: &mut impl ExecuteContext<'_>, ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId, task_id: TaskId,
#[cfg(feature = "verify_determinism")] no_output_set: bool,
new_output: Option<OutputValue>, new_output: Option<OutputValue>,
removed_data: &mut Vec<CachedDataItem>, removed_data: &mut Vec<CachedDataItem>,
is_now_immutable: bool, is_now_immutable: bool,
...@@ -2306,7 +2319,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2306,7 +2319,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
None None
}; };
let data_update = if old_dirty_state != new_dirty_state { let dirty_changed = old_dirty_state != new_dirty_state;
let data_update = if dirty_changed {
if let Some(new_dirty_state) = new_dirty_state { if let Some(new_dirty_state) = new_dirty_state {
task.insert(CachedDataItem::Dirty { task.insert(CachedDataItem::Dirty {
value: new_dirty_state, value: new_dirty_state,
...@@ -2353,17 +2367,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2353,17 +2367,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
None None
}; };
drop(task); #[cfg(feature = "verify_determinism")]
drop(old_content); let reschedule = (dirty_changed || no_output_set) && !task_id.is_transient();
#[cfg(not(feature = "verify_determinism"))]
let reschedule = false;
if reschedule {
task.add_new(CachedDataItem::InProgress {
value: InProgressState::Scheduled {
done_event,
reason: TaskExecutionReason::Stale,
},
});
drop(task);
} else {
drop(task);
// Notify dependent tasks that are waiting for this task to finish
done_event.notify(usize::MAX);
}
// Notify dependent tasks that are waiting for this task to finish drop(old_content);
done_event.notify(usize::MAX);
if let Some(data_update) = data_update { if let Some(data_update) = data_update {
AggregationUpdateQueue::run(data_update, ctx); AggregationUpdateQueue::run(data_update, ctx);
} }
false reschedule
} }
fn task_execution_completed_cleanup(&self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) { fn task_execution_completed_cleanup(&self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) {
...@@ -2652,12 +2681,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> { ...@@ -2652,12 +2681,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
task_id: TaskId, task_id: TaskId,
cell: CellId, cell: CellId,
content: CellContent, content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>, turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) { ) {
operation::UpdateCellOperation::run( operation::UpdateCellOperation::run(
task_id, task_id,
cell, cell,
content, content,
verification_mode,
self.execute_context(turbo_tasks), self.execute_context(turbo_tasks),
); );
} }
...@@ -3251,9 +3282,11 @@ impl<B: BackingStorage> Backend for TurboTasksBackend<B> { ...@@ -3251,9 +3282,11 @@ impl<B: BackingStorage> Backend for TurboTasksBackend<B> {
task_id: TaskId, task_id: TaskId,
cell: CellId, cell: CellId,
content: CellContent, content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<Self>, turbo_tasks: &dyn TurboTasksBackendApi<Self>,
) { ) {
self.0.update_task_cell(task_id, cell, content, turbo_tasks); self.0
.update_task_cell(task_id, cell, content, verification_mode, turbo_tasks);
} }
fn mark_own_task_as_finished( fn mark_own_task_as_finished(
......
...@@ -382,6 +382,7 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> { ...@@ -382,6 +382,7 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> {
/// Verify that the task guard restored the correct category /// Verify that the task guard restored the correct category
/// before accessing the data. /// before accessing the data.
#[inline] #[inline]
#[track_caller]
fn check_access(&self, category: TaskDataCategory) { fn check_access(&self, category: TaskDataCategory) {
{ {
match category { match category {
...@@ -434,6 +435,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -434,6 +435,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task_id self.task_id
} }
#[track_caller]
fn add(&mut self, item: CachedDataItem) -> bool { fn add(&mut self, item: CachedDataItem) -> bool {
let category = item.category(); let category = item.category();
self.check_access(category); self.check_access(category);
...@@ -446,12 +448,14 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -446,12 +448,14 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.add(item) self.task.add(item)
} }
#[track_caller]
fn add_new(&mut self, item: CachedDataItem) { fn add_new(&mut self, item: CachedDataItem) {
self.check_access(item.category()); self.check_access(item.category());
let added = self.add(item); let added = self.add(item);
assert!(added, "Item already exists"); assert!(added, "Item already exists");
} }
#[track_caller]
fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> { fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
let category = item.category(); let category = item.category();
self.check_access(category); self.check_access(category);
...@@ -461,6 +465,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -461,6 +465,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.insert(item) self.task.insert(item)
} }
#[track_caller]
fn update( fn update(
&mut self, &mut self,
key: CachedDataItemKey, key: CachedDataItemKey,
...@@ -474,6 +479,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -474,6 +479,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.update(key, update); self.task.update(key, update);
} }
#[track_caller]
fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> { fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
let category = key.category(); let category = key.category();
self.check_access(category); self.check_access(category);
...@@ -488,6 +494,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -488,6 +494,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get(key) self.task.get(key)
} }
#[track_caller]
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> { fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
let category = key.category(); let category = key.category();
self.check_access(category); self.check_access(category);
...@@ -497,6 +504,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -497,6 +504,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get_mut(key) self.task.get_mut(key)
} }
#[track_caller]
fn get_mut_or_insert_with( fn get_mut_or_insert_with(
&mut self, &mut self,
key: CachedDataItemKey, key: CachedDataItemKey,
...@@ -510,11 +518,13 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -510,11 +518,13 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get_mut_or_insert_with(key, insert) self.task.get_mut_or_insert_with(key, insert)
} }
#[track_caller]
fn has_key(&self, key: &CachedDataItemKey) -> bool { fn has_key(&self, key: &CachedDataItemKey) -> bool {
self.check_access(key.category()); self.check_access(key.category());
self.task.contains_key(key) self.task.contains_key(key)
} }
#[track_caller]
fn count(&self, ty: CachedDataItemType) -> usize { fn count(&self, ty: CachedDataItemType) -> usize {
self.check_access(ty.category()); self.check_access(ty.category());
self.task.count(ty) self.task.count(ty)
...@@ -532,6 +542,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> { ...@@ -532,6 +542,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.shrink_to_fit(ty) self.task.shrink_to_fit(ty)
} }
#[track_caller]
fn extract_if<'l, F>( fn extract_if<'l, F>(
&'l mut self, &'l mut self,
ty: CachedDataItemType, ty: CachedDataItemType,
......
...@@ -2,6 +2,8 @@ use std::mem::take; ...@@ -2,6 +2,8 @@ use std::mem::take;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use smallvec::SmallVec; use smallvec::SmallVec;
#[cfg(not(feature = "verify_determinism"))]
use turbo_tasks::backend::VerificationMode;
use turbo_tasks::{CellId, TaskId, TypedSharedReference, backend::CellContent}; use turbo_tasks::{CellId, TaskId, TypedSharedReference, backend::CellContent};
#[cfg(feature = "trace_task_dirty")] #[cfg(feature = "trace_task_dirty")]
...@@ -13,7 +15,7 @@ use crate::{ ...@@ -13,7 +15,7 @@ use crate::{
AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard, AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard,
invalidate::make_task_dirty_internal, invalidate::make_task_dirty_internal,
}, },
storage::{get_many, remove}, storage::{get, get_many, remove},
}, },
data::{CachedDataItem, CachedDataItemKey, CellRef}, data::{CachedDataItem, CachedDataItemKey, CellRef},
}; };
...@@ -40,18 +42,61 @@ pub enum UpdateCellOperation { ...@@ -40,18 +42,61 @@ pub enum UpdateCellOperation {
} }
impl UpdateCellOperation { impl UpdateCellOperation {
pub fn run(task_id: TaskId, cell: CellId, content: CellContent, mut ctx: impl ExecuteContext) { pub fn run(
task_id: TaskId,
cell: CellId,
content: CellContent,
#[cfg(feature = "verify_determinism")] verification_mode: VerificationMode,
#[cfg(not(feature = "verify_determinism"))] _verification_mode: VerificationMode,
mut ctx: impl ExecuteContext,
) {
let content = if let CellContent(Some(new_content)) = content {
Some(new_content.into_typed(cell.type_id))
} else {
None
};
let mut task = ctx.task(task_id, TaskDataCategory::All); let mut task = ctx.task(task_id, TaskDataCategory::All);
let is_stateful = task.has_key(&CachedDataItemKey::Stateful {});
// We need to detect recomputation, because here the content has not actually changed (even // We need to detect recomputation, because here the content has not actually changed (even
// if it's not equal to the old content, as not all values implement Eq). We have to // if it's not equal to the old content, as not all values implement Eq). We have to
// assume that tasks are deterministic and pure. // assume that tasks are deterministic and pure.
let should_invalidate = ctx.should_track_dependencies() let assume_unchanged = !ctx.should_track_dependencies()
&& (task.has_key(&CachedDataItemKey::Dirty {}) || || (!task.has_key(&CachedDataItemKey::Dirty {})
// This is a hack for the streaming hack. Stateful tasks are never recomputed, so this forces invalidation for them in case of this hack. // This is a hack for the streaming hack. Stateful tasks are never recomputed, so this forces invalidation for them in case of this hack.
task.has_key(&CachedDataItemKey::Stateful {})); && !is_stateful);
let old_content = get!(task, CellData { cell });
if assume_unchanged {
if old_content.is_some() {
// Never update cells when recomputing if they already have a value.
// It's not expected that content changes during recomputation.
// Check if this assumption holds.
#[cfg(feature = "verify_determinism")]
if !is_stateful
&& matches!(verification_mode, VerificationMode::EqualityCheck)
&& content.as_ref() != old_content
{
let task_description = ctx.get_task_description(task_id);
let cell_type = turbo_tasks::registry::get_value_type(cell.type_id).global_name;
eprintln!(
"Task {} updated cell #{} (type: {}) while recomputing",
task_description, cell.index, cell_type
);
}
return;
} else {
// Initial computation, or computation after a cell has been cleared.
// We can just set the content, but we don't want to notify dependent tasks,
// as we assume that content hasn't changed (deterministic tasks).
}
} else {
// When not recomputing, we need to notify dependent tasks if the content actually
// changes.
if should_invalidate {
let dependent_tasks: SmallVec<[TaskId; 4]> = get_many!( let dependent_tasks: SmallVec<[TaskId; 4]> = get_many!(
task, task,
CellDependent { cell: dependent_cell, task } CellDependent { cell: dependent_cell, task }
...@@ -78,12 +123,6 @@ impl UpdateCellOperation { ...@@ -78,12 +123,6 @@ impl UpdateCellOperation {
drop(task); drop(task);
drop(old_content); drop(old_content);
let content = if let CellContent(Some(new_content)) = content {
Some(new_content.into_typed(cell.type_id))
} else {
None
};
UpdateCellOperation::InvalidateWhenCellDependency { UpdateCellOperation::InvalidateWhenCellDependency {
cell_ref: CellRef { cell_ref: CellRef {
task: task_id, task: task_id,
...@@ -101,8 +140,7 @@ impl UpdateCellOperation { ...@@ -101,8 +140,7 @@ impl UpdateCellOperation {
// Fast path: We don't need to invalidate anything. // Fast path: We don't need to invalidate anything.
// So we can just update the cell content. // So we can just update the cell content.
let old_content = if let CellContent(Some(new_content)) = content { let old_content = if let Some(new_content) = content {
let new_content = new_content.into_typed(cell.type_id);
task.insert(CachedDataItem::CellData { task.insert(CachedDataItem::CellData {
cell, cell,
value: new_content, value: new_content,
......
...@@ -679,9 +679,53 @@ fn serialize_task_type( ...@@ -679,9 +679,53 @@ fn serialize_task_type(
let deserialize: Result<CachedTaskType, _> = serde_path_to_error::deserialize( let deserialize: Result<CachedTaskType, _> = serde_path_to_error::deserialize(
&mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?, &mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?,
); );
if let Err(err) = deserialize { match deserialize {
println!("Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}"); Err(err) => {
panic!("Task type would not be deserializable {task_id:?}: {err:?}"); println!(
"Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}"
);
panic!("Task type would not be deserializable {task_id:?}: {err:?}");
}
Ok(task_type2) => {
if &task_type2 != task_type {
println!(
"Task type would not round-trip {task_id:?}:\noriginal: \
{task_type:#?}\nround-tripped: {task_type2:#?}"
);
panic!(
"Task type would not round-trip {task_id:?}:\noriginal: \
{task_type:#?}\nround-tripped: {task_type2:#?}"
);
}
let mut bytes2 = Vec::new();
let result2 = POT_CONFIG.serialize_into(&task_type2, &mut bytes2);
match result2 {
Err(err) => {
println!(
"Task type would not be serializable the second time {task_id:?}: \
{err:?}\n{task_type2:#?}"
);
panic!(
"Task type would not be serializable the second time {task_id:?}: \
{err:?}\n{task_type2:#?}"
);
}
Ok(()) => {
if bytes2 != *task_type_bytes {
println!(
"Task type would not serialize to the same bytes the second time \
{task_id:?}:\noriginal: {:x?}\nsecond: {:x?}\n{task_type2:#?}",
task_type_bytes, bytes2
);
panic!(
"Task type would not serialize to the same bytes the second time \
{task_id:?}:\noriginal: {:x?}\nsecond: {:x?}\n{task_type2:#?}",
task_type_bytes, bytes2
);
}
}
}
}
} }
} }
Ok(()) Ok(())
...@@ -757,7 +801,10 @@ fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 1 ...@@ -757,7 +801,10 @@ fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 1
if let Err(err) = serde_path_to_error::serialize(&item, &mut serializer) { if let Err(err) = serde_path_to_error::serialize(&item, &mut serializer) {
if item.is_optional() { if item.is_optional() {
#[cfg(feature = "verify_serialization")] #[cfg(feature = "verify_serialization")]
println!("Skipping non-serializable optional item for {task}: {item:?}"); println!(
"Skipping non-serializable optional item for {task}: {item:?} due to \
{err}"
);
} else { } else {
error = Err(err).context({ error = Err(err).context({
anyhow!("Unable to serialize data item for {task}: {item:?}") anyhow!("Unable to serialize data item for {task}: {item:?}")
......
...@@ -18,6 +18,7 @@ static REGISTRATION: Registration = register!(); ...@@ -18,6 +18,7 @@ static REGISTRATION: Registration = register!();
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_spawns_detached() -> anyhow::Result<()> { async fn test_spawns_detached() -> anyhow::Result<()> {
run_once(&REGISTRATION, || async { run_once(&REGISTRATION, || async {
println!("test_spawns_detached");
// HACK: The watch channel we use has an incorrect implementation of `TraceRawVcs`, just // HACK: The watch channel we use has an incorrect implementation of `TraceRawVcs`, just
// disable GC for the test so this can't cause any problems. // disable GC for the test so this can't cause any problems.
prevent_gc(); prevent_gc();
...@@ -73,7 +74,9 @@ async fn spawns_detached( ...@@ -73,7 +74,9 @@ async fn spawns_detached(
sender: TransientInstance<WatchSenderTaskInput<Option<Vc<u32>>>>, sender: TransientInstance<WatchSenderTaskInput<Option<Vc<u32>>>>,
) -> Vc<()> { ) -> Vc<()> {
tokio::spawn(turbo_tasks().detached_for_testing(Box::pin(async move { tokio::spawn(turbo_tasks().detached_for_testing(Box::pin(async move {
println!("spawns_detached: waiting for notify");
notify.0.notified().await; notify.0.notified().await;
println!("spawns_detached: notified, sending value");
// creating cells after the normal lifetime of the task should be okay, as the parent task // creating cells after the normal lifetime of the task should be okay, as the parent task
// is waiting on us before exiting! // is waiting on us before exiting!
sender.0.send(Some(Vc::cell(42))).unwrap(); sender.0.send(Some(Vc::cell(42))).unwrap();
......
use std::{ use std::{
io::{self, ErrorKind}, io::{self, ErrorKind},
path::Path, path::{Path, PathBuf},
}; };
use anyhow::{Context, Result, anyhow}; use anyhow::{Context, Result, anyhow};
...@@ -19,10 +19,7 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio ...@@ -19,10 +19,7 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio
} }
} }
#[cfg(not(target_os = "windows"))]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> { pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
use turbo_unix_path::sys_to_unix;
let root_fs = root.fs; let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs) let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")? .context("Expected root to have a DiskFileSystem")?
...@@ -33,32 +30,26 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S ...@@ -33,32 +30,26 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
None => root, None => root,
}; };
let sys_path = root_fs.to_sys_path(&path); Ok(uri_from_path_buf(root_fs.to_sys_path(&path)))
}
#[cfg(not(target_os = "windows"))]
pub fn uri_from_path_buf(sys_path: PathBuf) -> String {
use turbo_unix_path::sys_to_unix;
let sys_path = sys_path.to_string_lossy(); let sys_path = sys_path.to_string_lossy();
Ok(format!( format!(
"file://{}", "file://{}",
sys_to_unix(&sys_path) sys_to_unix(&sys_path)
.split('/') .split('/')
.map(|s| urlencoding::encode(s)) .map(|s| urlencoding::encode(s))
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("/") .join("/")
)) )
} }
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> { pub fn uri_from_path_buf(sys_path: PathBuf) -> String {
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
let sys_path = match path {
Some(path) => root.join(path.into())?,
None => root,
};
let sys_path = root_fs.to_sys_path(&sys_path);
let raw_path = sys_path.to_string_lossy().to_string(); let raw_path = sys_path.to_string_lossy().to_string();
let normalized_path = raw_path.replace('\\', "/"); let normalized_path = raw_path.replace('\\', "/");
...@@ -70,7 +61,5 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S ...@@ -70,7 +61,5 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("/"); .join("/");
let uri = format!("file:///{}", encoded_path); format!("file:///{}", encoded_path)
Ok(uri)
} }
...@@ -18,7 +18,7 @@ use tokio::sync::mpsc::Receiver; ...@@ -18,7 +18,7 @@ use tokio::sync::mpsc::Receiver;
use turbo_tasks::{ use turbo_tasks::{
CellId, ExecutionId, InvalidationReason, LocalTaskId, MagicAny, RawVc, ReadCellOptions, CellId, ExecutionId, InvalidationReason, LocalTaskId, MagicAny, RawVc, ReadCellOptions,
ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId, TurboTasksApi, TurboTasksCallApi, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId, TurboTasksApi, TurboTasksCallApi,
backend::{CellContent, TaskCollectiblesMap, TypedCellContent}, backend::{CellContent, TaskCollectiblesMap, TypedCellContent, VerificationMode},
event::{Event, EventListener}, event::{Event, EventListener},
message_queue::CompilationEvent, message_queue::CompilationEvent,
test_helpers::with_turbo_tasks_for_testing, test_helpers::with_turbo_tasks_for_testing,
...@@ -261,7 +261,13 @@ impl TurboTasksApi for VcStorage { ...@@ -261,7 +261,13 @@ impl TurboTasksApi for VcStorage {
.into_typed(index.type_id)) .into_typed(index.type_id))
} }
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent) { fn update_own_task_cell(
&self,
task: TaskId,
index: CellId,
content: CellContent,
_verification_mode: VerificationMode,
) {
let mut map = self.cells.lock().unwrap(); let mut map = self.cells.lock().unwrap();
let cell = map.entry((task, index)).or_default(); let cell = map.entry((task, index)).or_default();
*cell = content; *cell = content;
......
use std::{fmt::Debug, future::Future, sync::Arc}; use std::{env, fmt::Debug, future::Future, sync::Arc};
use anyhow::Result; use anyhow::Result;
use turbo_tasks::{TurboTasksApi, trace::TraceRawVcs}; use turbo_tasks::{TurboTasksApi, trace::TraceRawVcs};
...@@ -98,22 +98,28 @@ where ...@@ -98,22 +98,28 @@ where
F: Future<Output = Result<T>> + Send + 'static, F: Future<Output = Result<T>> + Send + 'static,
T: Debug + PartialEq + Eq + TraceRawVcs + Send + 'static, T: Debug + PartialEq + Eq + TraceRawVcs + Send + 'static,
{ {
let single_run = env::var("SINGLE_RUN").is_ok();
let name = closure_to_name(&fut); let name = closure_to_name(&fut);
let tt = registration.create_turbo_tasks(&name, true); let tt = registration.create_turbo_tasks(&name, true);
println!("Run #1 (without cache)"); println!("Run #1 (without cache)");
let start = std::time::Instant::now(); let start = std::time::Instant::now();
let first = fut(tt.clone()).await?; let first = fut(tt.clone()).await?;
println!("Run #1 took {:?}", start.elapsed()); println!("Run #1 took {:?}", start.elapsed());
for i in 2..10 { if !single_run {
println!("Run #{i} (with memory cache, same TurboTasks instance)"); for i in 2..10 {
let start = std::time::Instant::now(); println!("Run #{i} (with memory cache, same TurboTasks instance)");
let second = fut(tt.clone()).await?; let start = std::time::Instant::now();
println!("Run #{i} took {:?}", start.elapsed()); let second = fut(tt.clone()).await?;
assert_eq!(first, second); println!("Run #{i} took {:?}", start.elapsed());
assert_eq!(first, second);
}
} }
let start = std::time::Instant::now(); let start = std::time::Instant::now();
tt.stop_and_wait().await; tt.stop_and_wait().await;
println!("Stopping TurboTasks took {:?}", start.elapsed()); println!("Stopping TurboTasks took {:?}", start.elapsed());
if single_run {
return Ok(());
}
for i in 10..20 { for i in 10..20 {
let tt = registration.create_turbo_tasks(&name, false); let tt = registration.create_turbo_tasks(&name, false);
println!("Run #{i} (with filesystem cache if available, new TurboTasks instance)"); println!("Run #{i} (with filesystem cache if available, new TurboTasks instance)");
......
...@@ -497,6 +497,11 @@ impl From<anyhow::Error> for TurboTasksExecutionError { ...@@ -497,6 +497,11 @@ impl From<anyhow::Error> for TurboTasksExecutionError {
} }
} }
pub enum VerificationMode {
EqualityCheck,
Skip,
}
pub trait Backend: Sync + Send { pub trait Backend: Sync + Send {
#[allow(unused_variables)] #[allow(unused_variables)]
fn startup(&self, turbo_tasks: &dyn TurboTasksBackendApi<Self>) {} fn startup(&self, turbo_tasks: &dyn TurboTasksBackendApi<Self>) {}
...@@ -621,6 +626,7 @@ pub trait Backend: Sync + Send { ...@@ -621,6 +626,7 @@ pub trait Backend: Sync + Send {
task: TaskId, task: TaskId,
index: CellId, index: CellId,
content: CellContent, content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<Self>, turbo_tasks: &dyn TurboTasksBackendApi<Self>,
); );
......
...@@ -25,7 +25,7 @@ use crate::{ ...@@ -25,7 +25,7 @@ use crate::{
VcValueTrait, VcValueType, VcValueTrait, VcValueType,
backend::{ backend::{
Backend, CachedTaskType, CellContent, TaskCollectiblesMap, TaskExecutionSpec, Backend, CachedTaskType, CellContent, TaskCollectiblesMap, TaskExecutionSpec,
TransientTaskType, TurboTasksExecutionError, TypedCellContent, TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode,
}, },
capture_future::CaptureFuture, capture_future::CaptureFuture,
event::{Event, EventListener}, event::{Event, EventListener},
...@@ -155,7 +155,13 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send { ...@@ -155,7 +155,13 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
index: CellId, index: CellId,
options: ReadCellOptions, options: ReadCellOptions,
) -> Result<TypedCellContent>; ) -> Result<TypedCellContent>;
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent); fn update_own_task_cell(
&self,
task: TaskId,
index: CellId,
content: CellContent,
verification_mode: VerificationMode,
);
fn mark_own_task_as_finished(&self, task: TaskId); fn mark_own_task_as_finished(&self, task: TaskId);
fn set_own_task_aggregation_number(&self, task: TaskId, aggregation_number: u32); fn set_own_task_aggregation_number(&self, task: TaskId, aggregation_number: u32);
fn mark_own_task_as_session_dependent(&self, task: TaskId); fn mark_own_task_as_session_dependent(&self, task: TaskId);
...@@ -1354,8 +1360,15 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> { ...@@ -1354,8 +1360,15 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> {
self.try_read_own_task_cell(task, index, options) self.try_read_own_task_cell(task, index, options)
} }
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent) { fn update_own_task_cell(
self.backend.update_task_cell(task, index, content, self); &self,
task: TaskId,
index: CellId,
content: CellContent,
verification_mode: VerificationMode,
) {
self.backend
.update_task_cell(task, index, content, verification_mode, self);
} }
fn connect_task(&self, task: TaskId) { fn connect_task(&self, task: TaskId) {
...@@ -1769,7 +1782,12 @@ impl CurrentCellRef { ...@@ -1769,7 +1782,12 @@ impl CurrentCellRef {
.ok(); .ok();
let update = functor(cell_content.as_ref().and_then(|cc| cc.1.0.as_ref())); let update = functor(cell_content.as_ref().and_then(|cc| cc.1.0.as_ref()));
if let Some(update) = update { if let Some(update) = update {
tt.update_own_task_cell(self.current_task, self.index, CellContent(Some(update))) tt.update_own_task_cell(
self.current_task,
self.index,
CellContent(Some(update)),
VerificationMode::EqualityCheck,
)
} }
} }
...@@ -1851,7 +1869,7 @@ impl CurrentCellRef { ...@@ -1851,7 +1869,7 @@ impl CurrentCellRef {
} }
/// Unconditionally updates the content of the cell. /// Unconditionally updates the content of the cell.
pub fn update<T>(&self, new_value: T) pub fn update<T>(&self, new_value: T, verification_mode: VerificationMode)
where where
T: VcValueType, T: VcValueType,
{ {
...@@ -1862,6 +1880,7 @@ impl CurrentCellRef { ...@@ -1862,6 +1880,7 @@ impl CurrentCellRef {
CellContent(Some(SharedReference::new(triomphe::Arc::new( CellContent(Some(SharedReference::new(triomphe::Arc::new(
<T::Read as VcRead<T>>::value_to_repr(new_value), <T::Read as VcRead<T>>::value_to_repr(new_value),
)))), )))),
verification_mode,
) )
} }
...@@ -1873,27 +1892,40 @@ impl CurrentCellRef { ...@@ -1873,27 +1892,40 @@ impl CurrentCellRef {
/// ///
/// The [`SharedReference`] is expected to use the `<T::Read as /// The [`SharedReference`] is expected to use the `<T::Read as
/// VcRead<T>>::Repr` type for its representation of the value. /// VcRead<T>>::Repr` type for its representation of the value.
pub fn update_with_shared_reference(&self, shared_ref: SharedReference) { pub fn update_with_shared_reference(
&self,
shared_ref: SharedReference,
verification_mode: VerificationMode,
) {
let tt = turbo_tasks(); let tt = turbo_tasks();
let content = tt let update = if matches!(verification_mode, VerificationMode::EqualityCheck) {
.read_own_task_cell( let content = tt
self.current_task, .read_own_task_cell(
self.index, self.current_task,
ReadCellOptions { self.index,
// INVALIDATION: Reading our own cell must be untracked ReadCellOptions {
tracking: ReadTracking::Untracked, // INVALIDATION: Reading our own cell must be untracked
..Default::default() tracking: ReadTracking::Untracked,
}, ..Default::default()
) },
.ok(); )
let update = if let Some(TypedCellContent(_, CellContent(Some(shared_ref_exp)))) = content { .ok();
// pointer equality (not value equality) if let Some(TypedCellContent(_, CellContent(Some(shared_ref_exp)))) = content {
shared_ref_exp != shared_ref // pointer equality (not value equality)
shared_ref_exp != shared_ref
} else {
true
}
} else { } else {
true true
}; };
if update { if update {
tt.update_own_task_cell(self.current_task, self.index, CellContent(Some(shared_ref))) tt.update_own_task_cell(
self.current_task,
self.index,
CellContent(Some(shared_ref)),
verification_mode,
)
} }
} }
} }
......
use std::{any::type_name, marker::PhantomData}; use std::{any::type_name, marker::PhantomData};
use super::{read::VcRead, traits::VcValueType}; use super::{read::VcRead, traits::VcValueType};
use crate::{RawVc, Vc, manager::find_cell_by_type, task::shared_reference::TypedSharedReference}; use crate::{
RawVc, Vc, backend::VerificationMode, manager::find_cell_by_type,
task::shared_reference::TypedSharedReference,
};
type VcReadTarget<T> = <<T as VcValueType>::Read as VcRead<T>>::Target; type VcReadTarget<T> = <<T as VcValueType>::Read as VcRead<T>>::Target;
type VcReadRepr<T> = <<T as VcValueType>::Read as VcRead<T>>::Repr; type VcReadRepr<T> = <<T as VcValueType>::Read as VcRead<T>>::Repr;
...@@ -37,7 +40,10 @@ where ...@@ -37,7 +40,10 @@ where
{ {
fn cell(inner: VcReadTarget<T>) -> Vc<T> { fn cell(inner: VcReadTarget<T>) -> Vc<T> {
let cell = find_cell_by_type(T::get_value_type_id()); let cell = find_cell_by_type(T::get_value_type_id());
cell.update(<T::Read as VcRead<T>>::target_to_value(inner)); cell.update(
<T::Read as VcRead<T>>::target_to_value(inner),
VerificationMode::Skip,
);
Vc { Vc {
node: cell.into(), node: cell.into(),
_t: PhantomData, _t: PhantomData,
...@@ -47,7 +53,7 @@ where ...@@ -47,7 +53,7 @@ where
fn raw_cell(content: TypedSharedReference) -> RawVc { fn raw_cell(content: TypedSharedReference) -> RawVc {
debug_assert_repr::<T>(&content); debug_assert_repr::<T>(&content);
let cell = find_cell_by_type(content.type_id); let cell = find_cell_by_type(content.type_id);
cell.update_with_shared_reference(content.reference); cell.update_with_shared_reference(content.reference, VerificationMode::Skip);
cell.into() cell.into()
} }
} }
......
...@@ -13,7 +13,7 @@ use turbopack_core::{ ...@@ -13,7 +13,7 @@ use turbopack_core::{
chunk::{ chunk::{
Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig,
ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset,
EvaluatableAssets, MinifyType, ModuleId, SourceMapsType, EvaluatableAssets, MinifyType, ModuleId, SourceMapSourceType, SourceMapsType,
availability_info::AvailabilityInfo, availability_info::AvailabilityInfo,
chunk_group::{MakeChunkGroupResult, make_chunk_group}, chunk_group::{MakeChunkGroupResult, make_chunk_group},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
...@@ -91,8 +91,8 @@ impl BrowserChunkingContextBuilder { ...@@ -91,8 +91,8 @@ impl BrowserChunkingContextBuilder {
self self
} }
pub fn use_file_source_map_uris(mut self) -> Self { pub fn source_map_source_type(mut self, source_map_source_type: SourceMapSourceType) -> Self {
self.chunking_context.should_use_file_source_map_uris = true; self.chunking_context.source_map_source_type = source_map_source_type;
self self
} }
...@@ -229,8 +229,8 @@ pub struct BrowserChunkingContext { ...@@ -229,8 +229,8 @@ pub struct BrowserChunkingContext {
name: Option<RcStr>, name: Option<RcStr>,
/// The root path of the project /// The root path of the project
root_path: FileSystemPath, root_path: FileSystemPath,
/// Whether to write file sources as file:// paths in source maps /// The strategy to use for generating source map source uris
should_use_file_source_map_uris: bool, source_map_source_type: SourceMapSourceType,
/// This path is used to compute the url to request chunks from /// This path is used to compute the url to request chunks from
output_root: FileSystemPath, output_root: FileSystemPath,
/// The relative path from the output_root to the root_path. /// The relative path from the output_root to the root_path.
...@@ -311,7 +311,7 @@ impl BrowserChunkingContext { ...@@ -311,7 +311,7 @@ impl BrowserChunkingContext {
client_root, client_root,
client_roots: Default::default(), client_roots: Default::default(),
chunk_root_path, chunk_root_path,
should_use_file_source_map_uris: false, source_map_source_type: SourceMapSourceType::TurbopackUri,
asset_root_path, asset_root_path,
asset_root_paths: Default::default(), asset_root_paths: Default::default(),
chunk_base_path: None, chunk_base_path: None,
...@@ -617,8 +617,8 @@ impl ChunkingContext for BrowserChunkingContext { ...@@ -617,8 +617,8 @@ impl ChunkingContext for BrowserChunkingContext {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn should_use_file_source_map_uris(&self) -> Vc<bool> { fn source_map_source_type(&self) -> Vc<SourceMapSourceType> {
Vc::cell(self.should_use_file_source_map_uris) self.source_map_source_type.cell()
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -6,7 +6,9 @@ use turbo_tasks_fs::FileSystemPath; ...@@ -6,7 +6,9 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_browser::{BrowserChunkingContext, react_refresh::assert_can_resolve_react_refresh}; use turbopack_browser::{BrowserChunkingContext, react_refresh::assert_can_resolve_react_refresh};
use turbopack_cli_utils::runtime_entry::{RuntimeEntries, RuntimeEntry}; use turbopack_cli_utils::runtime_entry::{RuntimeEntries, RuntimeEntry};
use turbopack_core::{ use turbopack_core::{
chunk::{ChunkableModule, ChunkingContext, EvaluatableAsset, SourceMapsType}, chunk::{
ChunkableModule, ChunkingContext, EvaluatableAsset, SourceMapSourceType, SourceMapsType,
},
environment::Environment, environment::Environment,
file_source::FileSource, file_source::FileSource,
module::Module, module::Module,
...@@ -51,7 +53,7 @@ pub async fn get_client_chunking_context( ...@@ -51,7 +53,7 @@ pub async fn get_client_chunking_context(
RuntimeType::Development, RuntimeType::Development,
) )
.hot_module_replacement() .hot_module_replacement()
.use_file_source_map_uris() .source_map_source_type(SourceMapSourceType::AbsoluteFileUri)
.dynamic_chunk_content_loading(true) .dynamic_chunk_content_loading(true)
.build(), .build(),
)) ))
......
use anyhow::Result; use anyhow::{Result, bail};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
...@@ -133,13 +133,22 @@ pub struct ChunkingConfig { ...@@ -133,13 +133,22 @@ pub struct ChunkingConfig {
#[turbo_tasks::value(transparent)] #[turbo_tasks::value(transparent)]
pub struct ChunkingConfigs(FxHashMap<ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig>); pub struct ChunkingConfigs(FxHashMap<ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig>);
#[turbo_tasks::value(shared)]
#[derive(Debug, Clone, Copy, Hash, TaskInput, Default)]
pub enum SourceMapSourceType {
AbsoluteFileUri,
RelativeUri,
#[default]
TurbopackUri,
}
/// A context for the chunking that influences the way chunks are created /// A context for the chunking that influences the way chunks are created
#[turbo_tasks::value_trait] #[turbo_tasks::value_trait]
pub trait ChunkingContext { pub trait ChunkingContext {
#[turbo_tasks::function] #[turbo_tasks::function]
fn name(self: Vc<Self>) -> Vc<RcStr>; fn name(self: Vc<Self>) -> Vc<RcStr>;
#[turbo_tasks::function] #[turbo_tasks::function]
fn should_use_file_source_map_uris(self: Vc<Self>) -> Vc<bool>; fn source_map_source_type(self: Vc<Self>) -> Vc<SourceMapSourceType>;
/// The root path of the project /// The root path of the project
#[turbo_tasks::function] #[turbo_tasks::function]
fn root_path(self: Vc<Self>) -> Vc<FileSystemPath>; fn root_path(self: Vc<Self>) -> Vc<FileSystemPath>;
...@@ -385,6 +394,13 @@ pub trait ChunkingContextExt { ...@@ -385,6 +394,13 @@ pub trait ChunkingContextExt {
) -> Vc<OutputAssetsWithReferenced> ) -> Vc<OutputAssetsWithReferenced>
where where
Self: Send; Self: Send;
/// Computes the relative path from the chunk output root to the project root.
///
/// This is used to compute relative paths for source maps in certain configurations.
fn relative_path_from_chunk_root_to_project_root(self: Vc<Self>) -> Vc<RcStr>
where
Self: Send;
} }
impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingContextExt for T { impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingContextExt for T {
...@@ -499,6 +515,52 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte ...@@ -499,6 +515,52 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
availability_info, availability_info,
) )
} }
fn relative_path_from_chunk_root_to_project_root(self: Vc<Self>) -> Vc<RcStr> {
relative_path_from_chunk_root_to_project_root(Vc::upcast_non_strict(self))
}
}
#[turbo_tasks::function]
async fn relative_path_from_chunk_root_to_project_root(
chunking_context: Vc<Box<dyn ChunkingContext>>,
) -> Result<Vc<RcStr>> {
// Example,
// project root: /project/root
// output root: /project/root/dist
// chunk root path: /project/root/dist/ssr/chunks
// output_root_to_chunk_root: ../
//
// Example2,
// project root: /project/root
// output root: /project/out
// chunk root path: /project/out/ssr/chunks
// output_root_to_chunk_root: ../root
//
// From that we want to return ../../../root to get from a path in `chunks` to a path in the
// project root.
let chunk_root_path = chunking_context.chunk_root_path().await?;
let output_root = chunking_context.output_root().await?;
let chunk_to_output_root = chunk_root_path.get_relative_path_to(&output_root);
let Some(chunk_to_output_root) = chunk_to_output_root else {
bail!(
"expected chunk_root_path: {chunk_root_path} to be inside of output_root: \
{output_root}",
chunk_root_path = chunk_root_path.value_to_string().await?,
output_root = output_root.value_to_string().await?
);
};
let output_root_to_chunk_root_path = chunking_context.output_root_to_root_path().await?;
// Note we cannot use `normalize_path` here since it rejects paths that start with `../`
Ok(Vc::cell(
format!(
"{}/{}",
chunk_to_output_root, output_root_to_chunk_root_path
)
.into(),
))
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -29,7 +29,8 @@ pub use self::{ ...@@ -29,7 +29,8 @@ pub use self::{
}, },
chunking_context::{ chunking_context::{
ChunkGroupResult, ChunkGroupType, ChunkingConfig, ChunkingConfigs, ChunkingContext, ChunkGroupResult, ChunkGroupType, ChunkingConfig, ChunkingConfigs, ChunkingContext,
ChunkingContextExt, EntryChunkGroupResult, MangleType, MinifyType, SourceMapsType, ChunkingContextExt, EntryChunkGroupResult, MangleType, MinifyType, SourceMapSourceType,
SourceMapsType,
}, },
data::{ChunkData, ChunkDataOption, ChunksData}, data::{ChunkData, ChunkDataOption, ChunksData},
evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets}, evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets},
......
...@@ -6,9 +6,10 @@ use once_cell::sync::Lazy; ...@@ -6,9 +6,10 @@ use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::value::RawValue; use serde_json::value::RawValue;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString}; use turbo_tasks::{ResolvedVc, ValueToString};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_file, DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_path_buf,
}; };
use url::Url; use url::Url;
...@@ -71,6 +72,7 @@ struct SourceMapJson { ...@@ -71,6 +72,7 @@ struct SourceMapJson {
ignore_list: Option<Box<RawValue>>, ignore_list: Option<Box<RawValue>>,
// A somewhat widespread non-standard extension // A somewhat widespread non-standard extension
#[serde(skip_serializing_if = "Option::is_none")]
debug_id: Option<Box<RawValue>>, debug_id: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
...@@ -224,12 +226,18 @@ fn unencoded_str_to_raw_value(unencoded: &str) -> Box<RawValue> { ...@@ -224,12 +226,18 @@ fn unencoded_str_to_raw_value(unencoded: &str) -> Box<RawValue> {
.expect("serde_json::to_string should produce valid JSON") .expect("serde_json::to_string should produce valid JSON")
} }
/// Turns `turbopack:///[project]` references in sourcemap sources into absolute `file://` uris. This /// Helper function to transform turbopack:/// file references in a sourcemap.
/// is useful for debugging environments. /// Handles parsing the sourcemap, resolving the filesystem, applying transformations, and
pub async fn fileify_source_map( /// serializing back.
/// The transform function is given the source string as found in the sourcemap (i.e. a URI).
async fn transform_relative_files<F>(
map: Option<&Rope>, map: Option<&Rope>,
context_path: FileSystemPath, context_path: &FileSystemPath,
) -> Result<Option<Rope>> { mut transform: F,
) -> Result<Option<Rope>>
where
F: FnMut(&DiskFileSystem, &str) -> Result<String>,
{
let Some(map) = map else { let Some(map) = map else {
return Ok(None); return Ok(None);
}; };
...@@ -243,29 +251,75 @@ pub async fn fileify_source_map( ...@@ -243,29 +251,75 @@ pub async fn fileify_source_map(
let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs) let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
.context("Expected the chunking context to have a DiskFileSystem")? .context("Expected the chunking context to have a DiskFileSystem")?
.await?; .await?;
let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL, context_fs.name()); let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL, context_fs.name());
let transform_source = async |src: &mut Option<String>| { let mut apply_transform = |src: &mut String| -> Result<()> {
if let Some(src) = src if let Some(src_rest) = src.strip_prefix(&prefix) {
&& let Some(src_rest) = src.strip_prefix(&prefix) *src = transform(context_fs, src_rest)?;
{
*src = uri_from_file(context_path.clone(), Some(src_rest)).await?;
} }
anyhow::Ok(()) Ok(())
}; };
for src in map.sources.iter_mut().flatten() { for src in map.sources.iter_mut().flatten().flatten() {
transform_source(src).await?; apply_transform(src)?;
} }
for section in map.sections.iter_mut().flatten() { for section in map.sections.iter_mut().flatten() {
for src in section.map.sources.iter_mut().flatten() { for src in section.map.sources.iter_mut().flatten().flatten() {
transform_source(src).await?; apply_transform(src)?;
} }
} }
let map = Rope::from(serde_json::to_vec(&map)?); Ok(Some(Rope::from(serde_json::to_vec(&map)?)))
}
Ok(Some(map)) /// Turns `turbopack:///[project]` references in sourcemap sources into absolute `file://` uris. This
/// is useful for debugging environments.
pub async fn absolute_fileify_source_map(
map: Option<&Rope>,
context_path: FileSystemPath,
) -> Result<Option<Rope>> {
transform_relative_files(map, &context_path, |context_fs, src_rest| {
let path = context_path.join(src_rest)?;
Ok(uri_from_path_buf(context_fs.to_sys_path(&path)))
})
.await
}
fn uri_encode_path(path: &str) -> String {
path.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
}
/// Turns `turbopack:///[project]` references in sourcemap sources into relative './' prefixed uris.
/// This is useful in server environments and especially build environments.
pub async fn relative_fileify_source_map(
map: Option<&Rope>,
context_path: FileSystemPath,
relative_path_to_output_root: RcStr,
) -> Result<Option<Rope>> {
let relative_path_to_output_root = relative_path_to_output_root
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/");
transform_relative_files(map, &context_path, |_context_fs, src_rest| {
// NOTE: we just include the relative path prefix here instead of using `sourceRoot`
// since the spec on sourceRoot is broken.
// TODO(bgw): this shouldn't be necessary to uri encode since the strings we get out of the
// source map should already be uri encoded, however in the case of the turbopack scheme in
// particular we are inconsistent so be defensive here.
let src_rest = uri_encode_path(src_rest);
if relative_path_to_output_root.is_empty() {
Ok(src_rest.to_string())
} else {
Ok(format!("{relative_path_to_output_root}/{src_rest}",))
}
})
.await
} }
#[cfg(test)] #[cfg(test)]
......
...@@ -18,8 +18,8 @@ use turbopack_core::{ ...@@ -18,8 +18,8 @@ use turbopack_core::{
chunk::{ chunk::{
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemBatchGroup, ChunkItemExt, AsyncModuleInfo, Chunk, ChunkItem, ChunkItemBatchGroup, ChunkItemExt,
ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo, ChunkType, ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo, ChunkType,
ChunkableModule, ChunkingContext, MinifyType, OutputChunk, OutputChunkRuntimeInfo, ChunkableModule, ChunkingContext, ChunkingContextExt, MinifyType, OutputChunk,
round_chunk_item_size, OutputChunkRuntimeInfo, SourceMapSourceType, round_chunk_item_size,
}, },
code_builder::{Code, CodeBuilder}, code_builder::{Code, CodeBuilder},
ident::AssetIdent, ident::AssetIdent,
...@@ -32,7 +32,10 @@ use turbopack_core::{ ...@@ -32,7 +32,10 @@ use turbopack_core::{
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
reference_type::ImportContext, reference_type::ImportContext,
server_fs::ServerFileSystem, server_fs::ServerFileSystem,
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, utils::fileify_source_map}, source_map::{
GenerateSourceMap, OptionStringifiedSourceMap,
utils::{absolute_fileify_source_map, relative_fileify_source_map},
},
}; };
use self::{single_item_chunk::chunk::SingleItemCssChunk, source_map::CssChunkSourceMapAsset}; use self::{single_item_chunk::chunk::SingleItemCssChunk, source_map::CssChunkSourceMapAsset};
...@@ -96,18 +99,27 @@ impl CssChunk { ...@@ -96,18 +99,27 @@ impl CssChunk {
let close = write_import_context(&mut body, content.import_context).await?; let close = write_import_context(&mut body, content.import_context).await?;
let source_map = if *self let chunking_context = self.chunking_context();
.chunking_context() let source_map = match *chunking_context.source_map_source_type().await? {
.should_use_file_source_map_uris() SourceMapSourceType::AbsoluteFileUri => {
.await? absolute_fileify_source_map(
{ content.source_map.as_ref(),
fileify_source_map( chunking_context.root_path().owned().await?,
content.source_map.as_ref(), )
self.chunking_context().root_path().owned().await?, .await?
) }
.await? SourceMapSourceType::RelativeUri => {
} else { relative_fileify_source_map(
content.source_map.clone() content.source_map.as_ref(),
chunking_context.root_path().owned().await?,
chunking_context
.relative_path_from_chunk_root_to_project_root()
.owned()
.await?,
)
.await?
}
SourceMapSourceType::TurbopackUri => content.source_map.clone(),
}; };
body.push_source(&content.inner_code, source_map); body.push_source(&content.inner_code, source_map);
......
...@@ -3,18 +3,21 @@ use std::io::Write; ...@@ -3,18 +3,21 @@ use std::io::Write;
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use smallvec::SmallVec; use smallvec::SmallVec;
use turbo_rcstr::rcstr; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ use turbo_tasks::{
NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, Upcast, ValueToString, Vc, NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, Upcast, ValueToString, Vc,
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
use turbo_tasks_fs::{FileSystemPath, rope::Rope}; use turbo_tasks_fs::{FileSystemPath, rope::Rope};
use turbopack_core::{ use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext, ModuleId}, chunk::{
AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
ChunkingContextExt, ModuleId, SourceMapSourceType,
},
code_builder::{Code, CodeBuilder}, code_builder::{Code, CodeBuilder},
error::PrettyPrintError, error::PrettyPrintError,
issue::{IssueExt, IssueSeverity, StyledString, code_gen::CodeGenerationIssue}, issue::{IssueExt, IssueSeverity, StyledString, code_gen::CodeGenerationIssue},
source_map::utils::fileify_source_map, source_map::utils::{absolute_fileify_source_map, relative_fileify_source_map},
}; };
use crate::{ use crate::{
...@@ -24,6 +27,26 @@ use crate::{ ...@@ -24,6 +27,26 @@ use crate::{
utils::StringifyJs, utils::StringifyJs,
}; };
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Hash,
Serialize,
Deserialize,
TraceRawVcs,
TaskInput,
NonLocalValue,
Default,
)]
pub enum RewriteSourcePath {
AbsoluteFilePath(FileSystemPath),
RelativeFilePath(FileSystemPath, RcStr),
#[default]
None,
}
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(Default, Clone)] #[derive(Default, Clone)]
pub struct EcmascriptChunkItemContent { pub struct EcmascriptChunkItemContent {
...@@ -31,7 +54,7 @@ pub struct EcmascriptChunkItemContent { ...@@ -31,7 +54,7 @@ pub struct EcmascriptChunkItemContent {
pub source_map: Option<Rope>, pub source_map: Option<Rope>,
pub additional_ids: SmallVec<[ResolvedVc<ModuleId>; 1]>, pub additional_ids: SmallVec<[ResolvedVc<ModuleId>; 1]>,
pub options: EcmascriptChunkItemOptions, pub options: EcmascriptChunkItemOptions,
pub rewrite_source_path: Option<FileSystemPath>, pub rewrite_source_path: RewriteSourcePath,
pub placeholder_for_future_extensions: (), pub placeholder_for_future_extensions: (),
} }
...@@ -53,10 +76,18 @@ impl EcmascriptChunkItemContent { ...@@ -53,10 +76,18 @@ impl EcmascriptChunkItemContent {
let strict = content.strict; let strict = content.strict;
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
rewrite_source_path: if *chunking_context.should_use_file_source_map_uris().await? { rewrite_source_path: match *chunking_context.source_map_source_type().await? {
Some(chunking_context.root_path().owned().await?) SourceMapSourceType::AbsoluteFileUri => {
} else { RewriteSourcePath::AbsoluteFilePath(chunking_context.root_path().owned().await?)
None }
SourceMapSourceType::RelativeUri => RewriteSourcePath::RelativeFilePath(
chunking_context.root_path().owned().await?,
chunking_context
.relative_path_from_chunk_root_to_project_root()
.owned()
.await?,
),
SourceMapSourceType::TurbopackUri => RewriteSourcePath::None,
}, },
inner_code: content.inner_code.clone(), inner_code: content.inner_code.clone(),
source_map: content.source_map.clone(), source_map: content.source_map.clone(),
...@@ -112,10 +143,19 @@ impl EcmascriptChunkItemContent { ...@@ -112,10 +143,19 @@ impl EcmascriptChunkItemContent {
)?; )?;
} }
let source_map = if let Some(rewrite_source_path) = &self.rewrite_source_path { let source_map = match &self.rewrite_source_path {
fileify_source_map(self.source_map.as_ref(), rewrite_source_path.clone()).await? RewriteSourcePath::AbsoluteFilePath(path) => {
} else { absolute_fileify_source_map(self.source_map.as_ref(), path.clone()).await?
self.source_map.clone() }
RewriteSourcePath::RelativeFilePath(path, relative_path) => {
relative_fileify_source_map(
self.source_map.as_ref(),
path.clone(),
relative_path.clone(),
)
.await?
}
RewriteSourcePath::None => self.source_map.clone(),
}; };
code.push_source(&self.inner_code, source_map); code.push_source(&self.inner_code, source_map);
......
...@@ -13,7 +13,7 @@ use crate::{ ...@@ -13,7 +13,7 @@ use crate::{
EcmascriptAnalyzableExt, EcmascriptAnalyzableExt,
chunk::{ chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemOptions, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemOptions,
EcmascriptChunkPlaceable, EcmascriptChunkType, EcmascriptChunkPlaceable, EcmascriptChunkType, item::RewriteSourcePath,
}, },
references::async_module::AsyncModuleOptions, references::async_module::AsyncModuleOptions,
runtime_functions::{TURBOPACK_EXPORT_NAMESPACE, TURBOPACK_IMPORT}, runtime_functions::{TURBOPACK_EXPORT_NAMESPACE, TURBOPACK_IMPORT},
...@@ -166,7 +166,7 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem { ...@@ -166,7 +166,7 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
Ok(EcmascriptChunkItemContent { Ok(EcmascriptChunkItemContent {
inner_code: code, inner_code: code,
source_map: None, source_map: None,
rewrite_source_path: None, rewrite_source_path: RewriteSourcePath::None,
options: EcmascriptChunkItemOptions { options: EcmascriptChunkItemOptions {
strict: true, strict: true,
async_module: if has_top_level_await { async_module: if has_top_level_await {
......
...@@ -16,8 +16,8 @@ use serde_json::Value as JsonValue; ...@@ -16,8 +16,8 @@ use serde_json::Value as JsonValue;
use turbo_rcstr::rcstr; use turbo_rcstr::rcstr;
use turbo_tasks::{ use turbo_tasks::{
Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, RawVc, ReadRef, ResolvedVc, Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, RawVc, ReadRef, ResolvedVc,
TaskInput, TryJoinIterExt, Vc, VcValueType, duration_span, fxindexmap, get_effects, TaskInput, TryJoinIterExt, Vc, VcValueType, backend::VerificationMode, duration_span,
mark_finished, prevent_gc, trace::TraceRawVcs, util::SharedError, fxindexmap, get_effects, mark_finished, prevent_gc, trace::TraceRawVcs, util::SharedError,
}; };
use turbo_tasks_bytes::{Bytes, Stream}; use turbo_tasks_bytes::{Bytes, Stream};
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
...@@ -401,10 +401,10 @@ pub async fn custom_evaluate( ...@@ -401,10 +401,10 @@ pub async fn custom_evaluate(
// We initialize the cell with a stream that is open, but has no values. // We initialize the cell with a stream that is open, but has no values.
// The first [compute_evaluate_stream] pipe call will pick up that stream. // The first [compute_evaluate_stream] pipe call will pick up that stream.
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
cell.update(JavaScriptEvaluation(JavaScriptStream::new_open( cell.update(
vec![], JavaScriptEvaluation(JavaScriptStream::new_open(vec![], Box::new(receiver))),
Box::new(receiver), VerificationMode::Skip,
))); );
let initial = Mutex::new(Some(sender)); let initial = Mutex::new(Some(sender));
// run the evaluation as side effect // run the evaluation as side effect
...@@ -418,10 +418,13 @@ pub async fn custom_evaluate( ...@@ -418,10 +418,13 @@ pub async fn custom_evaluate(
// In cases when only [compute_evaluate_stream] is (re)executed, we need to // In cases when only [compute_evaluate_stream] is (re)executed, we need to
// update the old stream with a new value. // update the old stream with a new value.
let (sender, receiver) = unbounded(); let (sender, receiver) = unbounded();
cell.update(JavaScriptEvaluation(JavaScriptStream::new_open( cell.update(
vec![], JavaScriptEvaluation(JavaScriptStream::new_open(
Box::new(receiver), vec![],
))); Box::new(receiver),
)),
VerificationMode::Skip,
);
sender sender
} }
}), }),
......
...@@ -8,7 +8,7 @@ use turbopack_core::{ ...@@ -8,7 +8,7 @@ use turbopack_core::{
chunk::{ chunk::{
Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig, Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig,
ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType,
ModuleId, SourceMapsType, ModuleId, SourceMapSourceType, SourceMapsType,
availability_info::AvailabilityInfo, availability_info::AvailabilityInfo,
chunk_group::{MakeChunkGroupResult, make_chunk_group}, chunk_group::{MakeChunkGroupResult, make_chunk_group},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
...@@ -99,8 +99,8 @@ impl NodeJsChunkingContextBuilder { ...@@ -99,8 +99,8 @@ impl NodeJsChunkingContextBuilder {
self self
} }
pub fn use_file_source_map_uris(mut self) -> Self { pub fn source_map_source_type(mut self, source_map_source_type: SourceMapSourceType) -> Self {
self.chunking_context.should_use_file_source_map_uris = true; self.chunking_context.source_map_source_type = source_map_source_type;
self self
} }
...@@ -182,8 +182,8 @@ pub struct NodeJsChunkingContext { ...@@ -182,8 +182,8 @@ pub struct NodeJsChunkingContext {
module_id_strategy: ResolvedVc<Box<dyn ModuleIdStrategy>>, module_id_strategy: ResolvedVc<Box<dyn ModuleIdStrategy>>,
/// The module export usage info, if available. /// The module export usage info, if available.
export_usage: Option<ResolvedVc<ExportUsageInfo>>, export_usage: Option<ResolvedVc<ExportUsageInfo>>,
/// Whether to use file:// uris for source map sources /// The strategy to use for generating source map source uris
should_use_file_source_map_uris: bool, source_map_source_type: SourceMapSourceType,
/// The chunking configs /// The chunking configs
chunking_configs: Vec<(ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig)>, chunking_configs: Vec<(ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig)>,
/// Enable debug IDs for chunks and source maps. /// Enable debug IDs for chunks and source maps.
...@@ -222,7 +222,7 @@ impl NodeJsChunkingContext { ...@@ -222,7 +222,7 @@ impl NodeJsChunkingContext {
minify_type: MinifyType::NoMinify, minify_type: MinifyType::NoMinify,
source_maps_type: SourceMapsType::Full, source_maps_type: SourceMapsType::Full,
manifest_chunks: false, manifest_chunks: false,
should_use_file_source_map_uris: false, source_map_source_type: SourceMapSourceType::TurbopackUri,
module_id_strategy: ResolvedVc::upcast(DevModuleIdStrategy::new_resolved()), module_id_strategy: ResolvedVc::upcast(DevModuleIdStrategy::new_resolved()),
export_usage: None, export_usage: None,
chunking_configs: Default::default(), chunking_configs: Default::default(),
...@@ -387,8 +387,8 @@ impl ChunkingContext for NodeJsChunkingContext { ...@@ -387,8 +387,8 @@ impl ChunkingContext for NodeJsChunkingContext {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
fn should_use_file_source_map_uris(&self) -> Vc<bool> { fn source_map_source_type(&self) -> Vc<SourceMapSourceType> {
Vc::cell(self.should_use_file_source_map_uris) self.source_map_source_type.cell()
} }
#[turbo_tasks::function] #[turbo_tasks::function]
......
...@@ -33,7 +33,7 @@ use turbopack_core::{ ...@@ -33,7 +33,7 @@ use turbopack_core::{
asset::Asset, asset::Asset,
chunk::{ chunk::{
ChunkingConfig, ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssetExt, ChunkingConfig, ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssetExt,
EvaluatableAssets, MinifyType, availability_info::AvailabilityInfo, EvaluatableAssets, MinifyType, SourceMapSourceType, availability_info::AvailabilityInfo,
}, },
compile_time_defines, compile_time_defines,
compile_time_info::{CompileTimeDefineValue, CompileTimeInfo, DefinableNameSegment}, compile_time_info::{CompileTimeDefineValue, CompileTimeInfo, DefinableNameSegment},
...@@ -91,6 +91,8 @@ struct SnapshotOptions { ...@@ -91,6 +91,8 @@ struct SnapshotOptions {
production_chunking: bool, production_chunking: bool,
#[serde(default)] #[serde(default)]
enable_debug_ids: bool, enable_debug_ids: bool,
#[serde(default)]
source_map_source_type: SourceMapSourceType,
} }
#[derive(Debug, Deserialize, Default)] #[derive(Debug, Deserialize, Default)]
...@@ -121,6 +123,7 @@ impl Default for SnapshotOptions { ...@@ -121,6 +123,7 @@ impl Default for SnapshotOptions {
scope_hoisting: false, scope_hoisting: false,
production_chunking: false, production_chunking: false,
enable_debug_ids: false, enable_debug_ids: false,
source_map_source_type: SourceMapSourceType::default(),
} }
} }
} }
...@@ -460,7 +463,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> { ...@@ -460,7 +463,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.minify_type(options.minify_type) .minify_type(options.minify_type)
.module_merging(options.scope_hoisting) .module_merging(options.scope_hoisting)
.export_usage(export_usage) .export_usage(export_usage)
.debug_ids(options.enable_debug_ids); .debug_ids(options.enable_debug_ids)
.source_map_source_type(options.source_map_source_type);
if options.production_chunking { if options.production_chunking {
builder = builder.chunking_config( builder = builder.chunking_config(
...@@ -489,7 +493,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> { ...@@ -489,7 +493,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.minify_type(options.minify_type) .minify_type(options.minify_type)
.module_merging(options.scope_hoisting) .module_merging(options.scope_hoisting)
.export_usage(export_usage) .export_usage(export_usage)
.debug_ids(options.enable_debug_ids); .debug_ids(options.enable_debug_ids)
.source_map_source_type(options.source_map_source_type);
if options.production_chunking { if options.production_chunking {
builder = builder.chunking_config( builder = builder.chunking_config(
......
...@@ -3,5 +3,6 @@ ...@@ -3,5 +3,6 @@
"runtimeType": "Development", "runtimeType": "Development",
"minifyType": "NoMinify", "minifyType": "NoMinify",
"environment": "NodeJs", "environment": "NodeJs",
"enableDebugIds": true "enableDebugIds": true,
"sourceMapSourceType": "RelativeUri"
} }
...@@ -3,5 +3,5 @@ ...@@ -3,5 +3,5 @@
"sources": [], "sources": [],
"debugId": "3d97c2d1-90e5-a3ef-c710-5c78b21b3b3a", "debugId": "3d97c2d1-90e5-a3ef-c710-5c78b21b3b3a",
"sections": [ "sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/debug-ids/node/input/index.js"],"sourcesContent":["console.log('Hello World')\n"],"names":[],"mappings":"AAAA,QAAQ,GAAG,CAAC"}}] {"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["../../../../../../../../turbopack/crates/turbopack-tests/tests/snapshot/debug-ids/node/input/index.js"],"sourcesContent":["console.log('Hello World')\n"],"names":[],"mappings":"AAAA,QAAQ,GAAG,CAAC"}}]
} }
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment