Unverified Commit 6bdd0475 authored by Tobias Koppers's avatar Tobias Koppers Committed by GitHub

Merge branch 'canary' into sokra/flaky-scroll-test

parents 07636dfc f7e66622
......@@ -58,10 +58,7 @@ impl NextMode {
}
pub fn is_production(&self) -> bool {
match self {
NextMode::Development => false,
NextMode::Build => true,
}
!self.is_development()
}
pub fn runtime_type(&self) -> RuntimeType {
......
......@@ -19,8 +19,8 @@ use turbopack_browser::{
};
use turbopack_core::{
chunk::{
ChunkingConfig, ChunkingContext, MangleType, MinifyType, SourceMapsType,
module_id_strategies::ModuleIdStrategy,
ChunkingConfig, ChunkingContext, MangleType, MinifyType, SourceMapSourceType,
SourceMapsType, module_id_strategies::ModuleIdStrategy,
},
compile_time_info::{CompileTimeDefines, CompileTimeInfo, FreeVarReference, FreeVarReferences},
environment::{BrowserEnvironment, Environment, ExecutionEnvironment},
......@@ -489,7 +489,7 @@ pub async fn get_client_chunking_context(
if next_mode.is_development() {
builder = builder
.hot_module_replacement()
.use_file_source_map_uris()
.source_map_source_type(SourceMapSourceType::AbsoluteFileUri)
.dynamic_chunk_content_loading(true);
} else {
builder = builder
......
......@@ -16,7 +16,7 @@ use turbopack::{
};
use turbopack_core::{
chunk::{
ChunkingConfig, MangleType, MinifyType, SourceMapsType,
ChunkingConfig, MangleType, MinifyType, SourceMapSourceType, SourceMapsType,
module_id_strategies::ModuleIdStrategy,
},
compile_time_defines,
......@@ -1060,9 +1060,13 @@ pub async fn get_server_chunking_context_with_client_assets(
.file_tracing(next_mode.is_production())
.debug_ids(*debug_ids.await?);
if next_mode.is_development() {
builder = builder.use_file_source_map_uris();
builder = builder.source_map_source_type(if next_mode.is_development() {
SourceMapSourceType::AbsoluteFileUri
} else {
// TODO(lukesandberg): switch to relative once next is compatible.
SourceMapSourceType::TurbopackUri
});
if next_mode.is_production() {
builder = builder
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
......@@ -1142,9 +1146,11 @@ pub async fn get_server_chunking_context(
.debug_ids(*debug_ids.await?);
if next_mode.is_development() {
builder = builder.use_file_source_map_uris()
builder = builder.source_map_source_type(SourceMapSourceType::AbsoluteFileUri);
} else {
builder = builder
// TODO(lukesandberg): switch to relative once next is compatible.
.source_map_source_type(SourceMapSourceType::TurbopackUri)
.chunking_config(
Vc::<EcmascriptChunkType>::default().to_resolved().await?,
ChunkingConfig {
......
......@@ -112,7 +112,10 @@ export default async function getChangedTests() {
.catch(() => false)
if (fileExists && file.match(/^test\/.*?\.test\.(js|ts|tsx)$/)) {
if (file.startsWith('test/e2e/')) {
if (
file.startsWith('test/e2e/') ||
file.startsWith('test/integration/')
) {
devTests.push(file)
prodTests.push(file)
} else if (file.startsWith('test/prod')) {
......
......@@ -58,24 +58,27 @@ describe('nextjs APIs in after()', () => {
`[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).toContain(
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).toContain(
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
)
})
await retry(() => {
const logs = getLogs()
expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).toContain(
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).toContain(
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] nested connection(): ok`)
......@@ -105,24 +108,27 @@ describe('nextjs APIs in after()', () => {
`[${path}] headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).toContain(
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] cookies(): ok`)
expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).toContain(
`[${path}] cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
)
})
await retry(() => {
const logs = isNextDev ? getLogs() : buildLogs // in `next start` the error was logged at build time
expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).not.toContain(`[${path}] nested headers(): ok`)
expect(logs).toContain(
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
`[${path}] nested headers(): error: Error: Route ${path} used \`headers()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] connection(): ok`)
expect(logs).not.toContain(`[${path}] nested cookies(): ok`)
expect(logs).toContain(
`[${path}] connection(): error: Error: Route ${path} used \`connection()\` inside \`after()\`.`
`[${path}] nested cookies(): error: Error: Route ${path} used \`cookies()\` inside \`after()\`. This is not supported.`
)
expect(logs).not.toContain(`[${path}] nested connection(): ok`)
......
......@@ -18,11 +18,7 @@ describe('New Link Behavior with material-ui', () => {
'@emotion/styled': 'latest',
'@mui/icons-material': 'latest',
'@mui/material': 'latest',
next: 'latest',
'prop-types': 'latest',
// Use minimum peer dep version instead of v9 of eslint to avoid breaking changes
eslint: '8.56.0',
'eslint-config-next': 'latest',
},
})
......
......@@ -8,7 +8,6 @@ import {
launchApp,
nextStart,
nextBuild,
check,
retry,
} from 'next-test-utils'
......@@ -39,17 +38,15 @@ const runTests = () => {
await browser.eval(() => window.next.router.push('/another'))
await check(
() => browser.eval(() => document.documentElement.innerHTML),
/hi from another/
)
await retry(async () => {
const html = await browser.eval(() => document.documentElement.innerHTML)
expect(html).toMatch(/hi from another/)
})
await browser.eval(() => (window.didHydrate = false))
await browser.eval(() => window.history.back())
await check(() => browser.eval(() => window.didHydrate), {
test(content) {
return content
},
await retry(async () => {
expect(await browser.eval(() => window.didHydrate)).toBe(true)
})
await retry(async () => {
......
......@@ -19,6 +19,7 @@ no_fast_stale = []
verify_serialization = []
verify_aggregation_graph = []
verify_immutable = []
verify_determinism = []
trace_aggregation_update = []
trace_find_and_schedule = []
trace_task_completion = []
......
......@@ -31,7 +31,7 @@ use turbo_tasks::{
TraitTypeId, TurboTasksBackendApi, ValueTypeId,
backend::{
Backend, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskRoot,
TransientTaskType, TurboTasksExecutionError, TypedCellContent,
TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode,
},
event::{Event, EventListener},
message_queue::TimingEvent,
......@@ -423,6 +423,8 @@ struct TaskExecutionCompletePrepareResult {
pub new_children: FxHashSet<TaskId>,
pub removed_data: Vec<CachedDataItem>,
pub is_now_immutable: bool,
#[cfg(feature = "verify_determinism")]
pub no_output_set: bool,
pub new_output: Option<OutputValue>,
pub output_dependent_tasks: SmallVec<[TaskId; 4]>,
}
......@@ -1765,6 +1767,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children,
mut removed_data,
is_now_immutable,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output,
output_dependent_tasks,
}) = self.task_execution_completed_prepare(
......@@ -1809,6 +1813,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
if self.task_execution_completed_finish(
&mut ctx,
task_id,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output,
&mut removed_data,
is_now_immutable,
......@@ -1843,6 +1849,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children: Default::default(),
removed_data: Default::default(),
is_now_immutable: false,
#[cfg(feature = "verify_determinism")]
no_output_set: false,
new_output: None,
output_dependent_tasks: Default::default(),
});
......@@ -2021,6 +2029,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
// Check if output need to be updated
let current_output = get!(task, Output);
#[cfg(feature = "verify_determinism")]
let no_output_set = current_output.is_none();
let new_output = match result {
Ok(RawVc::TaskOutput(output_task_id)) => {
if let Some(OutputValue::Output(current_task_id)) = current_output
......@@ -2092,6 +2102,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
new_children,
removed_data,
is_now_immutable,
#[cfg(feature = "verify_determinism")]
no_output_set,
new_output,
output_dependent_tasks,
})
......@@ -2232,6 +2244,7 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
&self,
ctx: &mut impl ExecuteContext<'_>,
task_id: TaskId,
#[cfg(feature = "verify_determinism")] no_output_set: bool,
new_output: Option<OutputValue>,
removed_data: &mut Vec<CachedDataItem>,
is_now_immutable: bool,
......@@ -2306,7 +2319,8 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
None
};
let data_update = if old_dirty_state != new_dirty_state {
let dirty_changed = old_dirty_state != new_dirty_state;
let data_update = if dirty_changed {
if let Some(new_dirty_state) = new_dirty_state {
task.insert(CachedDataItem::Dirty {
value: new_dirty_state,
......@@ -2353,17 +2367,32 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
None
};
drop(task);
drop(old_content);
#[cfg(feature = "verify_determinism")]
let reschedule = (dirty_changed || no_output_set) && !task_id.is_transient();
#[cfg(not(feature = "verify_determinism"))]
let reschedule = false;
if reschedule {
task.add_new(CachedDataItem::InProgress {
value: InProgressState::Scheduled {
done_event,
reason: TaskExecutionReason::Stale,
},
});
drop(task);
} else {
drop(task);
// Notify dependent tasks that are waiting for this task to finish
done_event.notify(usize::MAX);
}
// Notify dependent tasks that are waiting for this task to finish
done_event.notify(usize::MAX);
drop(old_content);
if let Some(data_update) = data_update {
AggregationUpdateQueue::run(data_update, ctx);
}
false
reschedule
}
fn task_execution_completed_cleanup(&self, ctx: &mut impl ExecuteContext<'_>, task_id: TaskId) {
......@@ -2652,12 +2681,14 @@ impl<B: BackingStorage> TurboTasksBackendInner<B> {
task_id: TaskId,
cell: CellId,
content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<TurboTasksBackend<B>>,
) {
operation::UpdateCellOperation::run(
task_id,
cell,
content,
verification_mode,
self.execute_context(turbo_tasks),
);
}
......@@ -3251,9 +3282,11 @@ impl<B: BackingStorage> Backend for TurboTasksBackend<B> {
task_id: TaskId,
cell: CellId,
content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<Self>,
) {
self.0.update_task_cell(task_id, cell, content, turbo_tasks);
self.0
.update_task_cell(task_id, cell, content, verification_mode, turbo_tasks);
}
fn mark_own_task_as_finished(
......
......@@ -382,6 +382,7 @@ impl<B: BackingStorage> TaskGuardImpl<'_, B> {
/// Verify that the task guard restored the correct category
/// before accessing the data.
#[inline]
#[track_caller]
fn check_access(&self, category: TaskDataCategory) {
{
match category {
......@@ -434,6 +435,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task_id
}
#[track_caller]
fn add(&mut self, item: CachedDataItem) -> bool {
let category = item.category();
self.check_access(category);
......@@ -446,12 +448,14 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.add(item)
}
#[track_caller]
fn add_new(&mut self, item: CachedDataItem) {
self.check_access(item.category());
let added = self.add(item);
assert!(added, "Item already exists");
}
#[track_caller]
fn insert(&mut self, item: CachedDataItem) -> Option<CachedDataItemValue> {
let category = item.category();
self.check_access(category);
......@@ -461,6 +465,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.insert(item)
}
#[track_caller]
fn update(
&mut self,
key: CachedDataItemKey,
......@@ -474,6 +479,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.update(key, update);
}
#[track_caller]
fn remove(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValue> {
let category = key.category();
self.check_access(category);
......@@ -488,6 +494,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get(key)
}
#[track_caller]
fn get_mut(&mut self, key: &CachedDataItemKey) -> Option<CachedDataItemValueRefMut<'_>> {
let category = key.category();
self.check_access(category);
......@@ -497,6 +504,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get_mut(key)
}
#[track_caller]
fn get_mut_or_insert_with(
&mut self,
key: CachedDataItemKey,
......@@ -510,11 +518,13 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.get_mut_or_insert_with(key, insert)
}
#[track_caller]
fn has_key(&self, key: &CachedDataItemKey) -> bool {
self.check_access(key.category());
self.task.contains_key(key)
}
#[track_caller]
fn count(&self, ty: CachedDataItemType) -> usize {
self.check_access(ty.category());
self.task.count(ty)
......@@ -532,6 +542,7 @@ impl<B: BackingStorage> TaskGuard for TaskGuardImpl<'_, B> {
self.task.shrink_to_fit(ty)
}
#[track_caller]
fn extract_if<'l, F>(
&'l mut self,
ty: CachedDataItemType,
......
......@@ -2,6 +2,8 @@ use std::mem::take;
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
#[cfg(not(feature = "verify_determinism"))]
use turbo_tasks::backend::VerificationMode;
use turbo_tasks::{CellId, TaskId, TypedSharedReference, backend::CellContent};
#[cfg(feature = "trace_task_dirty")]
......@@ -13,7 +15,7 @@ use crate::{
AggregationUpdateQueue, ExecuteContext, Operation, TaskGuard,
invalidate::make_task_dirty_internal,
},
storage::{get_many, remove},
storage::{get, get_many, remove},
},
data::{CachedDataItem, CachedDataItemKey, CellRef},
};
......@@ -40,18 +42,61 @@ pub enum UpdateCellOperation {
}
impl UpdateCellOperation {
pub fn run(task_id: TaskId, cell: CellId, content: CellContent, mut ctx: impl ExecuteContext) {
pub fn run(
task_id: TaskId,
cell: CellId,
content: CellContent,
#[cfg(feature = "verify_determinism")] verification_mode: VerificationMode,
#[cfg(not(feature = "verify_determinism"))] _verification_mode: VerificationMode,
mut ctx: impl ExecuteContext,
) {
let content = if let CellContent(Some(new_content)) = content {
Some(new_content.into_typed(cell.type_id))
} else {
None
};
let mut task = ctx.task(task_id, TaskDataCategory::All);
let is_stateful = task.has_key(&CachedDataItemKey::Stateful {});
// We need to detect recomputation, because here the content has not actually changed (even
// if it's not equal to the old content, as not all values implement Eq). We have to
// assume that tasks are deterministic and pure.
let should_invalidate = ctx.should_track_dependencies()
&& (task.has_key(&CachedDataItemKey::Dirty {}) ||
let assume_unchanged = !ctx.should_track_dependencies()
|| (!task.has_key(&CachedDataItemKey::Dirty {})
// This is a hack for the streaming hack. Stateful tasks are never recomputed, so this forces invalidation for them in case of this hack.
task.has_key(&CachedDataItemKey::Stateful {}));
&& !is_stateful);
let old_content = get!(task, CellData { cell });
if assume_unchanged {
if old_content.is_some() {
// Never update cells when recomputing if they already have a value.
// It's not expected that content changes during recomputation.
// Check if this assumption holds.
#[cfg(feature = "verify_determinism")]
if !is_stateful
&& matches!(verification_mode, VerificationMode::EqualityCheck)
&& content.as_ref() != old_content
{
let task_description = ctx.get_task_description(task_id);
let cell_type = turbo_tasks::registry::get_value_type(cell.type_id).global_name;
eprintln!(
"Task {} updated cell #{} (type: {}) while recomputing",
task_description, cell.index, cell_type
);
}
return;
} else {
// Initial computation, or computation after a cell has been cleared.
// We can just set the content, but we don't want to notify dependent tasks,
// as we assume that content hasn't changed (deterministic tasks).
}
} else {
// When not recomputing, we need to notify dependent tasks if the content actually
// changes.
if should_invalidate {
let dependent_tasks: SmallVec<[TaskId; 4]> = get_many!(
task,
CellDependent { cell: dependent_cell, task }
......@@ -78,12 +123,6 @@ impl UpdateCellOperation {
drop(task);
drop(old_content);
let content = if let CellContent(Some(new_content)) = content {
Some(new_content.into_typed(cell.type_id))
} else {
None
};
UpdateCellOperation::InvalidateWhenCellDependency {
cell_ref: CellRef {
task: task_id,
......@@ -101,8 +140,7 @@ impl UpdateCellOperation {
// Fast path: We don't need to invalidate anything.
// So we can just update the cell content.
let old_content = if let CellContent(Some(new_content)) = content {
let new_content = new_content.into_typed(cell.type_id);
let old_content = if let Some(new_content) = content {
task.insert(CachedDataItem::CellData {
cell,
value: new_content,
......
......@@ -679,9 +679,53 @@ fn serialize_task_type(
let deserialize: Result<CachedTaskType, _> = serde_path_to_error::deserialize(
&mut pot_de_symbol_list().deserializer_for_slice(&*task_type_bytes)?,
);
if let Err(err) = deserialize {
println!("Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}");
panic!("Task type would not be deserializable {task_id:?}: {err:?}");
match deserialize {
Err(err) => {
println!(
"Task type would not be deserializable {task_id:?}: {err:?}\n{task_type:#?}"
);
panic!("Task type would not be deserializable {task_id:?}: {err:?}");
}
Ok(task_type2) => {
if &task_type2 != task_type {
println!(
"Task type would not round-trip {task_id:?}:\noriginal: \
{task_type:#?}\nround-tripped: {task_type2:#?}"
);
panic!(
"Task type would not round-trip {task_id:?}:\noriginal: \
{task_type:#?}\nround-tripped: {task_type2:#?}"
);
}
let mut bytes2 = Vec::new();
let result2 = POT_CONFIG.serialize_into(&task_type2, &mut bytes2);
match result2 {
Err(err) => {
println!(
"Task type would not be serializable the second time {task_id:?}: \
{err:?}\n{task_type2:#?}"
);
panic!(
"Task type would not be serializable the second time {task_id:?}: \
{err:?}\n{task_type2:#?}"
);
}
Ok(()) => {
if bytes2 != *task_type_bytes {
println!(
"Task type would not serialize to the same bytes the second time \
{task_id:?}:\noriginal: {:x?}\nsecond: {:x?}\n{task_type2:#?}",
task_type_bytes, bytes2
);
panic!(
"Task type would not serialize to the same bytes the second time \
{task_id:?}:\noriginal: {:x?}\nsecond: {:x?}\n{task_type2:#?}",
task_type_bytes, bytes2
);
}
}
}
}
}
}
Ok(())
......@@ -757,7 +801,10 @@ fn serialize(task: TaskId, data: &Vec<CachedDataItem>) -> Result<SmallVec<[u8; 1
if let Err(err) = serde_path_to_error::serialize(&item, &mut serializer) {
if item.is_optional() {
#[cfg(feature = "verify_serialization")]
println!("Skipping non-serializable optional item for {task}: {item:?}");
println!(
"Skipping non-serializable optional item for {task}: {item:?} due to \
{err}"
);
} else {
error = Err(err).context({
anyhow!("Unable to serialize data item for {task}: {item:?}")
......
......@@ -18,6 +18,7 @@ static REGISTRATION: Registration = register!();
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_spawns_detached() -> anyhow::Result<()> {
run_once(&REGISTRATION, || async {
println!("test_spawns_detached");
// HACK: The watch channel we use has an incorrect implementation of `TraceRawVcs`, just
// disable GC for the test so this can't cause any problems.
prevent_gc();
......@@ -73,7 +74,9 @@ async fn spawns_detached(
sender: TransientInstance<WatchSenderTaskInput<Option<Vc<u32>>>>,
) -> Vc<()> {
tokio::spawn(turbo_tasks().detached_for_testing(Box::pin(async move {
println!("spawns_detached: waiting for notify");
notify.0.notified().await;
println!("spawns_detached: notified, sending value");
// creating cells after the normal lifetime of the task should be okay, as the parent task
// is waiting on us before exiting!
sender.0.send(Some(Vc::cell(42))).unwrap();
......
use std::{
io::{self, ErrorKind},
path::Path,
path::{Path, PathBuf},
};
use anyhow::{Context, Result, anyhow};
......@@ -19,10 +19,7 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio
}
}
#[cfg(not(target_os = "windows"))]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
use turbo_unix_path::sys_to_unix;
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
......@@ -33,32 +30,26 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
None => root,
};
let sys_path = root_fs.to_sys_path(&path);
Ok(uri_from_path_buf(root_fs.to_sys_path(&path)))
}
#[cfg(not(target_os = "windows"))]
pub fn uri_from_path_buf(sys_path: PathBuf) -> String {
use turbo_unix_path::sys_to_unix;
let sys_path = sys_path.to_string_lossy();
Ok(format!(
format!(
"file://{}",
sys_to_unix(&sys_path)
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
))
)
}
#[cfg(target_os = "windows")]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
let sys_path = match path {
Some(path) => root.join(path.into())?,
None => root,
};
let sys_path = root_fs.to_sys_path(&sys_path);
pub fn uri_from_path_buf(sys_path: PathBuf) -> String {
let raw_path = sys_path.to_string_lossy().to_string();
let normalized_path = raw_path.replace('\\', "/");
......@@ -70,7 +61,5 @@ pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<S
.collect::<Vec<_>>()
.join("/");
let uri = format!("file:///{}", encoded_path);
Ok(uri)
format!("file:///{}", encoded_path)
}
......@@ -18,7 +18,7 @@ use tokio::sync::mpsc::Receiver;
use turbo_tasks::{
CellId, ExecutionId, InvalidationReason, LocalTaskId, MagicAny, RawVc, ReadCellOptions,
ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId, TurboTasksApi, TurboTasksCallApi,
backend::{CellContent, TaskCollectiblesMap, TypedCellContent},
backend::{CellContent, TaskCollectiblesMap, TypedCellContent, VerificationMode},
event::{Event, EventListener},
message_queue::CompilationEvent,
test_helpers::with_turbo_tasks_for_testing,
......@@ -261,7 +261,13 @@ impl TurboTasksApi for VcStorage {
.into_typed(index.type_id))
}
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent) {
fn update_own_task_cell(
&self,
task: TaskId,
index: CellId,
content: CellContent,
_verification_mode: VerificationMode,
) {
let mut map = self.cells.lock().unwrap();
let cell = map.entry((task, index)).or_default();
*cell = content;
......
use std::{fmt::Debug, future::Future, sync::Arc};
use std::{env, fmt::Debug, future::Future, sync::Arc};
use anyhow::Result;
use turbo_tasks::{TurboTasksApi, trace::TraceRawVcs};
......@@ -98,22 +98,28 @@ where
F: Future<Output = Result<T>> + Send + 'static,
T: Debug + PartialEq + Eq + TraceRawVcs + Send + 'static,
{
let single_run = env::var("SINGLE_RUN").is_ok();
let name = closure_to_name(&fut);
let tt = registration.create_turbo_tasks(&name, true);
println!("Run #1 (without cache)");
let start = std::time::Instant::now();
let first = fut(tt.clone()).await?;
println!("Run #1 took {:?}", start.elapsed());
for i in 2..10 {
println!("Run #{i} (with memory cache, same TurboTasks instance)");
let start = std::time::Instant::now();
let second = fut(tt.clone()).await?;
println!("Run #{i} took {:?}", start.elapsed());
assert_eq!(first, second);
if !single_run {
for i in 2..10 {
println!("Run #{i} (with memory cache, same TurboTasks instance)");
let start = std::time::Instant::now();
let second = fut(tt.clone()).await?;
println!("Run #{i} took {:?}", start.elapsed());
assert_eq!(first, second);
}
}
let start = std::time::Instant::now();
tt.stop_and_wait().await;
println!("Stopping TurboTasks took {:?}", start.elapsed());
if single_run {
return Ok(());
}
for i in 10..20 {
let tt = registration.create_turbo_tasks(&name, false);
println!("Run #{i} (with filesystem cache if available, new TurboTasks instance)");
......
......@@ -497,6 +497,11 @@ impl From<anyhow::Error> for TurboTasksExecutionError {
}
}
pub enum VerificationMode {
EqualityCheck,
Skip,
}
pub trait Backend: Sync + Send {
#[allow(unused_variables)]
fn startup(&self, turbo_tasks: &dyn TurboTasksBackendApi<Self>) {}
......@@ -621,6 +626,7 @@ pub trait Backend: Sync + Send {
task: TaskId,
index: CellId,
content: CellContent,
verification_mode: VerificationMode,
turbo_tasks: &dyn TurboTasksBackendApi<Self>,
);
......
......@@ -25,7 +25,7 @@ use crate::{
VcValueTrait, VcValueType,
backend::{
Backend, CachedTaskType, CellContent, TaskCollectiblesMap, TaskExecutionSpec,
TransientTaskType, TurboTasksExecutionError, TypedCellContent,
TransientTaskType, TurboTasksExecutionError, TypedCellContent, VerificationMode,
},
capture_future::CaptureFuture,
event::{Event, EventListener},
......@@ -155,7 +155,13 @@ pub trait TurboTasksApi: TurboTasksCallApi + Sync + Send {
index: CellId,
options: ReadCellOptions,
) -> Result<TypedCellContent>;
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent);
fn update_own_task_cell(
&self,
task: TaskId,
index: CellId,
content: CellContent,
verification_mode: VerificationMode,
);
fn mark_own_task_as_finished(&self, task: TaskId);
fn set_own_task_aggregation_number(&self, task: TaskId, aggregation_number: u32);
fn mark_own_task_as_session_dependent(&self, task: TaskId);
......@@ -1354,8 +1360,15 @@ impl<B: Backend + 'static> TurboTasksApi for TurboTasks<B> {
self.try_read_own_task_cell(task, index, options)
}
fn update_own_task_cell(&self, task: TaskId, index: CellId, content: CellContent) {
self.backend.update_task_cell(task, index, content, self);
fn update_own_task_cell(
&self,
task: TaskId,
index: CellId,
content: CellContent,
verification_mode: VerificationMode,
) {
self.backend
.update_task_cell(task, index, content, verification_mode, self);
}
fn connect_task(&self, task: TaskId) {
......@@ -1769,7 +1782,12 @@ impl CurrentCellRef {
.ok();
let update = functor(cell_content.as_ref().and_then(|cc| cc.1.0.as_ref()));
if let Some(update) = update {
tt.update_own_task_cell(self.current_task, self.index, CellContent(Some(update)))
tt.update_own_task_cell(
self.current_task,
self.index,
CellContent(Some(update)),
VerificationMode::EqualityCheck,
)
}
}
......@@ -1851,7 +1869,7 @@ impl CurrentCellRef {
}
/// Unconditionally updates the content of the cell.
pub fn update<T>(&self, new_value: T)
pub fn update<T>(&self, new_value: T, verification_mode: VerificationMode)
where
T: VcValueType,
{
......@@ -1862,6 +1880,7 @@ impl CurrentCellRef {
CellContent(Some(SharedReference::new(triomphe::Arc::new(
<T::Read as VcRead<T>>::value_to_repr(new_value),
)))),
verification_mode,
)
}
......@@ -1873,27 +1892,40 @@ impl CurrentCellRef {
///
/// The [`SharedReference`] is expected to use the `<T::Read as
/// VcRead<T>>::Repr` type for its representation of the value.
pub fn update_with_shared_reference(&self, shared_ref: SharedReference) {
pub fn update_with_shared_reference(
&self,
shared_ref: SharedReference,
verification_mode: VerificationMode,
) {
let tt = turbo_tasks();
let content = tt
.read_own_task_cell(
self.current_task,
self.index,
ReadCellOptions {
// INVALIDATION: Reading our own cell must be untracked
tracking: ReadTracking::Untracked,
..Default::default()
},
)
.ok();
let update = if let Some(TypedCellContent(_, CellContent(Some(shared_ref_exp)))) = content {
// pointer equality (not value equality)
shared_ref_exp != shared_ref
let update = if matches!(verification_mode, VerificationMode::EqualityCheck) {
let content = tt
.read_own_task_cell(
self.current_task,
self.index,
ReadCellOptions {
// INVALIDATION: Reading our own cell must be untracked
tracking: ReadTracking::Untracked,
..Default::default()
},
)
.ok();
if let Some(TypedCellContent(_, CellContent(Some(shared_ref_exp)))) = content {
// pointer equality (not value equality)
shared_ref_exp != shared_ref
} else {
true
}
} else {
true
};
if update {
tt.update_own_task_cell(self.current_task, self.index, CellContent(Some(shared_ref)))
tt.update_own_task_cell(
self.current_task,
self.index,
CellContent(Some(shared_ref)),
verification_mode,
)
}
}
}
......
use std::{any::type_name, marker::PhantomData};
use super::{read::VcRead, traits::VcValueType};
use crate::{RawVc, Vc, manager::find_cell_by_type, task::shared_reference::TypedSharedReference};
use crate::{
RawVc, Vc, backend::VerificationMode, manager::find_cell_by_type,
task::shared_reference::TypedSharedReference,
};
type VcReadTarget<T> = <<T as VcValueType>::Read as VcRead<T>>::Target;
type VcReadRepr<T> = <<T as VcValueType>::Read as VcRead<T>>::Repr;
......@@ -37,7 +40,10 @@ where
{
fn cell(inner: VcReadTarget<T>) -> Vc<T> {
let cell = find_cell_by_type(T::get_value_type_id());
cell.update(<T::Read as VcRead<T>>::target_to_value(inner));
cell.update(
<T::Read as VcRead<T>>::target_to_value(inner),
VerificationMode::Skip,
);
Vc {
node: cell.into(),
_t: PhantomData,
......@@ -47,7 +53,7 @@ where
fn raw_cell(content: TypedSharedReference) -> RawVc {
debug_assert_repr::<T>(&content);
let cell = find_cell_by_type(content.type_id);
cell.update_with_shared_reference(content.reference);
cell.update_with_shared_reference(content.reference, VerificationMode::Skip);
cell.into()
}
}
......
......@@ -13,7 +13,7 @@ use turbopack_core::{
chunk::{
Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig,
ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAsset,
EvaluatableAssets, MinifyType, ModuleId, SourceMapsType,
EvaluatableAssets, MinifyType, ModuleId, SourceMapSourceType, SourceMapsType,
availability_info::AvailabilityInfo,
chunk_group::{MakeChunkGroupResult, make_chunk_group},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
......@@ -91,8 +91,8 @@ impl BrowserChunkingContextBuilder {
self
}
pub fn use_file_source_map_uris(mut self) -> Self {
self.chunking_context.should_use_file_source_map_uris = true;
pub fn source_map_source_type(mut self, source_map_source_type: SourceMapSourceType) -> Self {
self.chunking_context.source_map_source_type = source_map_source_type;
self
}
......@@ -229,8 +229,8 @@ pub struct BrowserChunkingContext {
name: Option<RcStr>,
/// The root path of the project
root_path: FileSystemPath,
/// Whether to write file sources as file:// paths in source maps
should_use_file_source_map_uris: bool,
/// The strategy to use for generating source map source uris
source_map_source_type: SourceMapSourceType,
/// This path is used to compute the url to request chunks from
output_root: FileSystemPath,
/// The relative path from the output_root to the root_path.
......@@ -311,7 +311,7 @@ impl BrowserChunkingContext {
client_root,
client_roots: Default::default(),
chunk_root_path,
should_use_file_source_map_uris: false,
source_map_source_type: SourceMapSourceType::TurbopackUri,
asset_root_path,
asset_root_paths: Default::default(),
chunk_base_path: None,
......@@ -617,8 +617,8 @@ impl ChunkingContext for BrowserChunkingContext {
}
#[turbo_tasks::function]
fn should_use_file_source_map_uris(&self) -> Vc<bool> {
Vc::cell(self.should_use_file_source_map_uris)
fn source_map_source_type(&self) -> Vc<SourceMapSourceType> {
self.source_map_source_type.cell()
}
#[turbo_tasks::function]
......
......@@ -6,7 +6,9 @@ use turbo_tasks_fs::FileSystemPath;
use turbopack_browser::{BrowserChunkingContext, react_refresh::assert_can_resolve_react_refresh};
use turbopack_cli_utils::runtime_entry::{RuntimeEntries, RuntimeEntry};
use turbopack_core::{
chunk::{ChunkableModule, ChunkingContext, EvaluatableAsset, SourceMapsType},
chunk::{
ChunkableModule, ChunkingContext, EvaluatableAsset, SourceMapSourceType, SourceMapsType,
},
environment::Environment,
file_source::FileSource,
module::Module,
......@@ -51,7 +53,7 @@ pub async fn get_client_chunking_context(
RuntimeType::Development,
)
.hot_module_replacement()
.use_file_source_map_uris()
.source_map_source_type(SourceMapSourceType::AbsoluteFileUri)
.dynamic_chunk_content_loading(true)
.build(),
))
......
use anyhow::Result;
use anyhow::{Result, bail};
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr;
......@@ -133,13 +133,22 @@ pub struct ChunkingConfig {
#[turbo_tasks::value(transparent)]
pub struct ChunkingConfigs(FxHashMap<ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig>);
#[turbo_tasks::value(shared)]
#[derive(Debug, Clone, Copy, Hash, TaskInput, Default)]
pub enum SourceMapSourceType {
AbsoluteFileUri,
RelativeUri,
#[default]
TurbopackUri,
}
/// A context for the chunking that influences the way chunks are created
#[turbo_tasks::value_trait]
pub trait ChunkingContext {
#[turbo_tasks::function]
fn name(self: Vc<Self>) -> Vc<RcStr>;
#[turbo_tasks::function]
fn should_use_file_source_map_uris(self: Vc<Self>) -> Vc<bool>;
fn source_map_source_type(self: Vc<Self>) -> Vc<SourceMapSourceType>;
/// The root path of the project
#[turbo_tasks::function]
fn root_path(self: Vc<Self>) -> Vc<FileSystemPath>;
......@@ -385,6 +394,13 @@ pub trait ChunkingContextExt {
) -> Vc<OutputAssetsWithReferenced>
where
Self: Send;
/// Computes the relative path from the chunk output root to the project root.
///
/// This is used to compute relative paths for source maps in certain configurations.
fn relative_path_from_chunk_root_to_project_root(self: Vc<Self>) -> Vc<RcStr>
where
Self: Send;
}
impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingContextExt for T {
......@@ -499,6 +515,52 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
availability_info,
)
}
fn relative_path_from_chunk_root_to_project_root(self: Vc<Self>) -> Vc<RcStr> {
relative_path_from_chunk_root_to_project_root(Vc::upcast_non_strict(self))
}
}
#[turbo_tasks::function]
async fn relative_path_from_chunk_root_to_project_root(
chunking_context: Vc<Box<dyn ChunkingContext>>,
) -> Result<Vc<RcStr>> {
// Example,
// project root: /project/root
// output root: /project/root/dist
// chunk root path: /project/root/dist/ssr/chunks
// output_root_to_chunk_root: ../
//
// Example2,
// project root: /project/root
// output root: /project/out
// chunk root path: /project/out/ssr/chunks
// output_root_to_chunk_root: ../root
//
// From that we want to return ../../../root to get from a path in `chunks` to a path in the
// project root.
let chunk_root_path = chunking_context.chunk_root_path().await?;
let output_root = chunking_context.output_root().await?;
let chunk_to_output_root = chunk_root_path.get_relative_path_to(&output_root);
let Some(chunk_to_output_root) = chunk_to_output_root else {
bail!(
"expected chunk_root_path: {chunk_root_path} to be inside of output_root: \
{output_root}",
chunk_root_path = chunk_root_path.value_to_string().await?,
output_root = output_root.value_to_string().await?
);
};
let output_root_to_chunk_root_path = chunking_context.output_root_to_root_path().await?;
// Note we cannot use `normalize_path` here since it rejects paths that start with `../`
Ok(Vc::cell(
format!(
"{}/{}",
chunk_to_output_root, output_root_to_chunk_root_path
)
.into(),
))
}
#[turbo_tasks::function]
......
......@@ -29,7 +29,8 @@ pub use self::{
},
chunking_context::{
ChunkGroupResult, ChunkGroupType, ChunkingConfig, ChunkingConfigs, ChunkingContext,
ChunkingContextExt, EntryChunkGroupResult, MangleType, MinifyType, SourceMapsType,
ChunkingContextExt, EntryChunkGroupResult, MangleType, MinifyType, SourceMapSourceType,
SourceMapsType,
},
data::{ChunkData, ChunkDataOption, ChunksData},
evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets},
......
......@@ -6,9 +6,10 @@ use once_cell::sync::Lazy;
use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, ValueToString};
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_file,
DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_path_buf,
};
use url::Url;
......@@ -71,6 +72,7 @@ struct SourceMapJson {
ignore_list: Option<Box<RawValue>>,
// A somewhat widespread non-standard extension
#[serde(skip_serializing_if = "Option::is_none")]
debug_id: Option<Box<RawValue>>,
#[serde(skip_serializing_if = "Option::is_none")]
......@@ -224,12 +226,18 @@ fn unencoded_str_to_raw_value(unencoded: &str) -> Box<RawValue> {
.expect("serde_json::to_string should produce valid JSON")
}
/// Turns `turbopack:///[project]` references in sourcemap sources into absolute `file://` uris. This
/// is useful for debugging environments.
pub async fn fileify_source_map(
/// Helper function to transform turbopack:/// file references in a sourcemap.
/// Handles parsing the sourcemap, resolving the filesystem, applying transformations, and
/// serializing back.
/// The transform function is given the source string as found in the sourcemap (i.e. a URI).
async fn transform_relative_files<F>(
map: Option<&Rope>,
context_path: FileSystemPath,
) -> Result<Option<Rope>> {
context_path: &FileSystemPath,
mut transform: F,
) -> Result<Option<Rope>>
where
F: FnMut(&DiskFileSystem, &str) -> Result<String>,
{
let Some(map) = map else {
return Ok(None);
};
......@@ -243,29 +251,75 @@ pub async fn fileify_source_map(
let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
.context("Expected the chunking context to have a DiskFileSystem")?
.await?;
let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL, context_fs.name());
let transform_source = async |src: &mut Option<String>| {
if let Some(src) = src
&& let Some(src_rest) = src.strip_prefix(&prefix)
{
*src = uri_from_file(context_path.clone(), Some(src_rest)).await?;
let mut apply_transform = |src: &mut String| -> Result<()> {
if let Some(src_rest) = src.strip_prefix(&prefix) {
*src = transform(context_fs, src_rest)?;
}
anyhow::Ok(())
Ok(())
};
for src in map.sources.iter_mut().flatten() {
transform_source(src).await?;
for src in map.sources.iter_mut().flatten().flatten() {
apply_transform(src)?;
}
for section in map.sections.iter_mut().flatten() {
for src in section.map.sources.iter_mut().flatten() {
transform_source(src).await?;
for src in section.map.sources.iter_mut().flatten().flatten() {
apply_transform(src)?;
}
}
let map = Rope::from(serde_json::to_vec(&map)?);
Ok(Some(Rope::from(serde_json::to_vec(&map)?)))
}
Ok(Some(map))
/// Turns `turbopack:///[project]` references in sourcemap sources into absolute `file://` uris. This
/// is useful for debugging environments.
pub async fn absolute_fileify_source_map(
map: Option<&Rope>,
context_path: FileSystemPath,
) -> Result<Option<Rope>> {
transform_relative_files(map, &context_path, |context_fs, src_rest| {
let path = context_path.join(src_rest)?;
Ok(uri_from_path_buf(context_fs.to_sys_path(&path)))
})
.await
}
fn uri_encode_path(path: &str) -> String {
path.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
}
/// Turns `turbopack:///[project]` references in sourcemap sources into relative './' prefixed uris.
/// This is useful in server environments and especially build environments.
pub async fn relative_fileify_source_map(
map: Option<&Rope>,
context_path: FileSystemPath,
relative_path_to_output_root: RcStr,
) -> Result<Option<Rope>> {
let relative_path_to_output_root = relative_path_to_output_root
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/");
transform_relative_files(map, &context_path, |_context_fs, src_rest| {
// NOTE: we just include the relative path prefix here instead of using `sourceRoot`
// since the spec on sourceRoot is broken.
// TODO(bgw): this shouldn't be necessary to uri encode since the strings we get out of the
// source map should already be uri encoded, however in the case of the turbopack scheme in
// particular we are inconsistent so be defensive here.
let src_rest = uri_encode_path(src_rest);
if relative_path_to_output_root.is_empty() {
Ok(src_rest.to_string())
} else {
Ok(format!("{relative_path_to_output_root}/{src_rest}",))
}
})
.await
}
#[cfg(test)]
......
......@@ -18,8 +18,8 @@ use turbopack_core::{
chunk::{
AsyncModuleInfo, Chunk, ChunkItem, ChunkItemBatchGroup, ChunkItemExt,
ChunkItemOrBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo, ChunkType,
ChunkableModule, ChunkingContext, MinifyType, OutputChunk, OutputChunkRuntimeInfo,
round_chunk_item_size,
ChunkableModule, ChunkingContext, ChunkingContextExt, MinifyType, OutputChunk,
OutputChunkRuntimeInfo, SourceMapSourceType, round_chunk_item_size,
},
code_builder::{Code, CodeBuilder},
ident::AssetIdent,
......@@ -32,7 +32,10 @@ use turbopack_core::{
output::{OutputAsset, OutputAssets},
reference_type::ImportContext,
server_fs::ServerFileSystem,
source_map::{GenerateSourceMap, OptionStringifiedSourceMap, utils::fileify_source_map},
source_map::{
GenerateSourceMap, OptionStringifiedSourceMap,
utils::{absolute_fileify_source_map, relative_fileify_source_map},
},
};
use self::{single_item_chunk::chunk::SingleItemCssChunk, source_map::CssChunkSourceMapAsset};
......@@ -96,18 +99,27 @@ impl CssChunk {
let close = write_import_context(&mut body, content.import_context).await?;
let source_map = if *self
.chunking_context()
.should_use_file_source_map_uris()
.await?
{
fileify_source_map(
content.source_map.as_ref(),
self.chunking_context().root_path().owned().await?,
)
.await?
} else {
content.source_map.clone()
let chunking_context = self.chunking_context();
let source_map = match *chunking_context.source_map_source_type().await? {
SourceMapSourceType::AbsoluteFileUri => {
absolute_fileify_source_map(
content.source_map.as_ref(),
chunking_context.root_path().owned().await?,
)
.await?
}
SourceMapSourceType::RelativeUri => {
relative_fileify_source_map(
content.source_map.as_ref(),
chunking_context.root_path().owned().await?,
chunking_context
.relative_path_from_chunk_root_to_project_root()
.owned()
.await?,
)
.await?
}
SourceMapSourceType::TurbopackUri => content.source_map.clone(),
};
body.push_source(&content.inner_code, source_map);
......
......@@ -3,18 +3,21 @@ use std::io::Write;
use anyhow::{Result, bail};
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
use turbo_rcstr::rcstr;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{
NonLocalValue, ResolvedVc, TaskInput, TryJoinIterExt, Upcast, ValueToString, Vc,
trace::TraceRawVcs,
};
use turbo_tasks_fs::{FileSystemPath, rope::Rope};
use turbopack_core::{
chunk::{AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext, ModuleId},
chunk::{
AsyncModuleInfo, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
ChunkingContextExt, ModuleId, SourceMapSourceType,
},
code_builder::{Code, CodeBuilder},
error::PrettyPrintError,
issue::{IssueExt, IssueSeverity, StyledString, code_gen::CodeGenerationIssue},
source_map::utils::fileify_source_map,
source_map::utils::{absolute_fileify_source_map, relative_fileify_source_map},
};
use crate::{
......@@ -24,6 +27,26 @@ use crate::{
utils::StringifyJs,
};
#[derive(
Debug,
Clone,
PartialEq,
Eq,
Hash,
Serialize,
Deserialize,
TraceRawVcs,
TaskInput,
NonLocalValue,
Default,
)]
pub enum RewriteSourcePath {
AbsoluteFilePath(FileSystemPath),
RelativeFilePath(FileSystemPath, RcStr),
#[default]
None,
}
#[turbo_tasks::value(shared)]
#[derive(Default, Clone)]
pub struct EcmascriptChunkItemContent {
......@@ -31,7 +54,7 @@ pub struct EcmascriptChunkItemContent {
pub source_map: Option<Rope>,
pub additional_ids: SmallVec<[ResolvedVc<ModuleId>; 1]>,
pub options: EcmascriptChunkItemOptions,
pub rewrite_source_path: Option<FileSystemPath>,
pub rewrite_source_path: RewriteSourcePath,
pub placeholder_for_future_extensions: (),
}
......@@ -53,10 +76,18 @@ impl EcmascriptChunkItemContent {
let strict = content.strict;
Ok(EcmascriptChunkItemContent {
rewrite_source_path: if *chunking_context.should_use_file_source_map_uris().await? {
Some(chunking_context.root_path().owned().await?)
} else {
None
rewrite_source_path: match *chunking_context.source_map_source_type().await? {
SourceMapSourceType::AbsoluteFileUri => {
RewriteSourcePath::AbsoluteFilePath(chunking_context.root_path().owned().await?)
}
SourceMapSourceType::RelativeUri => RewriteSourcePath::RelativeFilePath(
chunking_context.root_path().owned().await?,
chunking_context
.relative_path_from_chunk_root_to_project_root()
.owned()
.await?,
),
SourceMapSourceType::TurbopackUri => RewriteSourcePath::None,
},
inner_code: content.inner_code.clone(),
source_map: content.source_map.clone(),
......@@ -112,10 +143,19 @@ impl EcmascriptChunkItemContent {
)?;
}
let source_map = if let Some(rewrite_source_path) = &self.rewrite_source_path {
fileify_source_map(self.source_map.as_ref(), rewrite_source_path.clone()).await?
} else {
self.source_map.clone()
let source_map = match &self.rewrite_source_path {
RewriteSourcePath::AbsoluteFilePath(path) => {
absolute_fileify_source_map(self.source_map.as_ref(), path.clone()).await?
}
RewriteSourcePath::RelativeFilePath(path, relative_path) => {
relative_fileify_source_map(
self.source_map.as_ref(),
path.clone(),
relative_path.clone(),
)
.await?
}
RewriteSourcePath::None => self.source_map.clone(),
};
code.push_source(&self.inner_code, source_map);
......
......@@ -13,7 +13,7 @@ use crate::{
EcmascriptAnalyzableExt,
chunk::{
EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemOptions,
EcmascriptChunkPlaceable, EcmascriptChunkType,
EcmascriptChunkPlaceable, EcmascriptChunkType, item::RewriteSourcePath,
},
references::async_module::AsyncModuleOptions,
runtime_functions::{TURBOPACK_EXPORT_NAMESPACE, TURBOPACK_IMPORT},
......@@ -166,7 +166,7 @@ impl EcmascriptChunkItem for SideEffectsModuleChunkItem {
Ok(EcmascriptChunkItemContent {
inner_code: code,
source_map: None,
rewrite_source_path: None,
rewrite_source_path: RewriteSourcePath::None,
options: EcmascriptChunkItemOptions {
strict: true,
async_module: if has_top_level_await {
......
......@@ -16,8 +16,8 @@ use serde_json::Value as JsonValue;
use turbo_rcstr::rcstr;
use turbo_tasks::{
Completion, Effects, FxIndexMap, NonLocalValue, OperationVc, RawVc, ReadRef, ResolvedVc,
TaskInput, TryJoinIterExt, Vc, VcValueType, duration_span, fxindexmap, get_effects,
mark_finished, prevent_gc, trace::TraceRawVcs, util::SharedError,
TaskInput, TryJoinIterExt, Vc, VcValueType, backend::VerificationMode, duration_span,
fxindexmap, get_effects, mark_finished, prevent_gc, trace::TraceRawVcs, util::SharedError,
};
use turbo_tasks_bytes::{Bytes, Stream};
use turbo_tasks_env::{EnvMap, ProcessEnv};
......@@ -401,10 +401,10 @@ pub async fn custom_evaluate(
// We initialize the cell with a stream that is open, but has no values.
// The first [compute_evaluate_stream] pipe call will pick up that stream.
let (sender, receiver) = unbounded();
cell.update(JavaScriptEvaluation(JavaScriptStream::new_open(
vec![],
Box::new(receiver),
)));
cell.update(
JavaScriptEvaluation(JavaScriptStream::new_open(vec![], Box::new(receiver))),
VerificationMode::Skip,
);
let initial = Mutex::new(Some(sender));
// run the evaluation as side effect
......@@ -418,10 +418,13 @@ pub async fn custom_evaluate(
// In cases when only [compute_evaluate_stream] is (re)executed, we need to
// update the old stream with a new value.
let (sender, receiver) = unbounded();
cell.update(JavaScriptEvaluation(JavaScriptStream::new_open(
vec![],
Box::new(receiver),
)));
cell.update(
JavaScriptEvaluation(JavaScriptStream::new_open(
vec![],
Box::new(receiver),
)),
VerificationMode::Skip,
);
sender
}
}),
......
......@@ -8,7 +8,7 @@ use turbopack_core::{
chunk::{
Chunk, ChunkGroupResult, ChunkItem, ChunkType, ChunkableModule, ChunkingConfig,
ChunkingConfigs, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType,
ModuleId, SourceMapsType,
ModuleId, SourceMapSourceType, SourceMapsType,
availability_info::AvailabilityInfo,
chunk_group::{MakeChunkGroupResult, make_chunk_group},
module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy},
......@@ -99,8 +99,8 @@ impl NodeJsChunkingContextBuilder {
self
}
pub fn use_file_source_map_uris(mut self) -> Self {
self.chunking_context.should_use_file_source_map_uris = true;
pub fn source_map_source_type(mut self, source_map_source_type: SourceMapSourceType) -> Self {
self.chunking_context.source_map_source_type = source_map_source_type;
self
}
......@@ -182,8 +182,8 @@ pub struct NodeJsChunkingContext {
module_id_strategy: ResolvedVc<Box<dyn ModuleIdStrategy>>,
/// The module export usage info, if available.
export_usage: Option<ResolvedVc<ExportUsageInfo>>,
/// Whether to use file:// uris for source map sources
should_use_file_source_map_uris: bool,
/// The strategy to use for generating source map source uris
source_map_source_type: SourceMapSourceType,
/// The chunking configs
chunking_configs: Vec<(ResolvedVc<Box<dyn ChunkType>>, ChunkingConfig)>,
/// Enable debug IDs for chunks and source maps.
......@@ -222,7 +222,7 @@ impl NodeJsChunkingContext {
minify_type: MinifyType::NoMinify,
source_maps_type: SourceMapsType::Full,
manifest_chunks: false,
should_use_file_source_map_uris: false,
source_map_source_type: SourceMapSourceType::TurbopackUri,
module_id_strategy: ResolvedVc::upcast(DevModuleIdStrategy::new_resolved()),
export_usage: None,
chunking_configs: Default::default(),
......@@ -387,8 +387,8 @@ impl ChunkingContext for NodeJsChunkingContext {
}
#[turbo_tasks::function]
fn should_use_file_source_map_uris(&self) -> Vc<bool> {
Vc::cell(self.should_use_file_source_map_uris)
fn source_map_source_type(&self) -> Vc<SourceMapSourceType> {
self.source_map_source_type.cell()
}
#[turbo_tasks::function]
......
......@@ -33,7 +33,7 @@ use turbopack_core::{
asset::Asset,
chunk::{
ChunkingConfig, ChunkingContext, ChunkingContextExt, EvaluatableAsset, EvaluatableAssetExt,
EvaluatableAssets, MinifyType, availability_info::AvailabilityInfo,
EvaluatableAssets, MinifyType, SourceMapSourceType, availability_info::AvailabilityInfo,
},
compile_time_defines,
compile_time_info::{CompileTimeDefineValue, CompileTimeInfo, DefinableNameSegment},
......@@ -91,6 +91,8 @@ struct SnapshotOptions {
production_chunking: bool,
#[serde(default)]
enable_debug_ids: bool,
#[serde(default)]
source_map_source_type: SourceMapSourceType,
}
#[derive(Debug, Deserialize, Default)]
......@@ -121,6 +123,7 @@ impl Default for SnapshotOptions {
scope_hoisting: false,
production_chunking: false,
enable_debug_ids: false,
source_map_source_type: SourceMapSourceType::default(),
}
}
}
......@@ -460,7 +463,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.minify_type(options.minify_type)
.module_merging(options.scope_hoisting)
.export_usage(export_usage)
.debug_ids(options.enable_debug_ids);
.debug_ids(options.enable_debug_ids)
.source_map_source_type(options.source_map_source_type);
if options.production_chunking {
builder = builder.chunking_config(
......@@ -489,7 +493,8 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> {
.minify_type(options.minify_type)
.module_merging(options.scope_hoisting)
.export_usage(export_usage)
.debug_ids(options.enable_debug_ids);
.debug_ids(options.enable_debug_ids)
.source_map_source_type(options.source_map_source_type);
if options.production_chunking {
builder = builder.chunking_config(
......
......@@ -3,5 +3,6 @@
"runtimeType": "Development",
"minifyType": "NoMinify",
"environment": "NodeJs",
"enableDebugIds": true
"enableDebugIds": true,
"sourceMapSourceType": "RelativeUri"
}
......@@ -3,5 +3,5 @@
"sources": [],
"debugId": "3d97c2d1-90e5-a3ef-c710-5c78b21b3b3a",
"sections": [
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["turbopack:///[project]/turbopack/crates/turbopack-tests/tests/snapshot/debug-ids/node/input/index.js"],"sourcesContent":["console.log('Hello World')\n"],"names":[],"mappings":"AAAA,QAAQ,GAAG,CAAC"}}]
{"offset": {"line": 4, "column": 0}, "map": {"version":3,"sources":["../../../../../../../../turbopack/crates/turbopack-tests/tests/snapshot/debug-ids/node/input/index.js"],"sourcesContent":["console.log('Hello World')\n"],"names":[],"mappings":"AAAA,QAAQ,GAAG,CAAC"}}]
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment