Commit 5f6f616a authored by Benjamin Woodruff's avatar Benjamin Woodruff

Turbopack: Always encode `turbopack:///[fs]/` urls as valid URLs

parent b90559cd
...@@ -4313,6 +4313,7 @@ dependencies = [ ...@@ -4313,6 +4313,7 @@ dependencies = [
"turbopack-nodejs", "turbopack-nodejs",
"turbopack-static", "turbopack-static",
"turbopack-trace-utils", "turbopack-trace-utils",
"turbopack-url",
] ]
[[package]] [[package]]
...@@ -4880,9 +4881,9 @@ dependencies = [ ...@@ -4880,9 +4881,9 @@ dependencies = [
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.3.1" version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]] [[package]]
name = "pest" name = "pest"
...@@ -9284,6 +9285,7 @@ dependencies = [ ...@@ -9284,6 +9285,7 @@ dependencies = [
"mime", "mime",
"notify", "notify",
"parking_lot", "parking_lot",
"percent-encoding",
"regex", "regex",
"rstest", "rstest",
"rustc-hash 2.1.1", "rustc-hash 2.1.1",
...@@ -9302,6 +9304,7 @@ dependencies = [ ...@@ -9302,6 +9304,7 @@ dependencies = [
"turbo-tasks-hash", "turbo-tasks-hash",
"turbo-tasks-testing", "turbo-tasks-testing",
"turbo-unix-path", "turbo-unix-path",
"url",
"urlencoding", "urlencoding",
] ]
...@@ -9541,6 +9544,7 @@ dependencies = [ ...@@ -9541,6 +9544,7 @@ dependencies = [
"indexmap 2.9.0", "indexmap 2.9.0",
"once_cell", "once_cell",
"patricia_tree", "patricia_tree",
"percent-encoding",
"petgraph 0.6.3", "petgraph 0.6.3",
"ref-cast", "ref-cast",
"regex", "regex",
...@@ -9564,6 +9568,7 @@ dependencies = [ ...@@ -9564,6 +9568,7 @@ dependencies = [
"turbo-tasks-hash", "turbo-tasks-hash",
"turbo-tasks-testing", "turbo-tasks-testing",
"turbo-unix-path", "turbo-unix-path",
"turbopack-url",
"url", "url",
"urlencoding", "urlencoding",
] ]
...@@ -10026,6 +10031,14 @@ dependencies = [ ...@@ -10026,6 +10031,14 @@ dependencies = [
"turbopack-resolve", "turbopack-resolve",
] ]
[[package]]
name = "turbopack-url"
version = "0.1.0"
dependencies = [
"percent-encoding",
"url",
]
[[package]] [[package]]
name = "turbopack-wasm" name = "turbopack-wasm"
version = "0.1.0" version = "0.1.0"
......
...@@ -305,6 +305,7 @@ turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" } ...@@ -305,6 +305,7 @@ turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" }
turbopack-test-utils = { path = "turbopack/crates/turbopack-test-utils" } turbopack-test-utils = { path = "turbopack/crates/turbopack-test-utils" }
turbopack-trace-server = { path = "turbopack/crates/turbopack-trace-server" } turbopack-trace-server = { path = "turbopack/crates/turbopack-trace-server" }
turbopack-trace-utils = { path = "turbopack/crates/turbopack-trace-utils" } turbopack-trace-utils = { path = "turbopack/crates/turbopack-trace-utils" }
turbopack-url = { path = "turbopack/crates/turbopack-url" }
turbopack-wasm = { path = "turbopack/crates/turbopack-wasm" } turbopack-wasm = { path = "turbopack/crates/turbopack-wasm" }
# SWC crates # SWC crates
...@@ -398,6 +399,7 @@ owo-colors = "4.2.2" ...@@ -398,6 +399,7 @@ owo-colors = "4.2.2"
parcel_selectors = "0.28.2" parcel_selectors = "0.28.2"
parking_lot = "0.12.1" parking_lot = "0.12.1"
pathdiff = "0.2.1" pathdiff = "0.2.1"
percent-encoding = "2.3.2"
petgraph = "0.6.3" petgraph = "0.6.3"
pin-project-lite = "0.2.9" pin-project-lite = "0.2.9"
postcard = "1.0.4" postcard = "1.0.4"
......
use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration}; use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
use anyhow::{Context, Result, anyhow, bail}; use anyhow::{Context, Result, anyhow, bail};
use futures_util::TryFutureExt; use futures_util::TryFutureExt;
...@@ -24,7 +24,6 @@ use next_core::tracing_presets::{ ...@@ -24,7 +24,6 @@ use next_core::tracing_presets::{
TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBO_TASKS_TARGETS, TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBO_TASKS_TARGETS,
TRACING_NEXT_TURBOPACK_TARGETS, TRACING_NEXT_TURBOPACK_TARGETS,
}; };
use once_cell::sync::Lazy;
use rand::Rng; use rand::Rng;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::{io::AsyncWriteExt, runtime::Handle, time::Instant}; use tokio::{io::AsyncWriteExt, runtime::Handle, time::Instant};
...@@ -38,17 +37,17 @@ use turbo_tasks::{ ...@@ -38,17 +37,17 @@ use turbo_tasks::{
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons}; use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
use turbo_tasks_fs::{ use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file, use turbo_unix_path::sys_to_unix;
};
use turbo_unix_path::{get_relative_path_to, sys_to_unix};
use turbopack_core::{ use turbopack_core::{
PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL, PROJECT_FILESYSTEM_NAME,
diagnostics::PlainDiagnostic, diagnostics::PlainDiagnostic,
error::PrettyPrintError, error::PrettyPrintError,
issue::PlainIssue, issue::PlainIssue,
output::{OutputAsset, OutputAssets}, output::{OutputAsset, OutputAssets},
source_map::{OptionStringifiedSourceMap, SourceMap, Token}, source_map::{
OptionStringifiedSourceMap, SourceMap, Token, utils::turbopack_url_to_fs_relative_file_url,
},
version::{PartialUpdate, TotalUpdate, Update, VersionState}, version::{PartialUpdate, TotalUpdate, Update, VersionState},
}; };
use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, ResourceIdentifier}; use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, ResourceIdentifier};
...@@ -78,9 +77,6 @@ use crate::{ ...@@ -78,9 +77,6 @@ use crate::{
/// Used by [`benchmark_file_io`]. This is a noisy benchmark, so set the /// Used by [`benchmark_file_io`]. This is a noisy benchmark, so set the
/// threshold high. /// threshold high.
const SLOW_FILESYSTEM_THRESHOLD: Duration = Duration::from_millis(100); const SLOW_FILESYSTEM_THRESHOLD: Duration = Duration::from_millis(100);
static SOURCE_MAP_PREFIX: Lazy<String> = Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///"));
static SOURCE_MAP_PREFIX_PROJECT: Lazy<String> =
Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///[{PROJECT_FILESYSTEM_NAME}]/"));
#[napi(object)] #[napi(object)]
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
...@@ -1479,10 +1475,10 @@ pub fn get_source_map_rope_operation( ...@@ -1479,10 +1475,10 @@ pub fn get_source_map_rope_operation(
pub async fn project_trace_source_operation( pub async fn project_trace_source_operation(
container: ResolvedVc<ProjectContainer>, container: ResolvedVc<ProjectContainer>,
frame: StackFrame, frame: StackFrame,
current_directory_file_url: RcStr,
) -> Result<Vc<OptionStackFrame>> { ) -> Result<Vc<OptionStackFrame>> {
let Some(map) = let Some(map) =
&*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file)).await? &*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file.clone()))
.await?
else { else {
return Ok(Vc::cell(None)); return Ok(Vc::cell(None));
}; };
...@@ -1496,69 +1492,48 @@ pub async fn project_trace_source_operation( ...@@ -1496,69 +1492,48 @@ pub async fn project_trace_source_operation(
frame.column.unwrap_or(1).saturating_sub(1), frame.column.unwrap_or(1).saturating_sub(1),
); );
let (original_file, line, column, method_name) = match token { let (original_url, line, column, method_name) = match token {
Token::Original(token) => ( Token::Original(token) => (
match urlencoding::decode(&token.original_file)? { token.original_file,
Cow::Borrowed(_) => token.original_file,
Cow::Owned(original_file) => RcStr::from(original_file),
},
// JS stack frames are 1-indexed, source map tokens are 0-indexed // JS stack frames are 1-indexed, source map tokens are 0-indexed
Some(token.original_line + 1), Some(token.original_line + 1),
Some(token.original_column + 1), Some(token.original_column + 1),
token.name, token.name,
), ),
Token::Synthetic(token) => { Token::Synthetic(token) => {
let Some(original_file) = token.guessed_original_file else { let Some(original_url) = token.guessed_original_file else {
return Ok(Vc::cell(None)); return Ok(Vc::cell(None));
}; };
(original_file, None, None, None) (original_url, None, None, None)
} }
}; };
let project_root_uri = let url_schema = original_url.split_once(":").map(|(schema, _)| schema);
uri_from_file(container.project().project_root_path().owned().await?, None).await? + "/"; let (original_url, is_internal) = if let Some(url_schema) = url_schema
let (file, original_file, is_internal) = && url_schema.eq_ignore_ascii_case("turbopack")
if let Some(source_file) = original_file.strip_prefix(&project_root_uri) { {
// Client code uses file:// if let Some(relative_url) =
( turbopack_url_to_fs_relative_file_url(&original_url, PROJECT_FILESYSTEM_NAME)
RcStr::from( {
get_relative_path_to(&current_directory_file_url, &original_file) // Server code uses `turbopack:///[project]` urls?
// TODO(sokra) remove this to include a ./ here to make it a relative path // this returns a relative URL, but should it be an absolute file:// url?
.trim_start_matches("./"), //
), // If it's absolute, that conversion should happen outside of the turbo-task, as that
Some(RcStr::from(source_file)), // is not safe to store in the persistent cache (not portable across machines)
false, (Some(RcStr::from(relative_url)), false)
) } else {
} else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX_PROJECT) {
// Server code uses turbopack:///[project]
// TODO should this also be file://?
(
RcStr::from(
get_relative_path_to(
&current_directory_file_url,
&format!("{project_root_uri}{source_file}"),
)
// TODO(sokra) remove this to include a ./ here to make it a relative path
.trim_start_matches("./"),
),
Some(RcStr::from(source_file)),
false,
)
} else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX) {
// All other code like turbopack:///[turbopack] is internal code // All other code like turbopack:///[turbopack] is internal code
// TODO(veil): Should the protocol be preserved? (None, true)
(RcStr::from(source_file), None, true) }
} else { } else {
bail!( // Any other URL... Likely a relative or absolute file:// url. Just pass it through.
"Original file ({}) outside project ({})", // Client code uses `file://` urls?
original_file, (Some(RcStr::from(original_url)), false)
project_root_uri
)
}; };
Ok(Vc::cell(Some(StackFrame { Ok(Vc::cell(Some(StackFrame {
file, file: frame.file,
original_file, original_file: original_url,
method_name, method_name,
line, line,
column, column,
...@@ -1572,17 +1547,12 @@ pub async fn project_trace_source_operation( ...@@ -1572,17 +1547,12 @@ pub async fn project_trace_source_operation(
pub async fn project_trace_source( pub async fn project_trace_source(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>, #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
frame: StackFrame, frame: StackFrame,
current_directory_file_url: String,
) -> napi::Result<Option<StackFrame>> { ) -> napi::Result<Option<StackFrame>> {
let container = project.container; let container = project.container;
let ctx = &project.turbopack_ctx; let ctx = &project.turbopack_ctx;
ctx.turbo_tasks() ctx.turbo_tasks()
.run(async move { .run(async move {
let traced_frame = project_trace_source_operation( let traced_frame = project_trace_source_operation(container, frame)
container,
frame,
RcStr::from(current_directory_file_url),
)
.read_strongly_consistent() .read_strongly_consistent()
.await?; .await?;
Ok(ReadRef::into_owned(traced_frame)) Ok(ReadRef::into_owned(traced_frame))
......
...@@ -682,6 +682,7 @@ impl Project { ...@@ -682,6 +682,7 @@ impl Project {
DiskFileSystem::new(rcstr!("output"), self.root_path.clone()) DiskFileSystem::new(rcstr!("output"), self.root_path.clone())
} }
// TODO: This should not be a turbo-task function as it is not portable across machines.
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn dist_dir_absolute(&self) -> Result<Vc<RcStr>> { pub fn dist_dir_absolute(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell( Ok(Vc::cell(
......
...@@ -33,7 +33,7 @@ rustc-hash = { workspace = true } ...@@ -33,7 +33,7 @@ rustc-hash = { workspace = true }
react_remove_properties = { workspace = true } react_remove_properties = { workspace = true }
remove_console = { workspace = true } remove_console = { workspace = true }
itertools = { workspace = true } itertools = { workspace = true }
percent-encoding = "2.3.1" percent-encoding = { workspace = true }
serde_path_to_error = { workspace = true } serde_path_to_error = { workspace = true }
swc_sourcemap = { workspace = true } swc_sourcemap = { workspace = true }
...@@ -80,6 +80,7 @@ turbopack-node = { workspace = true } ...@@ -80,6 +80,7 @@ turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true } turbopack-nodejs = { workspace = true }
turbopack-static = { workspace = true } turbopack-static = { workspace = true }
turbopack-trace-utils = { workspace = true } turbopack-trace-utils = { workspace = true }
turbopack-url = { workspace = true }
[features] [features]
next-font-local = [] next-font-local = []
......
...@@ -2,6 +2,7 @@ use anyhow::Result; ...@@ -2,6 +2,7 @@ use anyhow::Result;
use either::Either; use either::Either;
use indoc::formatdoc; use indoc::formatdoc;
use itertools::Itertools; use itertools::Itertools;
use percent_encoding::utf8_percent_encode;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use serde::Serialize; use serde::Serialize;
use tracing::Instrument; use tracing::Instrument;
...@@ -19,13 +20,14 @@ use turbopack_core::{ ...@@ -19,13 +20,14 @@ use turbopack_core::{
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced}, output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
}; };
use turbopack_ecmascript::utils::StringifyJs; use turbopack_ecmascript::utils::StringifyJs;
use turbopack_url::ENCODE_URI_COMPONENT;
use crate::{ use crate::{
mode::NextMode, mode::NextMode,
next_app::ClientReferencesChunks, next_app::ClientReferencesChunks,
next_client_reference::{ClientReferenceGraphResult, ClientReferenceType}, next_client_reference::{ClientReferenceGraphResult, ClientReferenceType},
next_config::{CrossOriginConfig, NextConfig}, next_config::{CrossOriginConfig, NextConfig},
next_manifests::{ModuleId, encode_uri_component::encode_uri_component}, next_manifests::ModuleId,
util::NextRuntime, util::NextRuntime,
}; };
...@@ -290,7 +292,9 @@ async fn build_manifest( ...@@ -290,7 +292,9 @@ async fn build_manifest(
format!( format!(
"{}{}{}", "{}{}{}",
prefix_path, prefix_path,
path.split('/').map(encode_uri_component).format("/"), path.split('/')
.map(|segment| utf8_percent_encode(segment, ENCODE_URI_COMPONENT))
.format("/"),
suffix_path suffix_path
) )
}) })
......
//! Type definitions for the Next.js manifest formats. //! Type definitions for the Next.js manifest formats.
pub mod client_reference_manifest; pub mod client_reference_manifest;
mod encode_uri_component;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
......
...@@ -241,8 +241,7 @@ export interface Project { ...@@ -241,8 +241,7 @@ export interface Project {
getSourceMapSync(filePath: string): string | null getSourceMapSync(filePath: string): string | null
traceSource( traceSource(
stackFrame: TurbopackStackFrame, stackFrame: TurbopackStackFrame
currentDirectoryFileUrl: string
): Promise<TurbopackStackFrame | null> ): Promise<TurbopackStackFrame | null>
updateInfoSubscribe( updateInfoSubscribe(
......
...@@ -43,10 +43,6 @@ async function batchedTraceSource( ...@@ -43,10 +43,6 @@ async function batchedTraceSource(
frame: TurbopackStackFrame frame: TurbopackStackFrame
): Promise<{ frame: IgnorableStackFrame; source: string | null } | undefined> { ): Promise<{ frame: IgnorableStackFrame; source: string | null } | undefined> {
const file = frame.file const file = frame.file
? // TODO(veil): Why are the frames sent encoded?
decodeURIComponent(frame.file)
: undefined
if (!file) return if (!file) return
// For node internals they cannot traced the actual source code with project.traceSource, // For node internals they cannot traced the actual source code with project.traceSource,
...@@ -65,9 +61,7 @@ async function batchedTraceSource( ...@@ -65,9 +61,7 @@ async function batchedTraceSource(
} }
} }
const currentDirectoryFileUrl = pathToFileURL(process.cwd()).href const sourceFrame = await project.traceSource(frame)
const sourceFrame = await project.traceSource(frame, currentDirectoryFileUrl)
if (!sourceFrame) { if (!sourceFrame) {
return { return {
frame: { frame: {
......
...@@ -37,6 +37,7 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] } ...@@ -37,6 +37,7 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] }
mime = { workspace = true } mime = { workspace = true }
notify = { workspace = true } notify = { workspace = true }
parking_lot = { workspace = true } parking_lot = { workspace = true }
percent-encoding = { workspace = true }
regex = { workspace = true } regex = { workspace = true }
rustc-hash = { workspace = true } rustc-hash = { workspace = true }
serde = { workspace = true, features = ["rc"] } serde = { workspace = true, features = ["rc"] }
...@@ -50,6 +51,7 @@ turbo-rcstr = { workspace = true } ...@@ -50,6 +51,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true } turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true } turbo-unix-path = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
[dev-dependencies] [dev-dependencies]
......
...@@ -511,6 +511,10 @@ impl DiskFileSystem { ...@@ -511,6 +511,10 @@ impl DiskFileSystem {
}) })
} }
pub fn root_sys_path(&self) -> &Path {
self.inner.root_path()
}
pub fn to_sys_path(&self, fs_path: FileSystemPath) -> Result<PathBuf> { pub fn to_sys_path(&self, fs_path: FileSystemPath) -> Result<PathBuf> {
let path = self.inner.root_path(); let path = self.inner.root_path();
Ok(if fs_path.path.is_empty() { Ok(if fs_path.path.is_empty() {
......
...@@ -3,10 +3,7 @@ use std::{ ...@@ -3,10 +3,7 @@ use std::{
path::Path, path::Path,
}; };
use anyhow::{Context, Result, anyhow}; use anyhow::{Result, anyhow};
use turbo_tasks::ResolvedVc;
use crate::{DiskFileSystem, FileSystemPath};
/// Converts a disk access Result<T> into a Result<Some<T>>, where a NotFound /// Converts a disk access Result<T> into a Result<Some<T>>, where a NotFound
/// error results in a None value. This is purely to reduce boilerplate code /// error results in a None value. This is purely to reduce boilerplate code
...@@ -18,57 +15,3 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio ...@@ -18,57 +15,3 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio
Err(e) => Err(anyhow!(e).context(format!("reading file {}", path.display()))), Err(e) => Err(anyhow!(e).context(format!("reading file {}", path.display()))),
} }
} }
#[cfg(not(target_os = "windows"))]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
use turbo_unix_path::sys_to_unix;
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
Ok(format!(
"file://{}",
&sys_to_unix(
&root_fs
.to_sys_path(match path {
Some(path) => root.join(path)?,
None => root,
})?
.to_string_lossy()
)
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
))
}
#[cfg(target_os = "windows")]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
let sys_path = root_fs.to_sys_path(match path {
Some(path) => root.join(path.into())?,
None => root,
})?;
let raw_path = sys_path.to_string_lossy().to_string();
let normalized_path = raw_path.replace('\\', "/");
let mut segments = normalized_path.split('/');
let first = segments.next().unwrap_or_default(); // e.g., "C:"
let encoded_path = std::iter::once(first.to_string()) // keep "C:" intact
.chain(segments.map(|s| urlencoding::encode(s).into_owned()))
.collect::<Vec<_>>()
.join("/");
let uri = format!("file:///{}", encoded_path);
Ok(uri)
}
...@@ -24,6 +24,7 @@ either = { workspace = true } ...@@ -24,6 +24,7 @@ either = { workspace = true }
indexmap = { workspace = true } indexmap = { workspace = true }
once_cell = { workspace = true } once_cell = { workspace = true }
patricia_tree = "0.5.5" patricia_tree = "0.5.5"
percent-encoding = { workspace = true }
petgraph = { workspace = true, features = ["serde-1"] } petgraph = { workspace = true, features = ["serde-1"] }
roaring = { workspace = true, features = ["serde"] } roaring = { workspace = true, features = ["serde"] }
ref-cast = "1.0.20" ref-cast = "1.0.20"
...@@ -43,10 +44,10 @@ turbo-tasks-env = { workspace = true } ...@@ -43,10 +44,10 @@ turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true } turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true } turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true } turbo-unix-path = { workspace = true }
turbopack-url = { workspace = true }
url = { workspace = true } url = { workspace = true }
urlencoding = { workspace = true } urlencoding = { workspace = true }
[dev-dependencies] [dev-dependencies]
rstest = { workspace = true } rstest = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
......
...@@ -3,13 +3,13 @@ use std::{borrow::Cow, collections::HashSet, iter}; ...@@ -3,13 +3,13 @@ use std::{borrow::Cow, collections::HashSet, iter};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use const_format::concatcp; use const_format::concatcp;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use percent_encoding::{percent_decode_str, utf8_percent_encode};
use regex::Regex; use regex::Regex;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::value::RawValue; use serde_json::value::RawValue;
use turbo_tasks::{ResolvedVc, ValueToString}; use turbo_tasks::{ResolvedVc, ValueToString};
use turbo_tasks_fs::{ use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystemPath, rope::Rope};
DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_file, use turbopack_url::{SPECIAL_PATH_SEGMENT, percent_encode_str};
};
use url::Url; use url::Url;
use crate::SOURCE_URL_PROTOCOL; use crate::SOURCE_URL_PROTOCOL;
...@@ -132,11 +132,17 @@ pub async fn resolve_source_map_sources( ...@@ -132,11 +132,17 @@ pub async fn resolve_source_map_sources(
}; };
if let Some(fs_path) = fs_path { if let Some(fs_path) = fs_path {
// TODO: Encode `fs_str` and `fs_path_str` using `urlencoding`, so that these are
// valid URLs. However, `project_trace_source_operation` (and `uri_from_file`) need
// to handle percent encoding correctly first.
let fs_path_str = &fs_path.path; let fs_path_str = &fs_path.path;
*source_url = format!("{SOURCE_URL_PROTOCOL}///{fs_str}/{fs_path_str}"); *source_url = format!(
"{}///{}/{}",
SOURCE_URL_PROTOCOL,
utf8_percent_encode(fs_str, SPECIAL_PATH_SEGMENT),
fs_path_str
.split('/')
.map(|segment| percent_encode_str(segment, SPECIAL_PATH_SEGMENT))
.collect::<Vec<_>>()
.join("/"),
);
if let Some(source_content) = source_content if let Some(source_content) = source_content
&& source_content.is_none() && source_content.is_none()
...@@ -239,18 +245,40 @@ pub async fn fileify_source_map( ...@@ -239,18 +245,40 @@ pub async fn fileify_source_map(
return Ok(None); return Ok(None);
}; };
// read and unset `source_root`: we flatten URLs and should never emit `source_root`.
let source_root = map.source_root.take();
let context_fs = context_path.fs; let context_fs = context_path.fs;
let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs) let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
.context("Expected the chunking context to have a DiskFileSystem")? .context("Expected the chunking context to have a DiskFileSystem")?
.await?; .await?;
let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL, context_fs.name()); let context_fs_name = context_fs.name();
let context_base_url = Url::from_directory_path(context_fs.root_sys_path()).ok();
debug_assert_ne!(
context_base_url, None,
"failed to convert context_fs root to URL"
);
let transform_source = async |src: &mut Option<String>| { let transform_source = async |src: &mut Option<String>| {
if let Some(src) = src let Some(src) = src else { return Ok(()) };
&& let Some(src_rest) = src.strip_prefix(&prefix)
{ if let Some(root) = &source_root {
*src = uri_from_file(context_path.clone(), Some(src_rest)).await?; // flatten, regardless of url format. We unset `map.source_root` above.
*src = format!("{root}{src}");
} }
let Some(context_base_url) = &context_base_url else {
return anyhow::Ok(());
};
let Some(file_url) =
turbopack_url_to_file_url(src, context_fs_name, context_base_url.as_str())
else {
return anyhow::Ok(());
};
*src = file_url;
anyhow::Ok(()) anyhow::Ok(())
}; };
...@@ -268,6 +296,61 @@ pub async fn fileify_source_map( ...@@ -268,6 +296,61 @@ pub async fn fileify_source_map(
Ok(Some(map)) Ok(Some(map))
} }
/// Takes a `turbopack:///[fs_name]/` url and strips the prefix, leaving a percent-encoded relative
/// path string that's valid as a suffix in `file://` URLs.
pub fn turbopack_url_to_fs_relative_file_url<'a>(
turbopack_url: &'a str,
fs_name: &str,
) -> Option<&'a str> {
let (scheme, remaining_url) = turbopack_url.split_once(":")?;
// URL schemes don't need percent decoding, just case insensitive equality is sufficient
if !scheme.eq_ignore_ascii_case(&SOURCE_URL_PROTOCOL[..SOURCE_URL_PROTOCOL.len() - 1]) {
return None;
}
if cfg!(debug_assertions) {
let url = Url::parse(turbopack_url).expect("turbopack:// urls should parse as valid URLs");
assert_eq!(
url.host_str(),
None,
"turbopack:// URLs should not have a host",
);
}
// Modify the URL using string manipulation, as we can make some assumptions about the
// format of `turbopack://` urls, so it's cheaper.
let remaining_url = remaining_url.strip_prefix("///")?;
let (decoded_fs_name, remaining_url) =
remaining_url.split_once("/").unwrap_or((remaining_url, ""));
let decoded_fs_name = percent_decode_str(decoded_fs_name).decode_utf8().ok()?;
let decoded_fs_name_suffix = decoded_fs_name
.strip_prefix("[")?
.strip_prefix(fs_name)?
.strip_prefix("]")?;
if !decoded_fs_name_suffix.is_empty() {
if cfg!(debug_assertions) {
panic!("{turbopack_url} has a malformed filesystem name ({decoded_fs_name})");
}
return None;
}
Some(remaining_url)
}
/// Return an absolute URL contructed from `turbopack_url`, assuming `fs_name` matches.
///
/// Use [`Url::from_directory_path`] to convert a [`DiskFileSystem::root_sys_path`] to a `base_url`.
fn turbopack_url_to_file_url(turbopack_url: &str, fs_name: &str, base_url: &str) -> Option<String> {
// Both URL formats use the same percent encoding
Some(format!(
"{}{}",
base_url,
turbopack_url_to_fs_relative_file_url(turbopack_url, fs_name)?,
))
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::path::Path; use std::path::Path;
......
[package]
name = "turbopack-url"
version = "0.1.0"
description = "TBD"
license = "MIT"
edition = "2024"
[lib]
bench = false
[lints]
workspace = true
[dependencies]
url = { workspace = true }
percent-encoding = { workspace = true }
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; pub mod url_spec;
/// An `AsciiSet` that matches the behavior of JavaScript's `encodeURIComponent`. use std::borrow::Cow;
/// - It leaves `A-Z a-z 0-9 - _ . ~` unescaped.
/// - It percent-encodes all other ASCII characters (and of course non-ASCII).
/// - The `CONTROLS` set covers `\0`-`\x1F` and `\x7F`.
const ENCODE_URI_COMPONENT_SET: &AsciiSet = &CONTROLS
// Add everything else JS `encodeURIComponent` would encode:
.add(b' ')
// .add(b'!')
.add(b'"')
.add(b'#')
.add(b'$')
.add(b'%')
.add(b'&')
.add(b'\'')
// .add(b'(')
// .add(b')')
.add(b'*')
.add(b'+')
.add(b',')
.add(b'/')
.add(b':')
.add(b';')
.add(b'<')
.add(b'=')
.add(b'>')
.add(b'?')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'`')
.add(b'{')
.add(b'|')
// .add(b'~');
.add(b'}');
pub fn encode_uri_component(input: &str) -> String { use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
utf8_percent_encode(input, ENCODE_URI_COMPONENT_SET).to_string() pub use url_spec::*;
/// An [`AsciiSet`] that matches the behavior of JavaScript's [`encodeURIComponent`][mdn].
/// It escapes all characters except:
///
/// ```text
/// A-Z a-z 0–9 - _ . ! ~ * ' ( )
/// ```
///
/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent#description
///
/// Because this is designed to work with *any* URL component, it encodes more characters than the
/// spec requires. For less-agressive encoding sets, see [`FRAGMENT`], [`PATH`], [`USERINFO`],
/// [`PATH_SEGMENT`], [`SPECIAL_PATH_SEGMENT`], [`QUERY`], and [`SPECIAL_QUERY`].
pub const ENCODE_URI_COMPONENT: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'!')
.remove(b'~')
.remove(b'*')
.remove(b'\'')
.remove(b'(')
.remove(b')');
/// A convenience wrapper around [`percent_encoding::utf8_percent_encode`] that returns a
/// [`Cow<'_, str>`] instead of a [`percent_encoding::PercentEncode`].
///
/// If you're just using the [`Display`][std::fmt::Display] trait on the return value, you should
/// use the functions from [`percent_encoding`] directly, as the
/// [`PercentEncode`][percent_encoding::PercentEncode] type can avoid creating an intermediate
/// [`String`].
pub fn percent_encode_str<'a>(input: &'a str, ascii_set: &'static AsciiSet) -> Cow<'a, str> {
Cow::from(utf8_percent_encode(input, ascii_set))
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn test_percent_encode_str() {
assert!(matches!(
percent_encode_str("nowhitespace", NON_ALPHANUMERIC),
Cow::Borrowed(_),
));
assert!(matches!(
percent_encode_str("has whitespace", NON_ALPHANUMERIC),
Cow::Owned(_),
));
}
#[test] #[test]
fn test_encode_uri_component() { fn test_encode_uri_component() {
// Each (input, expected_output) // Each (input, expected_output)
let test_cases = vec![ let test_cases = [
("Hello", "Hello"), ("Hello", "Hello"),
("Hello World", "Hello%20World"), ("Hello World", "Hello%20World"),
( (
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~*'()",
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~*'()",
), ),
("This is 100% test!", "This%20is%20100%25%20test!"), ("This is 100% test!", "This%20is%20100%25%20test!"),
("你好", "%E4%BD%A0%E5%A5%BD"), ("你好", "%E4%BD%A0%E5%A5%BD"),
...@@ -105,7 +115,7 @@ mod tests { ...@@ -105,7 +115,7 @@ mod tests {
]; ];
for (input, expected) in test_cases { for (input, expected) in test_cases {
let actual = encode_uri_component(input); let actual = percent_encode_str(input, ENCODE_URI_COMPONENT);
assert_eq!( assert_eq!(
actual, expected, actual, expected,
"Failed on input='{input}': expected='{expected}', got='{actual}'", "Failed on input='{input}': expected='{expected}', got='{actual}'",
......
// Copyright 2013-2016 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Percent encoding sets defined in the WHATWG URL spec.
//!
//! These are taken from the `url` crate:
//! https://github.com/servo/rust-url/blob/22b925f93ad505a830f1089538a9ed6f5fd90612/url/src/parser.rs#L19-L46
use percent_encoding::{AsciiSet, CONTROLS};
/// https://url.spec.whatwg.org/#fragment-percent-encode-set
pub const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`');
/// https://url.spec.whatwg.org/#path-percent-encode-set
pub const PATH: &AsciiSet = &FRAGMENT.add(b'#').add(b'?').add(b'{').add(b'}');
/// https://url.spec.whatwg.org/#userinfo-percent-encode-set
pub const USERINFO: &AsciiSet = &PATH
.add(b'/')
.add(b':')
.add(b';')
.add(b'=')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'|');
pub const PATH_SEGMENT: &AsciiSet = &PATH.add(b'/').add(b'%');
/// An extended variant of [`PATH_SEGMENT`] for `http`, `https`, `ws`, `wss`, `ftp`, and `file` url
/// schemes.
///
/// The backslash (\) character is treated as a path separator in special URLs so it needs to be
/// additionally escaped in that case.
pub const SPECIAL_PATH_SEGMENT: &AsciiSet = &PATH_SEGMENT.add(b'\\');
// https://url.spec.whatwg.org/#query-state
pub const QUERY: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'#').add(b'<').add(b'>');
/// An extended variant of [`QUERY`] for `http`, `https`, `ws`, `wss`, `ftp`, and `file` url
/// schemes.
///
/// The backslash (\) character is treated as a path separator in special URLs so it needs to be
/// additionally escaped in that case.
pub const SPECIAL_QUERY: &AsciiSet = &QUERY.add(b'\'');
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment