Unverified Commit 0084361f authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Fix URL fragment and query handling in Turbopack (#79993)

Fix URL fragment and query handling in Turbopack

## What?
This PR improves how URL fragments and query strings are handled in Turbopack by:

1. Standardizing the representation of fragments and queries in `AssetIdent`
2. Ensuring fragments and queries are either empty or include their prefix characters (`#` and `?`)
3. Fixing the URL parsing logic to correctly handle fragments that contain `?` character
4. Removing some ad-hoc workarounds for this issue in the `resolver` and `Assetident::to_string`

## Why?
The previous implementation had inconsistencies in how fragments and queries were represented which lead to a variety of confusing issues e.g. does `AssetIdent::to_string` need to add a `?` ?, well sometimes depending on how it was constructed.
parent 1f39908c
...@@ -213,7 +213,7 @@ pub async fn to_rsc_context( ...@@ -213,7 +213,7 @@ pub async fn to_rsc_context(
// module. // module.
let source = FileSource::new_with_query( let source = FileSource::new_with_query(
client_module.ident().path().root().join(entry_path.into()), client_module.ident().path().root().join(entry_path.into()),
Vc::cell(entry_query.into()), entry_query.into(),
); );
let module = asset_context let module = asset_context
.process( .process(
......
...@@ -71,7 +71,11 @@ impl Transition for NextEcmascriptClientReferenceTransition { ...@@ -71,7 +71,11 @@ impl Transition for NextEcmascriptClientReferenceTransition {
.replace("next/dist/esm/", "next/dist/") .replace("next/dist/esm/", "next/dist/")
.into(), .into(),
); );
Vc::upcast(FileSource::new_with_query(path, *ident_ref.query)) Vc::upcast(FileSource::new_with_query_and_fragment(
path,
(*ident_ref.query.await?).clone(),
(*ident_ref.fragment.await?).clone(),
))
} else { } else {
source source
}; };
......
...@@ -13,38 +13,48 @@ use crate::{ ...@@ -13,38 +13,48 @@ use crate::{
/// references to other [Source]s. /// references to other [Source]s.
#[turbo_tasks::value] #[turbo_tasks::value]
pub struct FileSource { pub struct FileSource {
pub path: ResolvedVc<FileSystemPath>, path: ResolvedVc<FileSystemPath>,
pub query: ResolvedVc<RcStr>, query: RcStr,
fragment: RcStr,
}
impl FileSource {
pub fn new(path: Vc<FileSystemPath>) -> Vc<Self> {
FileSource::new_with_query_and_fragment(path, RcStr::default(), RcStr::default())
}
pub fn new_with_query(path: Vc<FileSystemPath>, query: RcStr) -> Vc<Self> {
FileSource::new_with_query_and_fragment(path, query, RcStr::default())
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl FileSource { impl FileSource {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(path: ResolvedVc<FileSystemPath>) -> Vc<Self> { pub fn new_with_query_and_fragment(
path: ResolvedVc<FileSystemPath>,
query: RcStr,
fragment: RcStr,
) -> Vc<Self> {
Self::cell(FileSource { Self::cell(FileSource {
path, path,
query: ResolvedVc::cell(RcStr::default()), query,
fragment,
}) })
} }
#[turbo_tasks::function]
pub async fn new_with_query(
path: ResolvedVc<FileSystemPath>,
query: ResolvedVc<RcStr>,
) -> Result<Vc<Self>> {
if query.await?.is_empty() {
Ok(Self::new(*path))
} else {
Ok(Self::cell(FileSource { path, query }))
}
}
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl Source for FileSource { impl Source for FileSource {
#[turbo_tasks::function] #[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> { fn ident(&self) -> Vc<AssetIdent> {
AssetIdent::from_path(*self.path).with_query(*self.query) let mut ident = AssetIdent::from_path(*self.path);
if !self.query.is_empty() {
ident = ident.with_query(Vc::cell(self.query.clone()));
}
if !self.fragment.is_empty() {
ident = ident.with_fragment(Vc::cell(self.fragment.clone()));
}
ident
} }
} }
......
...@@ -15,10 +15,12 @@ use crate::resolve::ModulePart; ...@@ -15,10 +15,12 @@ use crate::resolve::ModulePart;
pub struct AssetIdent { pub struct AssetIdent {
/// The primary path of the asset /// The primary path of the asset
pub path: ResolvedVc<FileSystemPath>, pub path: ResolvedVc<FileSystemPath>,
/// The query string of the asset (e.g. `?foo=bar`) /// The query string of the asset this is either the empty string or a query string that starts
/// with a `?` (e.g. `?foo=bar`)
pub query: ResolvedVc<RcStr>, pub query: ResolvedVc<RcStr>,
/// The fragment of the asset (e.g. `#foo`) /// The fragment of the asset, this is either the empty string or a fragment string that starts
pub fragment: Option<ResolvedVc<RcStr>>, /// with a `#` (e.g. `#foo`)
pub fragment: ResolvedVc<RcStr>,
/// The assets that are nested in this asset /// The assets that are nested in this asset
pub assets: Vec<(ResolvedVc<RcStr>, ResolvedVc<AssetIdent>)>, pub assets: Vec<(ResolvedVc<RcStr>, ResolvedVc<AssetIdent>)>,
/// The modifiers of this asset (e.g. `client chunks`) /// The modifiers of this asset (e.g. `client chunks`)
...@@ -57,14 +59,10 @@ impl ValueToString for AssetIdent { ...@@ -57,14 +59,10 @@ impl ValueToString for AssetIdent {
async fn to_string(&self) -> Result<Vc<RcStr>> { async fn to_string(&self) -> Result<Vc<RcStr>> {
let mut s = self.path.to_string().owned().await?.into_owned(); let mut s = self.path.to_string().owned().await?.into_owned();
let query = self.query.await?; // The query string is either empty or non-empty starting with `?` so we can just concat
if !query.is_empty() { s.push_str(&self.query.await?);
write!(s, "?{}", &*query)?; // ditto for fragment
} s.push_str(&self.fragment.await?);
if let Some(fragment) = &self.fragment {
write!(s, "#{}", fragment.await?)?;
}
if !self.assets.is_empty() { if !self.assets.is_empty() {
s.push_str(" {"); s.push_str(" {");
...@@ -121,8 +119,19 @@ impl ValueToString for AssetIdent { ...@@ -121,8 +119,19 @@ impl ValueToString for AssetIdent {
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl AssetIdent { impl AssetIdent {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new(ident: Value<AssetIdent>) -> Vc<Self> { pub async fn new(ident: Value<AssetIdent>) -> Result<Vc<Self>> {
ident.into_value().cell() let query = &*ident.query.await?;
// TODO(lukesandberg); downgrade to debug_assert
assert!(
query.is_empty() || query.starts_with("?"),
"query should be empty or start with a `?`"
);
let fragment = &*ident.fragment.await?;
assert!(
fragment.is_empty() || fragment.starts_with("#"),
"query should be empty or start with a `?`"
);
Ok(ident.into_value().cell())
} }
/// Creates an [AssetIdent] from a [Vc<FileSystemPath>] /// Creates an [AssetIdent] from a [Vc<FileSystemPath>]
...@@ -131,7 +140,7 @@ impl AssetIdent { ...@@ -131,7 +140,7 @@ impl AssetIdent {
Self::new(Value::new(AssetIdent { Self::new(Value::new(AssetIdent {
path, path,
query: ResolvedVc::cell(RcStr::default()), query: ResolvedVc::cell(RcStr::default()),
fragment: None, fragment: ResolvedVc::cell(RcStr::default()),
assets: Vec::new(), assets: Vec::new(),
modifiers: Vec::new(), modifiers: Vec::new(),
parts: Vec::new(), parts: Vec::new(),
...@@ -147,6 +156,13 @@ impl AssetIdent { ...@@ -147,6 +156,13 @@ impl AssetIdent {
Self::new(Value::new(this)) Self::new(Value::new(this))
} }
#[turbo_tasks::function]
pub fn with_fragment(&self, fragment: ResolvedVc<RcStr>) -> Vc<Self> {
let mut this = self.clone();
this.fragment = fragment;
Self::new(Value::new(this))
}
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn with_modifier(&self, modifier: ResolvedVc<RcStr>) -> Vc<Self> { pub fn with_modifier(&self, modifier: ResolvedVc<RcStr>) -> Vc<Self> {
let mut this = self.clone(); let mut this = self.clone();
...@@ -252,9 +268,10 @@ impl AssetIdent { ...@@ -252,9 +268,10 @@ impl AssetIdent {
query.deterministic_hash(&mut hasher); query.deterministic_hash(&mut hasher);
has_hash = true; has_hash = true;
} }
if let Some(fragment) = fragment { let fragment = fragment.await?;
if !fragment.is_empty() {
1_u8.deterministic_hash(&mut hasher); 1_u8.deterministic_hash(&mut hasher);
fragment.await?.deterministic_hash(&mut hasher); fragment.deterministic_hash(&mut hasher);
has_hash = true; has_hash = true;
} }
for (key, ident) in assets.iter() { for (key, ident) in assets.iter() {
......
...@@ -2203,18 +2203,17 @@ async fn resolve_relative_request( ...@@ -2203,18 +2203,17 @@ async fn resolve_relative_request(
let fragment_val = fragment.await?; let fragment_val = fragment.await?;
if !fragment_val.is_empty() { if !fragment_val.is_empty() {
new_path.push(Pattern::Alternatives( new_path.push(Pattern::Alternatives(vec![
once(Pattern::Constant("".into())) Pattern::Constant(RcStr::default()),
.chain(once(Pattern::Constant(format!("#{fragment_val}").into()))) Pattern::Constant((*fragment_val).clone()),
.collect(), ]));
));
} }
if !options_value.fully_specified { if !options_value.fully_specified {
// Add the extensions as alternatives to the path // Add the extensions as alternatives to the path
// read_matches keeps the order of alternatives intact // read_matches keeps the order of alternatives intact
new_path.push(Pattern::Alternatives( new_path.push(Pattern::Alternatives(
once(Pattern::Constant("".into())) once(Pattern::Constant(RcStr::default()))
.chain( .chain(
options_value options_value
.extensions .extensions
...@@ -2328,10 +2327,7 @@ async fn resolve_relative_request( ...@@ -2328,10 +2327,7 @@ async fn resolve_relative_request(
} }
if !fragment_val.is_empty() { if !fragment_val.is_empty() {
// If the fragment is not empty, we need to strip it from the matched pattern // If the fragment is not empty, we need to strip it from the matched pattern
if let Some(matched_pattern) = matched_pattern if let Some(matched_pattern) = matched_pattern.strip_suffix(&**fragment_val) {
.strip_suffix(&**fragment_val)
.and_then(|s| s.strip_suffix('#'))
{
results.push( results.push(
resolved( resolved(
RequestKey::new(matched_pattern.into()), RequestKey::new(matched_pattern.into()),
...@@ -2864,9 +2860,13 @@ async fn resolved( ...@@ -2864,9 +2860,13 @@ async fn resolved(
Ok(*ResolveResult::source_with_affecting_sources( Ok(*ResolveResult::source_with_affecting_sources(
request_key, request_key,
ResolvedVc::upcast( ResolvedVc::upcast(
FileSource::new_with_query(**path, query) FileSource::new_with_query_and_fragment(
.to_resolved() **path,
.await?, (*query.await?).clone(),
(*fragment.await?).clone(),
)
.to_resolved()
.await?,
), ),
symlinks symlinks
.iter() .iter()
......
...@@ -61,30 +61,32 @@ pub enum Request { ...@@ -61,30 +61,32 @@ pub enum Request {
}, },
} }
fn split_off_query_fragment(raw: RcStr) -> (Pattern, Vc<RcStr>, Vc<RcStr>) { /// Splits a string like `foo?bar#baz` into `(Pattern::Constant('foo'), '?bar', '#baz')`
let Some((raw, query)) = raw.split_once('?') else { ///
if let Some((raw, fragment)) = raw.split_once('#') { /// If the hash or query portion are missing they will be empty strings otherwise they will be
return ( /// non-empty along with their prepender characters
Pattern::Constant(raw.into()), fn split_off_query_fragment(mut raw: &str) -> (Pattern, RcStr, RcStr) {
Vc::<RcStr>::default(), // Per the URI spec fragments can contain `?` characters, so we should trim it off first
Vc::cell(fragment.into()), // https://datatracker.ietf.org/doc/html/rfc3986#section-3.5
);
let hash = match raw.as_bytes().iter().position(|&b| b == b'#') {
Some(pos) => {
let (prefix, hash) = raw.split_at(pos);
raw = prefix;
RcStr::from(hash)
} }
None => RcStr::default(),
return (
Pattern::Constant(raw),
Vc::<RcStr>::default(),
Vc::<RcStr>::default(),
);
}; };
let (query, fragment) = query.split_once('#').unwrap_or((query, "")); let query = match raw.as_bytes().iter().position(|&b| b == b'?') {
Some(pos) => {
( let (prefix, query) = raw.split_at(pos);
Pattern::Constant(raw.into()), raw = prefix;
Vc::cell(format!("?{query}").into()), RcStr::from(query)
Vc::cell(format!("#{fragment}").into()), }
) None => RcStr::default(),
};
(Pattern::Constant(RcStr::from(raw)), query, hash)
} }
lazy_static! { lazy_static! {
...@@ -164,12 +166,12 @@ impl Request { ...@@ -164,12 +166,12 @@ impl Request {
} }
if r.starts_with('/') { if r.starts_with('/') {
let (path, query, fragment) = split_off_query_fragment(r); let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::ServerRelative { return Ok(Request::ServerRelative {
path, path,
query: query.to_resolved().await?, query: ResolvedVc::cell(query),
fragment: fragment.to_resolved().await?, fragment: ResolvedVc::cell(fragment),
}); });
} }
...@@ -180,23 +182,23 @@ impl Request { ...@@ -180,23 +182,23 @@ impl Request {
} }
if r.starts_with("./") || r.starts_with("../") || r == "." || r == ".." { if r.starts_with("./") || r.starts_with("../") || r == "." || r == ".." {
let (path, query, fragment) = split_off_query_fragment(r); let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::Relative { return Ok(Request::Relative {
path, path,
force_in_lookup_dir: false, force_in_lookup_dir: false,
query: query.to_resolved().await?, query: ResolvedVc::cell(query),
fragment: fragment.to_resolved().await?, fragment: ResolvedVc::cell(fragment),
}); });
} }
if WINDOWS_PATH.is_match(&r) { if WINDOWS_PATH.is_match(&r) {
let (path, query, fragment) = split_off_query_fragment(r); let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::Windows { return Ok(Request::Windows {
path, path,
query: query.to_resolved().await?, query: ResolvedVc::cell(query),
fragment: fragment.to_resolved().await?, fragment: ResolvedVc::cell(fragment),
}); });
} }
...@@ -227,13 +229,13 @@ impl Request { ...@@ -227,13 +229,13 @@ impl Request {
.captures(&r) .captures(&r)
.and_then(|caps| caps.get(1).zip(caps.get(2))) .and_then(|caps| caps.get(1).zip(caps.get(2)))
{ {
let (path, query, fragment) = split_off_query_fragment(path.as_str().into()); let (path, query, fragment) = split_off_query_fragment(path.as_str());
return Ok(Request::Module { return Ok(Request::Module {
module: module.as_str().into(), module: module.as_str().into(),
path, path,
query: query.to_resolved().await?, query: ResolvedVc::cell(query),
fragment: fragment.to_resolved().await?, fragment: ResolvedVc::cell(fragment),
}); });
} }
...@@ -817,3 +819,63 @@ pub async fn stringify_data_uri( ...@@ -817,3 +819,63 @@ pub async fn stringify_data_uri(
data.await? data.await?
)) ))
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_split_query_fragment() {
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::default()
),
split_off_query_fragment("foo")
);
// These two cases are a bit odd, but it is important to treat `import './foo?'` differently
// from `import './foo'`, ditto for fragments.
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?"),
RcStr::default()
),
split_off_query_fragment("foo?")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::from("#")
),
split_off_query_fragment("foo#")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?bar=baz"),
RcStr::default()
),
split_off_query_fragment("foo?bar=baz")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::from("#stuff?bar=baz")
),
split_off_query_fragment("foo#stuff?bar=baz")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?bar=baz"),
RcStr::from("#stuff")
),
split_off_query_fragment("foo?bar=baz#stuff")
);
}
}
...@@ -194,7 +194,7 @@ impl CssChunk { ...@@ -194,7 +194,7 @@ impl CssChunk {
ServerFileSystem::new().root().to_resolved().await? ServerFileSystem::new().root().to_resolved().await?
}, },
query: ResolvedVc::cell(RcStr::default()), query: ResolvedVc::cell(RcStr::default()),
fragment: None, fragment: ResolvedVc::cell(RcStr::default()),
assets, assets,
modifiers: Vec::new(), modifiers: Vec::new(),
parts: Vec::new(), parts: Vec::new(),
......
...@@ -123,7 +123,7 @@ impl Chunk for EcmascriptChunk { ...@@ -123,7 +123,7 @@ impl Chunk for EcmascriptChunk {
ServerFileSystem::new().root().to_resolved().await? ServerFileSystem::new().root().to_resolved().await?
}, },
query: ResolvedVc::cell(RcStr::default()), query: ResolvedVc::cell(RcStr::default()),
fragment: None, fragment: ResolvedVc::cell(RcStr::default()),
assets, assets,
modifiers: Vec::new(), modifiers: Vec::new(),
parts: Vec::new(), parts: Vec::new(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment