Unverified Commit 0084361f authored by Luke Sandberg's avatar Luke Sandberg Committed by GitHub

[turbopack] Fix URL fragment and query handling in Turbopack (#79993)

Fix URL fragment and query handling in Turbopack

## What?
This PR improves how URL fragments and query strings are handled in Turbopack by:

1. Standardizing the representation of fragments and queries in `AssetIdent`
2. Ensuring fragments and queries are either empty or include their prefix characters (`#` and `?`)
3. Fixing the URL parsing logic to correctly handle fragments that contain `?` character
4. Removing some ad-hoc workarounds for this issue in the `resolver` and `Assetident::to_string`

## Why?
The previous implementation had inconsistencies in how fragments and queries were represented which lead to a variety of confusing issues e.g. does `AssetIdent::to_string` need to add a `?` ?, well sometimes depending on how it was constructed.
parent 1f39908c
......@@ -213,7 +213,7 @@ pub async fn to_rsc_context(
// module.
let source = FileSource::new_with_query(
client_module.ident().path().root().join(entry_path.into()),
Vc::cell(entry_query.into()),
entry_query.into(),
);
let module = asset_context
.process(
......
......@@ -71,7 +71,11 @@ impl Transition for NextEcmascriptClientReferenceTransition {
.replace("next/dist/esm/", "next/dist/")
.into(),
);
Vc::upcast(FileSource::new_with_query(path, *ident_ref.query))
Vc::upcast(FileSource::new_with_query_and_fragment(
path,
(*ident_ref.query.await?).clone(),
(*ident_ref.fragment.await?).clone(),
))
} else {
source
};
......
......@@ -13,38 +13,48 @@ use crate::{
/// references to other [Source]s.
#[turbo_tasks::value]
pub struct FileSource {
pub path: ResolvedVc<FileSystemPath>,
pub query: ResolvedVc<RcStr>,
path: ResolvedVc<FileSystemPath>,
query: RcStr,
fragment: RcStr,
}
impl FileSource {
pub fn new(path: Vc<FileSystemPath>) -> Vc<Self> {
FileSource::new_with_query_and_fragment(path, RcStr::default(), RcStr::default())
}
pub fn new_with_query(path: Vc<FileSystemPath>, query: RcStr) -> Vc<Self> {
FileSource::new_with_query_and_fragment(path, query, RcStr::default())
}
}
#[turbo_tasks::value_impl]
impl FileSource {
#[turbo_tasks::function]
pub fn new(path: ResolvedVc<FileSystemPath>) -> Vc<Self> {
pub fn new_with_query_and_fragment(
path: ResolvedVc<FileSystemPath>,
query: RcStr,
fragment: RcStr,
) -> Vc<Self> {
Self::cell(FileSource {
path,
query: ResolvedVc::cell(RcStr::default()),
query,
fragment,
})
}
#[turbo_tasks::function]
pub async fn new_with_query(
path: ResolvedVc<FileSystemPath>,
query: ResolvedVc<RcStr>,
) -> Result<Vc<Self>> {
if query.await?.is_empty() {
Ok(Self::new(*path))
} else {
Ok(Self::cell(FileSource { path, query }))
}
}
}
#[turbo_tasks::value_impl]
impl Source for FileSource {
#[turbo_tasks::function]
fn ident(&self) -> Vc<AssetIdent> {
AssetIdent::from_path(*self.path).with_query(*self.query)
let mut ident = AssetIdent::from_path(*self.path);
if !self.query.is_empty() {
ident = ident.with_query(Vc::cell(self.query.clone()));
}
if !self.fragment.is_empty() {
ident = ident.with_fragment(Vc::cell(self.fragment.clone()));
}
ident
}
}
......
......@@ -15,10 +15,12 @@ use crate::resolve::ModulePart;
pub struct AssetIdent {
/// The primary path of the asset
pub path: ResolvedVc<FileSystemPath>,
/// The query string of the asset (e.g. `?foo=bar`)
/// The query string of the asset this is either the empty string or a query string that starts
/// with a `?` (e.g. `?foo=bar`)
pub query: ResolvedVc<RcStr>,
/// The fragment of the asset (e.g. `#foo`)
pub fragment: Option<ResolvedVc<RcStr>>,
/// The fragment of the asset, this is either the empty string or a fragment string that starts
/// with a `#` (e.g. `#foo`)
pub fragment: ResolvedVc<RcStr>,
/// The assets that are nested in this asset
pub assets: Vec<(ResolvedVc<RcStr>, ResolvedVc<AssetIdent>)>,
/// The modifiers of this asset (e.g. `client chunks`)
......@@ -57,14 +59,10 @@ impl ValueToString for AssetIdent {
async fn to_string(&self) -> Result<Vc<RcStr>> {
let mut s = self.path.to_string().owned().await?.into_owned();
let query = self.query.await?;
if !query.is_empty() {
write!(s, "?{}", &*query)?;
}
if let Some(fragment) = &self.fragment {
write!(s, "#{}", fragment.await?)?;
}
// The query string is either empty or non-empty starting with `?` so we can just concat
s.push_str(&self.query.await?);
// ditto for fragment
s.push_str(&self.fragment.await?);
if !self.assets.is_empty() {
s.push_str(" {");
......@@ -121,8 +119,19 @@ impl ValueToString for AssetIdent {
#[turbo_tasks::value_impl]
impl AssetIdent {
#[turbo_tasks::function]
pub fn new(ident: Value<AssetIdent>) -> Vc<Self> {
ident.into_value().cell()
pub async fn new(ident: Value<AssetIdent>) -> Result<Vc<Self>> {
let query = &*ident.query.await?;
// TODO(lukesandberg); downgrade to debug_assert
assert!(
query.is_empty() || query.starts_with("?"),
"query should be empty or start with a `?`"
);
let fragment = &*ident.fragment.await?;
assert!(
fragment.is_empty() || fragment.starts_with("#"),
"query should be empty or start with a `?`"
);
Ok(ident.into_value().cell())
}
/// Creates an [AssetIdent] from a [Vc<FileSystemPath>]
......@@ -131,7 +140,7 @@ impl AssetIdent {
Self::new(Value::new(AssetIdent {
path,
query: ResolvedVc::cell(RcStr::default()),
fragment: None,
fragment: ResolvedVc::cell(RcStr::default()),
assets: Vec::new(),
modifiers: Vec::new(),
parts: Vec::new(),
......@@ -147,6 +156,13 @@ impl AssetIdent {
Self::new(Value::new(this))
}
#[turbo_tasks::function]
pub fn with_fragment(&self, fragment: ResolvedVc<RcStr>) -> Vc<Self> {
let mut this = self.clone();
this.fragment = fragment;
Self::new(Value::new(this))
}
#[turbo_tasks::function]
pub fn with_modifier(&self, modifier: ResolvedVc<RcStr>) -> Vc<Self> {
let mut this = self.clone();
......@@ -252,9 +268,10 @@ impl AssetIdent {
query.deterministic_hash(&mut hasher);
has_hash = true;
}
if let Some(fragment) = fragment {
let fragment = fragment.await?;
if !fragment.is_empty() {
1_u8.deterministic_hash(&mut hasher);
fragment.await?.deterministic_hash(&mut hasher);
fragment.deterministic_hash(&mut hasher);
has_hash = true;
}
for (key, ident) in assets.iter() {
......
......@@ -2203,18 +2203,17 @@ async fn resolve_relative_request(
let fragment_val = fragment.await?;
if !fragment_val.is_empty() {
new_path.push(Pattern::Alternatives(
once(Pattern::Constant("".into()))
.chain(once(Pattern::Constant(format!("#{fragment_val}").into())))
.collect(),
));
new_path.push(Pattern::Alternatives(vec![
Pattern::Constant(RcStr::default()),
Pattern::Constant((*fragment_val).clone()),
]));
}
if !options_value.fully_specified {
// Add the extensions as alternatives to the path
// read_matches keeps the order of alternatives intact
new_path.push(Pattern::Alternatives(
once(Pattern::Constant("".into()))
once(Pattern::Constant(RcStr::default()))
.chain(
options_value
.extensions
......@@ -2328,10 +2327,7 @@ async fn resolve_relative_request(
}
if !fragment_val.is_empty() {
// If the fragment is not empty, we need to strip it from the matched pattern
if let Some(matched_pattern) = matched_pattern
.strip_suffix(&**fragment_val)
.and_then(|s| s.strip_suffix('#'))
{
if let Some(matched_pattern) = matched_pattern.strip_suffix(&**fragment_val) {
results.push(
resolved(
RequestKey::new(matched_pattern.into()),
......@@ -2864,9 +2860,13 @@ async fn resolved(
Ok(*ResolveResult::source_with_affecting_sources(
request_key,
ResolvedVc::upcast(
FileSource::new_with_query(**path, query)
.to_resolved()
.await?,
FileSource::new_with_query_and_fragment(
**path,
(*query.await?).clone(),
(*fragment.await?).clone(),
)
.to_resolved()
.await?,
),
symlinks
.iter()
......
......@@ -61,30 +61,32 @@ pub enum Request {
},
}
fn split_off_query_fragment(raw: RcStr) -> (Pattern, Vc<RcStr>, Vc<RcStr>) {
let Some((raw, query)) = raw.split_once('?') else {
if let Some((raw, fragment)) = raw.split_once('#') {
return (
Pattern::Constant(raw.into()),
Vc::<RcStr>::default(),
Vc::cell(fragment.into()),
);
/// Splits a string like `foo?bar#baz` into `(Pattern::Constant('foo'), '?bar', '#baz')`
///
/// If the hash or query portion are missing they will be empty strings otherwise they will be
/// non-empty along with their prepender characters
fn split_off_query_fragment(mut raw: &str) -> (Pattern, RcStr, RcStr) {
// Per the URI spec fragments can contain `?` characters, so we should trim it off first
// https://datatracker.ietf.org/doc/html/rfc3986#section-3.5
let hash = match raw.as_bytes().iter().position(|&b| b == b'#') {
Some(pos) => {
let (prefix, hash) = raw.split_at(pos);
raw = prefix;
RcStr::from(hash)
}
return (
Pattern::Constant(raw),
Vc::<RcStr>::default(),
Vc::<RcStr>::default(),
);
None => RcStr::default(),
};
let (query, fragment) = query.split_once('#').unwrap_or((query, ""));
(
Pattern::Constant(raw.into()),
Vc::cell(format!("?{query}").into()),
Vc::cell(format!("#{fragment}").into()),
)
let query = match raw.as_bytes().iter().position(|&b| b == b'?') {
Some(pos) => {
let (prefix, query) = raw.split_at(pos);
raw = prefix;
RcStr::from(query)
}
None => RcStr::default(),
};
(Pattern::Constant(RcStr::from(raw)), query, hash)
}
lazy_static! {
......@@ -164,12 +166,12 @@ impl Request {
}
if r.starts_with('/') {
let (path, query, fragment) = split_off_query_fragment(r);
let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::ServerRelative {
path,
query: query.to_resolved().await?,
fragment: fragment.to_resolved().await?,
query: ResolvedVc::cell(query),
fragment: ResolvedVc::cell(fragment),
});
}
......@@ -180,23 +182,23 @@ impl Request {
}
if r.starts_with("./") || r.starts_with("../") || r == "." || r == ".." {
let (path, query, fragment) = split_off_query_fragment(r);
let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::Relative {
path,
force_in_lookup_dir: false,
query: query.to_resolved().await?,
fragment: fragment.to_resolved().await?,
query: ResolvedVc::cell(query),
fragment: ResolvedVc::cell(fragment),
});
}
if WINDOWS_PATH.is_match(&r) {
let (path, query, fragment) = split_off_query_fragment(r);
let (path, query, fragment) = split_off_query_fragment(&r);
return Ok(Request::Windows {
path,
query: query.to_resolved().await?,
fragment: fragment.to_resolved().await?,
query: ResolvedVc::cell(query),
fragment: ResolvedVc::cell(fragment),
});
}
......@@ -227,13 +229,13 @@ impl Request {
.captures(&r)
.and_then(|caps| caps.get(1).zip(caps.get(2)))
{
let (path, query, fragment) = split_off_query_fragment(path.as_str().into());
let (path, query, fragment) = split_off_query_fragment(path.as_str());
return Ok(Request::Module {
module: module.as_str().into(),
path,
query: query.to_resolved().await?,
fragment: fragment.to_resolved().await?,
query: ResolvedVc::cell(query),
fragment: ResolvedVc::cell(fragment),
});
}
......@@ -817,3 +819,63 @@ pub async fn stringify_data_uri(
data.await?
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_split_query_fragment() {
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::default()
),
split_off_query_fragment("foo")
);
// These two cases are a bit odd, but it is important to treat `import './foo?'` differently
// from `import './foo'`, ditto for fragments.
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?"),
RcStr::default()
),
split_off_query_fragment("foo?")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::from("#")
),
split_off_query_fragment("foo#")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?bar=baz"),
RcStr::default()
),
split_off_query_fragment("foo?bar=baz")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::default(),
RcStr::from("#stuff?bar=baz")
),
split_off_query_fragment("foo#stuff?bar=baz")
);
assert_eq!(
(
Pattern::Constant("foo".into()),
RcStr::from("?bar=baz"),
RcStr::from("#stuff")
),
split_off_query_fragment("foo?bar=baz#stuff")
);
}
}
......@@ -194,7 +194,7 @@ impl CssChunk {
ServerFileSystem::new().root().to_resolved().await?
},
query: ResolvedVc::cell(RcStr::default()),
fragment: None,
fragment: ResolvedVc::cell(RcStr::default()),
assets,
modifiers: Vec::new(),
parts: Vec::new(),
......
......@@ -123,7 +123,7 @@ impl Chunk for EcmascriptChunk {
ServerFileSystem::new().root().to_resolved().await?
},
query: ResolvedVc::cell(RcStr::default()),
fragment: None,
fragment: ResolvedVc::cell(RcStr::default()),
assets,
modifiers: Vec::new(),
parts: Vec::new(),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment