Commit 5f6f616a authored by Benjamin Woodruff's avatar Benjamin Woodruff

Turbopack: Always encode `turbopack:///[fs]/` urls as valid URLs

parent b90559cd
......@@ -4313,6 +4313,7 @@ dependencies = [
"turbopack-nodejs",
"turbopack-static",
"turbopack-trace-utils",
"turbopack-url",
]
[[package]]
......@@ -4880,9 +4881,9 @@ dependencies = [
[[package]]
name = "percent-encoding"
version = "2.3.1"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pest"
......@@ -9284,6 +9285,7 @@ dependencies = [
"mime",
"notify",
"parking_lot",
"percent-encoding",
"regex",
"rstest",
"rustc-hash 2.1.1",
......@@ -9302,6 +9304,7 @@ dependencies = [
"turbo-tasks-hash",
"turbo-tasks-testing",
"turbo-unix-path",
"url",
"urlencoding",
]
......@@ -9541,6 +9544,7 @@ dependencies = [
"indexmap 2.9.0",
"once_cell",
"patricia_tree",
"percent-encoding",
"petgraph 0.6.3",
"ref-cast",
"regex",
......@@ -9564,6 +9568,7 @@ dependencies = [
"turbo-tasks-hash",
"turbo-tasks-testing",
"turbo-unix-path",
"turbopack-url",
"url",
"urlencoding",
]
......@@ -10026,6 +10031,14 @@ dependencies = [
"turbopack-resolve",
]
[[package]]
name = "turbopack-url"
version = "0.1.0"
dependencies = [
"percent-encoding",
"url",
]
[[package]]
name = "turbopack-wasm"
version = "0.1.0"
......
......@@ -305,6 +305,7 @@ turbopack-swc-utils = { path = "turbopack/crates/turbopack-swc-utils" }
turbopack-test-utils = { path = "turbopack/crates/turbopack-test-utils" }
turbopack-trace-server = { path = "turbopack/crates/turbopack-trace-server" }
turbopack-trace-utils = { path = "turbopack/crates/turbopack-trace-utils" }
turbopack-url = { path = "turbopack/crates/turbopack-url" }
turbopack-wasm = { path = "turbopack/crates/turbopack-wasm" }
# SWC crates
......@@ -398,6 +399,7 @@ owo-colors = "4.2.2"
parcel_selectors = "0.28.2"
parking_lot = "0.12.1"
pathdiff = "0.2.1"
percent-encoding = "2.3.2"
petgraph = "0.6.3"
pin-project-lite = "0.2.9"
postcard = "1.0.4"
......
use std::{borrow::Cow, io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration};
use anyhow::{Context, Result, anyhow, bail};
use futures_util::TryFutureExt;
......@@ -24,7 +24,6 @@ use next_core::tracing_presets::{
TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBO_TASKS_TARGETS,
TRACING_NEXT_TURBOPACK_TARGETS,
};
use once_cell::sync::Lazy;
use rand::Rng;
use serde::{Deserialize, Serialize};
use tokio::{io::AsyncWriteExt, runtime::Handle, time::Instant};
......@@ -38,17 +37,17 @@ use turbo_tasks::{
trace::TraceRawVcs,
};
use turbo_tasks_backend::{BackingStorage, db_invalidation::invalidation_reasons};
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystem, FileSystemPath, util::uri_from_file,
};
use turbo_unix_path::{get_relative_path_to, sys_to_unix};
use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath};
use turbo_unix_path::sys_to_unix;
use turbopack_core::{
PROJECT_FILESYSTEM_NAME, SOURCE_URL_PROTOCOL,
PROJECT_FILESYSTEM_NAME,
diagnostics::PlainDiagnostic,
error::PrettyPrintError,
issue::PlainIssue,
output::{OutputAsset, OutputAssets},
source_map::{OptionStringifiedSourceMap, SourceMap, Token},
source_map::{
OptionStringifiedSourceMap, SourceMap, Token, utils::turbopack_url_to_fs_relative_file_url,
},
version::{PartialUpdate, TotalUpdate, Update, VersionState},
};
use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, Issue, ResourceIdentifier};
......@@ -78,9 +77,6 @@ use crate::{
/// Used by [`benchmark_file_io`]. This is a noisy benchmark, so set the
/// threshold high.
const SLOW_FILESYSTEM_THRESHOLD: Duration = Duration::from_millis(100);
static SOURCE_MAP_PREFIX: Lazy<String> = Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///"));
static SOURCE_MAP_PREFIX_PROJECT: Lazy<String> =
Lazy::new(|| format!("{SOURCE_URL_PROTOCOL}///[{PROJECT_FILESYSTEM_NAME}]/"));
#[napi(object)]
#[derive(Clone, Debug)]
......@@ -1479,10 +1475,10 @@ pub fn get_source_map_rope_operation(
pub async fn project_trace_source_operation(
container: ResolvedVc<ProjectContainer>,
frame: StackFrame,
current_directory_file_url: RcStr,
) -> Result<Vc<OptionStackFrame>> {
let Some(map) =
&*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file)).await?
&*SourceMap::new_from_rope_cached(get_source_map_rope(*container, frame.file.clone()))
.await?
else {
return Ok(Vc::cell(None));
};
......@@ -1496,69 +1492,48 @@ pub async fn project_trace_source_operation(
frame.column.unwrap_or(1).saturating_sub(1),
);
let (original_file, line, column, method_name) = match token {
let (original_url, line, column, method_name) = match token {
Token::Original(token) => (
match urlencoding::decode(&token.original_file)? {
Cow::Borrowed(_) => token.original_file,
Cow::Owned(original_file) => RcStr::from(original_file),
},
token.original_file,
// JS stack frames are 1-indexed, source map tokens are 0-indexed
Some(token.original_line + 1),
Some(token.original_column + 1),
token.name,
),
Token::Synthetic(token) => {
let Some(original_file) = token.guessed_original_file else {
let Some(original_url) = token.guessed_original_file else {
return Ok(Vc::cell(None));
};
(original_file, None, None, None)
(original_url, None, None, None)
}
};
let project_root_uri =
uri_from_file(container.project().project_root_path().owned().await?, None).await? + "/";
let (file, original_file, is_internal) =
if let Some(source_file) = original_file.strip_prefix(&project_root_uri) {
// Client code uses file://
(
RcStr::from(
get_relative_path_to(&current_directory_file_url, &original_file)
// TODO(sokra) remove this to include a ./ here to make it a relative path
.trim_start_matches("./"),
),
Some(RcStr::from(source_file)),
false,
)
} else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX_PROJECT) {
// Server code uses turbopack:///[project]
// TODO should this also be file://?
(
RcStr::from(
get_relative_path_to(
&current_directory_file_url,
&format!("{project_root_uri}{source_file}"),
)
// TODO(sokra) remove this to include a ./ here to make it a relative path
.trim_start_matches("./"),
),
Some(RcStr::from(source_file)),
false,
)
} else if let Some(source_file) = original_file.strip_prefix(&*SOURCE_MAP_PREFIX) {
// All other code like turbopack:///[turbopack] is internal code
// TODO(veil): Should the protocol be preserved?
(RcStr::from(source_file), None, true)
let url_schema = original_url.split_once(":").map(|(schema, _)| schema);
let (original_url, is_internal) = if let Some(url_schema) = url_schema
&& url_schema.eq_ignore_ascii_case("turbopack")
{
if let Some(relative_url) =
turbopack_url_to_fs_relative_file_url(&original_url, PROJECT_FILESYSTEM_NAME)
{
// Server code uses `turbopack:///[project]` urls?
// this returns a relative URL, but should it be an absolute file:// url?
//
// If it's absolute, that conversion should happen outside of the turbo-task, as that
// is not safe to store in the persistent cache (not portable across machines)
(Some(RcStr::from(relative_url)), false)
} else {
bail!(
"Original file ({}) outside project ({})",
original_file,
project_root_uri
)
};
// All other code like turbopack:///[turbopack] is internal code
(None, true)
}
} else {
// Any other URL... Likely a relative or absolute file:// url. Just pass it through.
// Client code uses `file://` urls?
(Some(RcStr::from(original_url)), false)
};
Ok(Vc::cell(Some(StackFrame {
file,
original_file,
file: frame.file,
original_file: original_url,
method_name,
line,
column,
......@@ -1572,19 +1547,14 @@ pub async fn project_trace_source_operation(
pub async fn project_trace_source(
#[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
frame: StackFrame,
current_directory_file_url: String,
) -> napi::Result<Option<StackFrame>> {
let container = project.container;
let ctx = &project.turbopack_ctx;
ctx.turbo_tasks()
.run(async move {
let traced_frame = project_trace_source_operation(
container,
frame,
RcStr::from(current_directory_file_url),
)
.read_strongly_consistent()
.await?;
let traced_frame = project_trace_source_operation(container, frame)
.read_strongly_consistent()
.await?;
Ok(ReadRef::into_owned(traced_frame))
})
// HACK: Don't use `TurbopackInternalError`, this function is race-condition prone (the
......
......@@ -682,6 +682,7 @@ impl Project {
DiskFileSystem::new(rcstr!("output"), self.root_path.clone())
}
// TODO: This should not be a turbo-task function as it is not portable across machines.
#[turbo_tasks::function]
pub fn dist_dir_absolute(&self) -> Result<Vc<RcStr>> {
Ok(Vc::cell(
......
......@@ -33,7 +33,7 @@ rustc-hash = { workspace = true }
react_remove_properties = { workspace = true }
remove_console = { workspace = true }
itertools = { workspace = true }
percent-encoding = "2.3.1"
percent-encoding = { workspace = true }
serde_path_to_error = { workspace = true }
swc_sourcemap = { workspace = true }
......@@ -80,6 +80,7 @@ turbopack-node = { workspace = true }
turbopack-nodejs = { workspace = true }
turbopack-static = { workspace = true }
turbopack-trace-utils = { workspace = true }
turbopack-url = { workspace = true }
[features]
next-font-local = []
......
......@@ -2,6 +2,7 @@ use anyhow::Result;
use either::Either;
use indoc::formatdoc;
use itertools::Itertools;
use percent_encoding::utf8_percent_encode;
use rustc_hash::FxHashMap;
use serde::Serialize;
use tracing::Instrument;
......@@ -19,13 +20,14 @@ use turbopack_core::{
output::{OutputAsset, OutputAssets, OutputAssetsWithReferenced},
};
use turbopack_ecmascript::utils::StringifyJs;
use turbopack_url::ENCODE_URI_COMPONENT;
use crate::{
mode::NextMode,
next_app::ClientReferencesChunks,
next_client_reference::{ClientReferenceGraphResult, ClientReferenceType},
next_config::{CrossOriginConfig, NextConfig},
next_manifests::{ModuleId, encode_uri_component::encode_uri_component},
next_manifests::ModuleId,
util::NextRuntime,
};
......@@ -290,7 +292,9 @@ async fn build_manifest(
format!(
"{}{}{}",
prefix_path,
path.split('/').map(encode_uri_component).format("/"),
path.split('/')
.map(|segment| utf8_percent_encode(segment, ENCODE_URI_COMPONENT))
.format("/"),
suffix_path
)
})
......
//! Type definitions for the Next.js manifest formats.
pub mod client_reference_manifest;
mod encode_uri_component;
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
......
......@@ -241,8 +241,7 @@ export interface Project {
getSourceMapSync(filePath: string): string | null
traceSource(
stackFrame: TurbopackStackFrame,
currentDirectoryFileUrl: string
stackFrame: TurbopackStackFrame
): Promise<TurbopackStackFrame | null>
updateInfoSubscribe(
......
......@@ -43,10 +43,6 @@ async function batchedTraceSource(
frame: TurbopackStackFrame
): Promise<{ frame: IgnorableStackFrame; source: string | null } | undefined> {
const file = frame.file
? // TODO(veil): Why are the frames sent encoded?
decodeURIComponent(frame.file)
: undefined
if (!file) return
// For node internals they cannot traced the actual source code with project.traceSource,
......@@ -65,9 +61,7 @@ async function batchedTraceSource(
}
}
const currentDirectoryFileUrl = pathToFileURL(process.cwd()).href
const sourceFrame = await project.traceSource(frame, currentDirectoryFileUrl)
const sourceFrame = await project.traceSource(frame)
if (!sourceFrame) {
return {
frame: {
......
......@@ -37,6 +37,7 @@ jsonc-parser = { version = "0.26.3", features = ["serde"] }
mime = { workspace = true }
notify = { workspace = true }
parking_lot = { workspace = true }
percent-encoding = { workspace = true }
regex = { workspace = true }
rustc-hash = { workspace = true }
serde = { workspace = true, features = ["rc"] }
......@@ -50,6 +51,7 @@ turbo-rcstr = { workspace = true }
turbo-tasks = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }
[dev-dependencies]
......
......@@ -511,6 +511,10 @@ impl DiskFileSystem {
})
}
pub fn root_sys_path(&self) -> &Path {
self.inner.root_path()
}
pub fn to_sys_path(&self, fs_path: FileSystemPath) -> Result<PathBuf> {
let path = self.inner.root_path();
Ok(if fs_path.path.is_empty() {
......
......@@ -3,10 +3,7 @@ use std::{
path::Path,
};
use anyhow::{Context, Result, anyhow};
use turbo_tasks::ResolvedVc;
use crate::{DiskFileSystem, FileSystemPath};
use anyhow::{Result, anyhow};
/// Converts a disk access Result<T> into a Result<Some<T>>, where a NotFound
/// error results in a None value. This is purely to reduce boilerplate code
......@@ -18,57 +15,3 @@ pub fn extract_disk_access<T>(value: io::Result<T>, path: &Path) -> Result<Optio
Err(e) => Err(anyhow!(e).context(format!("reading file {}", path.display()))),
}
}
#[cfg(not(target_os = "windows"))]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
use turbo_unix_path::sys_to_unix;
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
Ok(format!(
"file://{}",
&sys_to_unix(
&root_fs
.to_sys_path(match path {
Some(path) => root.join(path)?,
None => root,
})?
.to_string_lossy()
)
.split('/')
.map(|s| urlencoding::encode(s))
.collect::<Vec<_>>()
.join("/")
))
}
#[cfg(target_os = "windows")]
pub async fn uri_from_file(root: FileSystemPath, path: Option<&str>) -> Result<String> {
let root_fs = root.fs;
let root_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(root_fs)
.context("Expected root to have a DiskFileSystem")?
.await?;
let sys_path = root_fs.to_sys_path(match path {
Some(path) => root.join(path.into())?,
None => root,
})?;
let raw_path = sys_path.to_string_lossy().to_string();
let normalized_path = raw_path.replace('\\', "/");
let mut segments = normalized_path.split('/');
let first = segments.next().unwrap_or_default(); // e.g., "C:"
let encoded_path = std::iter::once(first.to_string()) // keep "C:" intact
.chain(segments.map(|s| urlencoding::encode(s).into_owned()))
.collect::<Vec<_>>()
.join("/");
let uri = format!("file:///{}", encoded_path);
Ok(uri)
}
......@@ -24,6 +24,7 @@ either = { workspace = true }
indexmap = { workspace = true }
once_cell = { workspace = true }
patricia_tree = "0.5.5"
percent-encoding = { workspace = true }
petgraph = { workspace = true, features = ["serde-1"] }
roaring = { workspace = true, features = ["serde"] }
ref-cast = "1.0.20"
......@@ -43,10 +44,10 @@ turbo-tasks-env = { workspace = true }
turbo-tasks-fs = { workspace = true }
turbo-tasks-hash = { workspace = true }
turbo-unix-path = { workspace = true }
turbopack-url = { workspace = true }
url = { workspace = true }
urlencoding = { workspace = true }
[dev-dependencies]
rstest = { workspace = true }
tokio = { workspace = true }
......
......@@ -3,13 +3,13 @@ use std::{borrow::Cow, collections::HashSet, iter};
use anyhow::{Context, Result};
use const_format::concatcp;
use once_cell::sync::Lazy;
use percent_encoding::{percent_decode_str, utf8_percent_encode};
use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use turbo_tasks::{ResolvedVc, ValueToString};
use turbo_tasks_fs::{
DiskFileSystem, FileContent, FileSystemPath, rope::Rope, util::uri_from_file,
};
use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystemPath, rope::Rope};
use turbopack_url::{SPECIAL_PATH_SEGMENT, percent_encode_str};
use url::Url;
use crate::SOURCE_URL_PROTOCOL;
......@@ -132,11 +132,17 @@ pub async fn resolve_source_map_sources(
};
if let Some(fs_path) = fs_path {
// TODO: Encode `fs_str` and `fs_path_str` using `urlencoding`, so that these are
// valid URLs. However, `project_trace_source_operation` (and `uri_from_file`) need
// to handle percent encoding correctly first.
let fs_path_str = &fs_path.path;
*source_url = format!("{SOURCE_URL_PROTOCOL}///{fs_str}/{fs_path_str}");
*source_url = format!(
"{}///{}/{}",
SOURCE_URL_PROTOCOL,
utf8_percent_encode(fs_str, SPECIAL_PATH_SEGMENT),
fs_path_str
.split('/')
.map(|segment| percent_encode_str(segment, SPECIAL_PATH_SEGMENT))
.collect::<Vec<_>>()
.join("/"),
);
if let Some(source_content) = source_content
&& source_content.is_none()
......@@ -239,18 +245,40 @@ pub async fn fileify_source_map(
return Ok(None);
};
// read and unset `source_root`: we flatten URLs and should never emit `source_root`.
let source_root = map.source_root.take();
let context_fs = context_path.fs;
let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
.context("Expected the chunking context to have a DiskFileSystem")?
.await?;
let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL, context_fs.name());
let context_fs_name = context_fs.name();
let context_base_url = Url::from_directory_path(context_fs.root_sys_path()).ok();
debug_assert_ne!(
context_base_url, None,
"failed to convert context_fs root to URL"
);
let transform_source = async |src: &mut Option<String>| {
if let Some(src) = src
&& let Some(src_rest) = src.strip_prefix(&prefix)
{
*src = uri_from_file(context_path.clone(), Some(src_rest)).await?;
let Some(src) = src else { return Ok(()) };
if let Some(root) = &source_root {
// flatten, regardless of url format. We unset `map.source_root` above.
*src = format!("{root}{src}");
}
let Some(context_base_url) = &context_base_url else {
return anyhow::Ok(());
};
let Some(file_url) =
turbopack_url_to_file_url(src, context_fs_name, context_base_url.as_str())
else {
return anyhow::Ok(());
};
*src = file_url;
anyhow::Ok(())
};
......@@ -268,6 +296,61 @@ pub async fn fileify_source_map(
Ok(Some(map))
}
/// Takes a `turbopack:///[fs_name]/` url and strips the prefix, leaving a percent-encoded relative
/// path string that's valid as a suffix in `file://` URLs.
pub fn turbopack_url_to_fs_relative_file_url<'a>(
turbopack_url: &'a str,
fs_name: &str,
) -> Option<&'a str> {
let (scheme, remaining_url) = turbopack_url.split_once(":")?;
// URL schemes don't need percent decoding, just case insensitive equality is sufficient
if !scheme.eq_ignore_ascii_case(&SOURCE_URL_PROTOCOL[..SOURCE_URL_PROTOCOL.len() - 1]) {
return None;
}
if cfg!(debug_assertions) {
let url = Url::parse(turbopack_url).expect("turbopack:// urls should parse as valid URLs");
assert_eq!(
url.host_str(),
None,
"turbopack:// URLs should not have a host",
);
}
// Modify the URL using string manipulation, as we can make some assumptions about the
// format of `turbopack://` urls, so it's cheaper.
let remaining_url = remaining_url.strip_prefix("///")?;
let (decoded_fs_name, remaining_url) =
remaining_url.split_once("/").unwrap_or((remaining_url, ""));
let decoded_fs_name = percent_decode_str(decoded_fs_name).decode_utf8().ok()?;
let decoded_fs_name_suffix = decoded_fs_name
.strip_prefix("[")?
.strip_prefix(fs_name)?
.strip_prefix("]")?;
if !decoded_fs_name_suffix.is_empty() {
if cfg!(debug_assertions) {
panic!("{turbopack_url} has a malformed filesystem name ({decoded_fs_name})");
}
return None;
}
Some(remaining_url)
}
/// Return an absolute URL contructed from `turbopack_url`, assuming `fs_name` matches.
///
/// Use [`Url::from_directory_path`] to convert a [`DiskFileSystem::root_sys_path`] to a `base_url`.
fn turbopack_url_to_file_url(turbopack_url: &str, fs_name: &str, base_url: &str) -> Option<String> {
// Both URL formats use the same percent encoding
Some(format!(
"{}{}",
base_url,
turbopack_url_to_fs_relative_file_url(turbopack_url, fs_name)?,
))
}
#[cfg(test)]
mod tests {
use std::path::Path;
......
[package]
name = "turbopack-url"
version = "0.1.0"
description = "TBD"
license = "MIT"
edition = "2024"
[lib]
bench = false
[lints]
workspace = true
[dependencies]
url = { workspace = true }
percent-encoding = { workspace = true }
use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode};
pub mod url_spec;
/// An `AsciiSet` that matches the behavior of JavaScript's `encodeURIComponent`.
/// - It leaves `A-Z a-z 0-9 - _ . ~` unescaped.
/// - It percent-encodes all other ASCII characters (and of course non-ASCII).
/// - The `CONTROLS` set covers `\0`-`\x1F` and `\x7F`.
const ENCODE_URI_COMPONENT_SET: &AsciiSet = &CONTROLS
// Add everything else JS `encodeURIComponent` would encode:
.add(b' ')
// .add(b'!')
.add(b'"')
.add(b'#')
.add(b'$')
.add(b'%')
.add(b'&')
.add(b'\'')
// .add(b'(')
// .add(b')')
.add(b'*')
.add(b'+')
.add(b',')
.add(b'/')
.add(b':')
.add(b';')
.add(b'<')
.add(b'=')
.add(b'>')
.add(b'?')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'`')
.add(b'{')
.add(b'|')
// .add(b'~');
.add(b'}');
use std::borrow::Cow;
pub fn encode_uri_component(input: &str) -> String {
utf8_percent_encode(input, ENCODE_URI_COMPONENT_SET).to_string()
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
pub use url_spec::*;
/// An [`AsciiSet`] that matches the behavior of JavaScript's [`encodeURIComponent`][mdn].
/// It escapes all characters except:
///
/// ```text
/// A-Z a-z 0–9 - _ . ! ~ * ' ( )
/// ```
///
/// [mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent#description
///
/// Because this is designed to work with *any* URL component, it encodes more characters than the
/// spec requires. For less-agressive encoding sets, see [`FRAGMENT`], [`PATH`], [`USERINFO`],
/// [`PATH_SEGMENT`], [`SPECIAL_PATH_SEGMENT`], [`QUERY`], and [`SPECIAL_QUERY`].
pub const ENCODE_URI_COMPONENT: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'!')
.remove(b'~')
.remove(b'*')
.remove(b'\'')
.remove(b'(')
.remove(b')');
/// A convenience wrapper around [`percent_encoding::utf8_percent_encode`] that returns a
/// [`Cow<'_, str>`] instead of a [`percent_encoding::PercentEncode`].
///
/// If you're just using the [`Display`][std::fmt::Display] trait on the return value, you should
/// use the functions from [`percent_encoding`] directly, as the
/// [`PercentEncode`][percent_encoding::PercentEncode] type can avoid creating an intermediate
/// [`String`].
pub fn percent_encode_str<'a>(input: &'a str, ascii_set: &'static AsciiSet) -> Cow<'a, str> {
Cow::from(utf8_percent_encode(input, ascii_set))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_percent_encode_str() {
assert!(matches!(
percent_encode_str("nowhitespace", NON_ALPHANUMERIC),
Cow::Borrowed(_),
));
assert!(matches!(
percent_encode_str("has whitespace", NON_ALPHANUMERIC),
Cow::Owned(_),
));
}
#[test]
fn test_encode_uri_component() {
// Each (input, expected_output)
let test_cases = vec![
let test_cases = [
("Hello", "Hello"),
("Hello World", "Hello%20World"),
(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~",
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~",
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~*'()",
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~*'()",
),
("This is 100% test!", "This%20is%20100%25%20test!"),
("你好", "%E4%BD%A0%E5%A5%BD"),
......@@ -105,7 +115,7 @@ mod tests {
];
for (input, expected) in test_cases {
let actual = encode_uri_component(input);
let actual = percent_encode_str(input, ENCODE_URI_COMPONENT);
assert_eq!(
actual, expected,
"Failed on input='{input}': expected='{expected}', got='{actual}'",
......
// Copyright 2013-2016 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! Percent encoding sets defined in the WHATWG URL spec.
//!
//! These are taken from the `url` crate:
//! https://github.com/servo/rust-url/blob/22b925f93ad505a830f1089538a9ed6f5fd90612/url/src/parser.rs#L19-L46
use percent_encoding::{AsciiSet, CONTROLS};
/// https://url.spec.whatwg.org/#fragment-percent-encode-set
pub const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`');
/// https://url.spec.whatwg.org/#path-percent-encode-set
pub const PATH: &AsciiSet = &FRAGMENT.add(b'#').add(b'?').add(b'{').add(b'}');
/// https://url.spec.whatwg.org/#userinfo-percent-encode-set
pub const USERINFO: &AsciiSet = &PATH
.add(b'/')
.add(b':')
.add(b';')
.add(b'=')
.add(b'@')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'|');
pub const PATH_SEGMENT: &AsciiSet = &PATH.add(b'/').add(b'%');
/// An extended variant of [`PATH_SEGMENT`] for `http`, `https`, `ws`, `wss`, `ftp`, and `file` url
/// schemes.
///
/// The backslash (\) character is treated as a path separator in special URLs so it needs to be
/// additionally escaped in that case.
pub const SPECIAL_PATH_SEGMENT: &AsciiSet = &PATH_SEGMENT.add(b'\\');
// https://url.spec.whatwg.org/#query-state
pub const QUERY: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'#').add(b'<').add(b'>');
/// An extended variant of [`QUERY`] for `http`, `https`, `ws`, `wss`, `ftp`, and `file` url
/// schemes.
///
/// The backslash (\) character is treated as a path separator in special URLs so it needs to be
/// additionally escaped in that case.
pub const SPECIAL_QUERY: &AsciiSet = &QUERY.add(b'\'');
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment