Unverified Commit e9eddf01 authored by Benjamin Woodruff's avatar Benjamin Woodruff Committed by GitHub

Turbopack: Make turbo-tasks-fetch a bit more OOP-like (#81995)

This was a request from @lukesandberg to make this API a bit more object-oriented: https://github.com/vercel/next.js/pull/81818#discussion_r2223136594

- `ReqwestClientConfig` is now just `FetchClient`.
- `fetch` is a method on `FetchClient` instead of a function taking a config.
- **Bonus change:** `lib.rs` is now a tiny stub that just re-exports things from other modules (`client.rs`, `error.rs`, `response.rs`).
parent 1964b17e
...@@ -9,7 +9,7 @@ use turbo_tasks::{ ...@@ -9,7 +9,7 @@ use turbo_tasks::{
trace::TraceRawVcs, trace::TraceRawVcs,
}; };
use turbo_tasks_env::{EnvMap, ProcessEnv}; use turbo_tasks_env::{EnvMap, ProcessEnv};
use turbo_tasks_fetch::ReqwestClientConfig; use turbo_tasks_fetch::FetchClient;
use turbo_tasks_fs::FileSystemPath; use turbo_tasks_fs::FileSystemPath;
use turbopack::module_options::{ use turbopack::module_options::{
ConditionItem, ConditionPath, LoaderRuleItem, OptionWebpackRules, ConditionItem, ConditionPath, LoaderRuleItem, OptionWebpackRules,
...@@ -1725,10 +1725,7 @@ impl NextConfig { ...@@ -1725,10 +1725,7 @@ impl NextConfig {
} }
#[turbo_tasks::function] #[turbo_tasks::function]
pub async fn reqwest_client_config( pub async fn fetch_client(&self, env: Vc<Box<dyn ProcessEnv>>) -> Result<Vc<FetchClient>> {
&self,
env: Vc<Box<dyn ProcessEnv>>,
) -> Result<Vc<ReqwestClientConfig>> {
// Support both an env var and the experimental flag to provide more flexibility to // Support both an env var and the experimental flag to provide more flexibility to
// developers on locked down systems, depending on if they want to configure this on a // developers on locked down systems, depending on if they want to configure this on a
// per-system or per-project basis. // per-system or per-project basis.
...@@ -1742,7 +1739,7 @@ impl NextConfig { ...@@ -1742,7 +1739,7 @@ impl NextConfig {
}) })
.or(self.experimental.turbopack_use_system_tls_certs) .or(self.experimental.turbopack_use_system_tls_certs)
.unwrap_or(false); .unwrap_or(false);
Ok(ReqwestClientConfig { Ok(FetchClient {
tls_built_in_webpki_certs: !use_system_tls_certs, tls_built_in_webpki_certs: !use_system_tls_certs,
tls_built_in_native_certs: use_system_tls_certs, tls_built_in_native_certs: use_system_tls_certs,
} }
......
...@@ -9,7 +9,7 @@ use turbo_rcstr::{RcStr, rcstr}; ...@@ -9,7 +9,7 @@ use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{Completion, FxIndexMap, ResolvedVc, Vc}; use turbo_tasks::{Completion, FxIndexMap, ResolvedVc, Vc};
use turbo_tasks_bytes::stream::SingleValue; use turbo_tasks_bytes::stream::SingleValue;
use turbo_tasks_env::{CommandLineProcessEnv, ProcessEnv}; use turbo_tasks_env::{CommandLineProcessEnv, ProcessEnv};
use turbo_tasks_fetch::{HttpResponseBody, ReqwestClientConfig, fetch}; use turbo_tasks_fetch::{FetchClient, HttpResponseBody};
use turbo_tasks_fs::{ use turbo_tasks_fs::{
DiskFileSystem, File, FileContent, FileSystem, FileSystemPath, DiskFileSystem, File, FileContent, FileSystem, FileSystemPath,
json::parse_json_with_source_context, json::parse_json_with_source_context,
...@@ -182,7 +182,7 @@ pub struct NextFontGoogleCssModuleReplacer { ...@@ -182,7 +182,7 @@ pub struct NextFontGoogleCssModuleReplacer {
project_path: FileSystemPath, project_path: FileSystemPath,
execution_context: ResolvedVc<ExecutionContext>, execution_context: ResolvedVc<ExecutionContext>,
next_mode: ResolvedVc<NextMode>, next_mode: ResolvedVc<NextMode>,
reqwest_client_config: ResolvedVc<ReqwestClientConfig>, fetch_client: ResolvedVc<FetchClient>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
...@@ -192,13 +192,13 @@ impl NextFontGoogleCssModuleReplacer { ...@@ -192,13 +192,13 @@ impl NextFontGoogleCssModuleReplacer {
project_path: FileSystemPath, project_path: FileSystemPath,
execution_context: ResolvedVc<ExecutionContext>, execution_context: ResolvedVc<ExecutionContext>,
next_mode: ResolvedVc<NextMode>, next_mode: ResolvedVc<NextMode>,
reqwest_client_config: ResolvedVc<ReqwestClientConfig>, fetch_client: ResolvedVc<FetchClient>,
) -> Vc<Self> { ) -> Vc<Self> {
Self::cell(NextFontGoogleCssModuleReplacer { Self::cell(NextFontGoogleCssModuleReplacer {
project_path, project_path,
execution_context, execution_context,
next_mode, next_mode,
reqwest_client_config, fetch_client,
}) })
} }
...@@ -233,9 +233,9 @@ impl NextFontGoogleCssModuleReplacer { ...@@ -233,9 +233,9 @@ impl NextFontGoogleCssModuleReplacer {
.map_or_else( .map_or_else(
|| { || {
fetch_real_stylesheet( fetch_real_stylesheet(
*self.fetch_client,
stylesheet_url.clone(), stylesheet_url.clone(),
css_virtual_path.clone(), css_virtual_path.clone(),
*self.reqwest_client_config,
) )
.boxed() .boxed()
}, },
...@@ -375,19 +375,16 @@ struct NextFontGoogleFontFileOptions { ...@@ -375,19 +375,16 @@ struct NextFontGoogleFontFileOptions {
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
pub struct NextFontGoogleFontFileReplacer { pub struct NextFontGoogleFontFileReplacer {
project_path: FileSystemPath, project_path: FileSystemPath,
reqwest_client_config: ResolvedVc<ReqwestClientConfig>, fetch_client: ResolvedVc<FetchClient>,
} }
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl NextFontGoogleFontFileReplacer { impl NextFontGoogleFontFileReplacer {
#[turbo_tasks::function] #[turbo_tasks::function]
pub fn new( pub fn new(project_path: FileSystemPath, fetch_client: ResolvedVc<FetchClient>) -> Vc<Self> {
project_path: FileSystemPath,
reqwest_client_config: ResolvedVc<ReqwestClientConfig>,
) -> Vc<Self> {
Self::cell(NextFontGoogleFontFileReplacer { Self::cell(NextFontGoogleFontFileReplacer {
project_path, project_path,
reqwest_client_config, fetch_client,
}) })
} }
} }
...@@ -443,12 +440,9 @@ impl ImportMappingReplacement for NextFontGoogleFontFileReplacer { ...@@ -443,12 +440,9 @@ impl ImportMappingReplacement for NextFontGoogleFontFileReplacer {
// doesn't seem ideal to download the font into a string, but probably doesn't // doesn't seem ideal to download the font into a string, but probably doesn't
// really matter either. // really matter either.
let Some(font) = fetch_from_google_fonts( let Some(font) =
url.into(), fetch_from_google_fonts(*self.fetch_client, url.into(), font_virtual_path.clone())
font_virtual_path.clone(), .await?
*self.reqwest_client_config,
)
.await?
else { else {
return Ok(ImportMapResult::Result(ResolveResult::unresolvable()).cell()); return Ok(ImportMapResult::Result(ResolveResult::unresolvable()).cell());
}; };
...@@ -670,27 +664,23 @@ fn font_file_options_from_query_map(query: &RcStr) -> Result<NextFontGoogleFontF ...@@ -670,27 +664,23 @@ fn font_file_options_from_query_map(query: &RcStr) -> Result<NextFontGoogleFontF
} }
async fn fetch_real_stylesheet( async fn fetch_real_stylesheet(
fetch_client: Vc<FetchClient>,
stylesheet_url: RcStr, stylesheet_url: RcStr,
css_virtual_path: FileSystemPath, css_virtual_path: FileSystemPath,
reqwest_client_config: Vc<ReqwestClientConfig>,
) -> Result<Option<Vc<RcStr>>> { ) -> Result<Option<Vc<RcStr>>> {
let body = let body = fetch_from_google_fonts(fetch_client, stylesheet_url, css_virtual_path).await?;
fetch_from_google_fonts(stylesheet_url, css_virtual_path, reqwest_client_config).await?;
Ok(body.map(|body| body.to_string())) Ok(body.map(|body| body.to_string()))
} }
async fn fetch_from_google_fonts( async fn fetch_from_google_fonts(
fetch_client: Vc<FetchClient>,
url: RcStr, url: RcStr,
virtual_path: FileSystemPath, virtual_path: FileSystemPath,
reqwest_client_config: Vc<ReqwestClientConfig>,
) -> Result<Option<Vc<HttpResponseBody>>> { ) -> Result<Option<Vc<HttpResponseBody>>> {
let result = fetch( let result = fetch_client
url, .fetch(url, Some(rcstr!(USER_AGENT_FOR_GOOGLE_FONTS)))
Some(rcstr!(USER_AGENT_FOR_GOOGLE_FONTS)), .await?;
reqwest_client_config,
)
.await?;
Ok(match *result { Ok(match *result {
Ok(r) => Some(*r.await?.body), Ok(r) => Some(*r.await?.body),
......
...@@ -1022,7 +1022,7 @@ async fn insert_next_shared_aliases( ...@@ -1022,7 +1022,7 @@ async fn insert_next_shared_aliases(
next_font_google_replacer_mapping, next_font_google_replacer_mapping,
); );
let reqwest_client_config = next_config.reqwest_client_config(execution_context.env()); let fetch_client = next_config.fetch_client(execution_context.env());
import_map.insert_alias( import_map.insert_alias(
AliasPattern::exact("@vercel/turbopack-next/internal/font/google/cssmodule.module.css"), AliasPattern::exact("@vercel/turbopack-next/internal/font/google/cssmodule.module.css"),
ImportMapping::Dynamic(ResolvedVc::upcast( ImportMapping::Dynamic(ResolvedVc::upcast(
...@@ -1030,7 +1030,7 @@ async fn insert_next_shared_aliases( ...@@ -1030,7 +1030,7 @@ async fn insert_next_shared_aliases(
project_path.clone(), project_path.clone(),
execution_context, execution_context,
next_mode, next_mode,
reqwest_client_config, fetch_client,
) )
.to_resolved() .to_resolved()
.await?, .await?,
...@@ -1041,7 +1041,7 @@ async fn insert_next_shared_aliases( ...@@ -1041,7 +1041,7 @@ async fn insert_next_shared_aliases(
import_map.insert_alias( import_map.insert_alias(
AliasPattern::exact(GOOGLE_FONTS_INTERNAL_PREFIX), AliasPattern::exact(GOOGLE_FONTS_INTERNAL_PREFIX),
ImportMapping::Dynamic(ResolvedVc::upcast( ImportMapping::Dynamic(ResolvedVc::upcast(
NextFontGoogleFontFileReplacer::new(project_path.clone(), reqwest_client_config) NextFontGoogleFontFileReplacer::new(project_path.clone(), fetch_client)
.to_resolved() .to_resolved()
.await?, .await?,
)) ))
......
...@@ -20,7 +20,7 @@ workspace = true ...@@ -20,7 +20,7 @@ workspace = true
[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies] [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
reqwest = { workspace = true, features = ["native-tls"] } reqwest = { workspace = true, features = ["native-tls"] }
# If you modify this cfg, make sure you update `ReqwestClientConfig::try_build`. # If you modify this cfg, make sure you update `FetchClient::try_build_uncached_reqwest_client`.
[target.'cfg(not(any(all(target_os = "windows", target_arch = "aarch64"), target_arch="wasm32")))'.dependencies] [target.'cfg(not(any(all(target_os = "windows", target_arch = "aarch64"), target_arch="wasm32")))'.dependencies]
reqwest = { workspace = true, features = ["rustls-tls-webpki-roots", "rustls-tls-native-roots"] } reqwest = { workspace = true, features = ["rustls-tls-webpki-roots", "rustls-tls-native-roots"] }
......
use std::{hash::Hash, sync::LazyLock}; use std::{hash::Hash, sync::LazyLock};
use anyhow::Result;
use quick_cache::sync::Cache; use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
use turbo_tasks::{NonLocalValue, ReadRef, trace::TraceRawVcs}; use turbo_tasks::{
NonLocalValue, ReadRef, Vc, duration_span, mark_session_dependent, trace::TraceRawVcs,
};
use crate::{FetchError, FetchResult, HttpResponse, HttpResponseBody};
const MAX_CLIENTS: usize = 16; const MAX_CLIENTS: usize = 16;
static CLIENT_CACHE: LazyLock<Cache<ReadRef<ReqwestClientConfig>, reqwest::Client>> = static CLIENT_CACHE: LazyLock<Cache<ReadRef<FetchClient>, reqwest::Client>> =
LazyLock::new(|| Cache::new(MAX_CLIENTS)); LazyLock::new(|| Cache::new(MAX_CLIENTS));
#[derive(Hash, PartialEq, Eq, Serialize, Deserialize, NonLocalValue, Debug, TraceRawVcs)] #[derive(Hash, PartialEq, Eq, Serialize, Deserialize, NonLocalValue, Debug, TraceRawVcs)]
...@@ -24,7 +29,7 @@ pub enum ProxyConfig { ...@@ -24,7 +29,7 @@ pub enum ProxyConfig {
/// factory cannot be a closure because closures do not implement `Eq` or `Hash`. /// factory cannot be a closure because closures do not implement `Eq` or `Hash`.
#[turbo_tasks::value(shared)] #[turbo_tasks::value(shared)]
#[derive(Hash)] #[derive(Hash)]
pub struct ReqwestClientConfig { pub struct FetchClient {
/// Whether to load embedded webpki root certs with rustls. Default is true. /// Whether to load embedded webpki root certs with rustls. Default is true.
/// ///
/// Ignored for: /// Ignored for:
...@@ -40,7 +45,7 @@ pub struct ReqwestClientConfig { ...@@ -40,7 +45,7 @@ pub struct ReqwestClientConfig {
pub tls_built_in_native_certs: bool, pub tls_built_in_native_certs: bool,
} }
impl Default for ReqwestClientConfig { impl Default for FetchClient {
fn default() -> Self { fn default() -> Self {
Self { Self {
tls_built_in_webpki_certs: true, tls_built_in_webpki_certs: true,
...@@ -49,8 +54,29 @@ impl Default for ReqwestClientConfig { ...@@ -49,8 +54,29 @@ impl Default for ReqwestClientConfig {
} }
} }
impl ReqwestClientConfig { impl FetchClient {
fn try_build(&self) -> reqwest::Result<reqwest::Client> { /// Returns a cached instance of `reqwest::Client` it exists, otherwise constructs a new one.
///
/// The cache is bound in size to prevent accidental blowups or leaks. However, in practice,
/// very few clients should be created, likely only when the bundler configuration changes.
///
/// Client construction is largely deterministic, aside from changes to system TLS
/// configuration.
///
/// The reqwest client fails to construct if the TLS backend cannot be initialized, or the
/// resolver cannot load the system configuration. These failures should be treated as
/// cached for some amount of time, but ultimately transient (e.g. using
/// [`turbo_tasks::mark_session_dependent`]).
pub fn try_get_cached_reqwest_client(
self: ReadRef<FetchClient>,
) -> reqwest::Result<reqwest::Client> {
CLIENT_CACHE.get_or_insert_with(&self, {
let this = ReadRef::clone(&self);
move || this.try_build_uncached_reqwest_client()
})
}
fn try_build_uncached_reqwest_client(&self) -> reqwest::Result<reqwest::Client> {
let client_builder = reqwest::Client::builder(); let client_builder = reqwest::Client::builder();
// make sure this cfg matches the one in `Cargo.toml`! // make sure this cfg matches the one in `Cargo.toml`!
...@@ -67,23 +93,56 @@ impl ReqwestClientConfig { ...@@ -67,23 +93,56 @@ impl ReqwestClientConfig {
} }
} }
/// Given a config, returns a cached instance if it exists, otherwise constructs a new one. #[turbo_tasks::value_impl]
/// impl FetchClient {
/// The cache is bound in size to prevent accidental blowups or leaks. However, in practice, very #[turbo_tasks::function(network)]
/// few clients should be created, likely only when the bundler configuration changes. pub async fn fetch(
/// self: Vc<FetchClient>,
/// Client construction is largely deterministic, aside from changes to system TLS configuration. url: RcStr,
/// user_agent: Option<RcStr>,
/// The reqwest client fails to construct if the TLS backend cannot be initialized, or the resolver ) -> Result<Vc<FetchResult>> {
/// cannot load the system configuration. These failures should be treated as cached for some amount let url_ref = &*url;
/// of time, but ultimately transient (e.g. using [`turbo_tasks::mark_session_dependent`]). let this = self.await?;
pub fn try_get_cached_reqwest_client( let response_result: reqwest::Result<HttpResponse> = async move {
config: ReadRef<ReqwestClientConfig>, let reqwest_client = this.try_get_cached_reqwest_client()?;
) -> reqwest::Result<reqwest::Client> {
CLIENT_CACHE.get_or_insert_with(&config, { let mut builder = reqwest_client.get(url_ref);
let config = ReadRef::clone(&config); if let Some(user_agent) = user_agent {
move || config.try_build() builder = builder.header("User-Agent", user_agent.as_str());
}) }
let response = {
let _span = duration_span!("fetch request", url = url_ref);
builder.send().await
}
.and_then(|r| r.error_for_status())?;
let status = response.status().as_u16();
let body = {
let _span = duration_span!("fetch response", url = url_ref);
response.bytes().await?
}
.to_vec();
Ok(HttpResponse {
status,
body: HttpResponseBody(body).resolved_cell(),
})
}
.await;
match response_result {
Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))),
Err(err) => {
// the client failed to construct or the HTTP request failed
mark_session_dependent();
Ok(Vc::cell(Err(
FetchError::from_reqwest_error(&err, &url).resolved_cell()
)))
}
}
}
} }
#[doc(hidden)] #[doc(hidden)]
......
use anyhow::Result;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc};
use turbo_tasks_fs::FileSystemPath;
use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString};
#[derive(Debug)]
#[turbo_tasks::value(shared)]
pub enum FetchErrorKind {
Connect,
Timeout,
Status(u16),
Other,
}
#[turbo_tasks::value(shared)]
pub struct FetchError {
pub url: ResolvedVc<RcStr>,
pub kind: ResolvedVc<FetchErrorKind>,
pub detail: ResolvedVc<StyledString>,
}
impl FetchError {
pub(crate) fn from_reqwest_error(error: &reqwest::Error, url: &str) -> FetchError {
let kind = if error.is_connect() {
FetchErrorKind::Connect
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else if let Some(status) = error.status() {
FetchErrorKind::Status(status.as_u16())
} else {
FetchErrorKind::Other
};
FetchError {
detail: StyledString::Text(error.to_string().into()).resolved_cell(),
url: ResolvedVc::cell(url.into()),
kind: kind.resolved_cell(),
}
}
}
#[turbo_tasks::value_impl]
impl FetchError {
#[turbo_tasks::function]
pub fn to_issue(
&self,
severity: IssueSeverity,
issue_context: FileSystemPath,
) -> Vc<FetchIssue> {
FetchIssue {
issue_context,
severity,
url: self.url,
kind: self.kind,
detail: self.detail,
}
.cell()
}
}
#[turbo_tasks::value(shared)]
pub struct FetchIssue {
pub issue_context: FileSystemPath,
pub severity: IssueSeverity,
pub url: ResolvedVc<RcStr>,
pub kind: ResolvedVc<FetchErrorKind>,
pub detail: ResolvedVc<StyledString>,
}
#[turbo_tasks::value_impl]
impl Issue for FetchIssue {
#[turbo_tasks::function]
fn file_path(&self) -> Vc<FileSystemPath> {
self.issue_context.clone().cell()
}
fn severity(&self) -> IssueSeverity {
self.severity
}
#[turbo_tasks::function]
fn title(&self) -> Vc<StyledString> {
StyledString::Text(rcstr!("Error while requesting resource")).cell()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Load.into()
}
#[turbo_tasks::function]
async fn description(&self) -> Result<Vc<OptionStyledString>> {
let url = &*self.url.await?;
let kind = &*self.kind.await?;
Ok(Vc::cell(Some(
StyledString::Text(match kind {
FetchErrorKind::Connect => {
format!("There was an issue establishing a connection while requesting {url}.")
.into()
}
FetchErrorKind::Status(status) => {
format!("Received response with status {status} when requesting {url}").into()
}
FetchErrorKind::Timeout => {
format!("Connection timed out when requesting {url}").into()
}
FetchErrorKind::Other => format!("There was an issue requesting {url}").into(),
})
.resolved_cell(),
)))
}
#[turbo_tasks::function]
fn detail(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(self.detail))
}
}
...@@ -2,18 +2,17 @@ ...@@ -2,18 +2,17 @@
#![feature(arbitrary_self_types)] #![feature(arbitrary_self_types)]
#![feature(arbitrary_self_types_pointers)] #![feature(arbitrary_self_types_pointers)]
mod reqwest_client_cache; mod client;
mod error;
use anyhow::Result; mod response;
use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::{ResolvedVc, Vc, duration_span, mark_session_dependent}; pub use crate::{
use turbo_tasks_fs::FileSystemPath; client::{
use turbopack_core::issue::{Issue, IssueSeverity, IssueStage, OptionStyledString, StyledString}; __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchClient,
ProxyConfig,
use crate::reqwest_client_cache::try_get_cached_reqwest_client; },
pub use crate::reqwest_client_cache::{ error::{FetchError, FetchErrorKind, FetchIssue},
__test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, ProxyConfig, response::{FetchResult, HttpResponse, HttpResponseBody},
ReqwestClientConfig,
}; };
pub fn register() { pub fn register() {
...@@ -22,189 +21,3 @@ pub fn register() { ...@@ -22,189 +21,3 @@ pub fn register() {
turbopack_core::register(); turbopack_core::register();
include!(concat!(env!("OUT_DIR"), "/register.rs")); include!(concat!(env!("OUT_DIR"), "/register.rs"));
} }
#[turbo_tasks::value(transparent)]
pub struct FetchResult(Result<ResolvedVc<HttpResponse>, ResolvedVc<FetchError>>);
#[turbo_tasks::value(shared)]
#[derive(Debug)]
pub struct HttpResponse {
pub status: u16,
pub body: ResolvedVc<HttpResponseBody>,
}
#[turbo_tasks::value(shared)]
#[derive(Debug)]
pub struct HttpResponseBody(pub Vec<u8>);
#[turbo_tasks::value_impl]
impl HttpResponseBody {
#[turbo_tasks::function]
pub async fn to_string(self: Vc<Self>) -> Result<Vc<RcStr>> {
let this = &*self.await?;
Ok(Vc::cell(std::str::from_utf8(&this.0)?.into()))
}
}
#[turbo_tasks::function(network)]
pub async fn fetch(
url: RcStr,
user_agent: Option<RcStr>,
client_config: Vc<ReqwestClientConfig>,
) -> Result<Vc<FetchResult>> {
let url_ref = &*url;
let client_config = client_config.await?;
let response_result: reqwest::Result<HttpResponse> = async move {
let client = try_get_cached_reqwest_client(client_config)?;
let mut builder = client.get(url_ref);
if let Some(user_agent) = user_agent {
builder = builder.header("User-Agent", user_agent.as_str());
}
let response = {
let _span = duration_span!("fetch request", url = url_ref);
builder.send().await
}
.and_then(|r| r.error_for_status())?;
let status = response.status().as_u16();
let body = {
let _span = duration_span!("fetch response", url = url_ref);
response.bytes().await?
}
.to_vec();
Ok(HttpResponse {
status,
body: HttpResponseBody(body).resolved_cell(),
})
}
.await;
match response_result {
Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))),
Err(err) => {
// the client failed to construct or the HTTP request failed
mark_session_dependent();
Ok(Vc::cell(Err(
FetchError::from_reqwest_error(&err, &url).resolved_cell()
)))
}
}
}
#[derive(Debug)]
#[turbo_tasks::value(shared)]
pub enum FetchErrorKind {
Connect,
Timeout,
Status(u16),
Other,
}
#[turbo_tasks::value(shared)]
pub struct FetchError {
pub url: ResolvedVc<RcStr>,
pub kind: ResolvedVc<FetchErrorKind>,
pub detail: ResolvedVc<StyledString>,
}
impl FetchError {
fn from_reqwest_error(error: &reqwest::Error, url: &str) -> FetchError {
let kind = if error.is_connect() {
FetchErrorKind::Connect
} else if error.is_timeout() {
FetchErrorKind::Timeout
} else if let Some(status) = error.status() {
FetchErrorKind::Status(status.as_u16())
} else {
FetchErrorKind::Other
};
FetchError {
detail: StyledString::Text(error.to_string().into()).resolved_cell(),
url: ResolvedVc::cell(url.into()),
kind: kind.resolved_cell(),
}
}
}
#[turbo_tasks::value_impl]
impl FetchError {
#[turbo_tasks::function]
pub fn to_issue(
&self,
severity: IssueSeverity,
issue_context: FileSystemPath,
) -> Vc<FetchIssue> {
FetchIssue {
issue_context,
severity,
url: self.url,
kind: self.kind,
detail: self.detail,
}
.cell()
}
}
#[turbo_tasks::value(shared)]
pub struct FetchIssue {
pub issue_context: FileSystemPath,
pub severity: IssueSeverity,
pub url: ResolvedVc<RcStr>,
pub kind: ResolvedVc<FetchErrorKind>,
pub detail: ResolvedVc<StyledString>,
}
#[turbo_tasks::value_impl]
impl Issue for FetchIssue {
#[turbo_tasks::function]
fn file_path(&self) -> Vc<FileSystemPath> {
self.issue_context.clone().cell()
}
fn severity(&self) -> IssueSeverity {
self.severity
}
#[turbo_tasks::function]
fn title(&self) -> Vc<StyledString> {
StyledString::Text(rcstr!("Error while requesting resource")).cell()
}
#[turbo_tasks::function]
fn stage(&self) -> Vc<IssueStage> {
IssueStage::Load.into()
}
#[turbo_tasks::function]
async fn description(&self) -> Result<Vc<OptionStyledString>> {
let url = &*self.url.await?;
let kind = &*self.kind.await?;
Ok(Vc::cell(Some(
StyledString::Text(match kind {
FetchErrorKind::Connect => {
format!("There was an issue establishing a connection while requesting {url}.")
.into()
}
FetchErrorKind::Status(status) => {
format!("Received response with status {status} when requesting {url}").into()
}
FetchErrorKind::Timeout => {
format!("Connection timed out when requesting {url}").into()
}
FetchErrorKind::Other => format!("There was an issue requesting {url}").into(),
})
.resolved_cell(),
)))
}
#[turbo_tasks::function]
fn detail(&self) -> Vc<OptionStyledString> {
Vc::cell(Some(self.detail))
}
}
use anyhow::Result;
use turbo_rcstr::RcStr;
use turbo_tasks::{ResolvedVc, Vc};
use crate::FetchError;
#[turbo_tasks::value(transparent)]
pub struct FetchResult(Result<ResolvedVc<HttpResponse>, ResolvedVc<FetchError>>);
#[turbo_tasks::value(shared)]
#[derive(Debug)]
pub struct HttpResponse {
pub status: u16,
pub body: ResolvedVc<HttpResponseBody>,
}
#[turbo_tasks::value(shared)]
#[derive(Debug)]
pub struct HttpResponseBody(pub Vec<u8>);
#[turbo_tasks::value_impl]
impl HttpResponseBody {
#[turbo_tasks::function]
pub async fn to_string(self: Vc<Self>) -> Result<Vc<RcStr>> {
let this = &*self.await?;
Ok(Vc::cell(std::str::from_utf8(&this.0)?.into()))
}
}
...@@ -5,8 +5,8 @@ use tokio::sync::Mutex as TokioMutex; ...@@ -5,8 +5,8 @@ use tokio::sync::Mutex as TokioMutex;
use turbo_rcstr::{RcStr, rcstr}; use turbo_rcstr::{RcStr, rcstr};
use turbo_tasks::Vc; use turbo_tasks::Vc;
use turbo_tasks_fetch::{ use turbo_tasks_fetch::{
__test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchErrorKind, __test_only_reqwest_client_cache_clear, __test_only_reqwest_client_cache_len, FetchClient,
ReqwestClientConfig, fetch, FetchErrorKind,
}; };
use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath}; use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath};
use turbo_tasks_testing::{Registration, register, run}; use turbo_tasks_testing::{Registration, register, run};
...@@ -29,15 +29,15 @@ async fn basic_get() { ...@@ -29,15 +29,15 @@ async fn basic_get() {
.create_async() .create_async()
.await; .await;
let config_vc = ReqwestClientConfig::default().cell(); let client_vc = FetchClient::default().cell();
let response = &*fetch( let response = &*client_vc
RcStr::from(format!("{}/foo.woff", server.url())), .fetch(
/* user_agent */ None, RcStr::from(format!("{}/foo.woff", server.url())),
config_vc, /* user_agent */ None,
) )
.await? .await?
.unwrap() .unwrap()
.await?; .await?;
resource_mock.assert_async().await; resource_mock.assert_async().await;
...@@ -63,15 +63,15 @@ async fn sends_user_agent() { ...@@ -63,15 +63,15 @@ async fn sends_user_agent() {
eprintln!("{}", server.url()); eprintln!("{}", server.url());
let config_vc = ReqwestClientConfig::default().cell(); let client_vc = FetchClient::default().cell();
let response = &*fetch( let response = &*client_vc
RcStr::from(format!("{}/foo.woff", server.url())), .fetch(
Some(rcstr!("mock-user-agent")), RcStr::from(format!("{}/foo.woff", server.url())),
config_vc, Some(rcstr!("mock-user-agent")),
) )
.await? .await?
.unwrap() .unwrap()
.await?; .await?;
resource_mock.assert_async().await; resource_mock.assert_async().await;
...@@ -98,8 +98,9 @@ async fn invalidation_does_not_invalidate() { ...@@ -98,8 +98,9 @@ async fn invalidation_does_not_invalidate() {
.await; .await;
let url = RcStr::from(format!("{}/foo.woff", server.url())); let url = RcStr::from(format!("{}/foo.woff", server.url()));
let config_vc = ReqwestClientConfig::default().cell(); let client_vc = FetchClient::default().cell();
let response = &*fetch(url.clone(), /* user_agent */ None, config_vc) let response = &*client_vc
.fetch(url.clone(), /* user_agent */ None)
.await? .await?
.unwrap() .unwrap()
.await?; .await?;
...@@ -109,7 +110,8 @@ async fn invalidation_does_not_invalidate() { ...@@ -109,7 +110,8 @@ async fn invalidation_does_not_invalidate() {
assert_eq!(response.status, 200); assert_eq!(response.status, 200);
assert_eq!(*response.body.to_string().await?, "responsebody"); assert_eq!(*response.body.to_string().await?, "responsebody");
let second_response = &*fetch(url.clone(), /* user_agent */ None, config_vc) let second_response = &*client_vc
.fetch(url.clone(), /* user_agent */ None)
.await? .await?
.unwrap() .unwrap()
.await?; .await?;
...@@ -136,8 +138,8 @@ async fn errors_on_failed_connection() { ...@@ -136,8 +138,8 @@ async fn errors_on_failed_connection() {
// `ECONNREFUSED`. // `ECONNREFUSED`.
// Other values (e.g. domain name, reserved IP address block) may result in long timeouts. // Other values (e.g. domain name, reserved IP address block) may result in long timeouts.
let url = rcstr!("http://127.0.0.1:0/foo.woff"); let url = rcstr!("http://127.0.0.1:0/foo.woff");
let config_vc = ReqwestClientConfig::default().cell(); let client_vc = FetchClient::default().cell();
let response_vc = fetch(url.clone(), None, config_vc); let response_vc = client_vc.fetch(url.clone(), None);
let err_vc = &*response_vc.await?.unwrap_err(); let err_vc = &*response_vc.await?.unwrap_err();
let err = err_vc.await?; let err = err_vc.await?;
...@@ -171,8 +173,8 @@ async fn errors_on_404() { ...@@ -171,8 +173,8 @@ async fn errors_on_404() {
.await; .await;
let url = RcStr::from(server.url()); let url = RcStr::from(server.url());
let config_vc = ReqwestClientConfig::default().cell(); let client_vc = FetchClient::default().cell();
let response_vc = fetch(url.clone(), None, config_vc); let response_vc = client_vc.fetch(url.clone(), None);
let err_vc = &*response_vc.await?.unwrap_err(); let err_vc = &*response_vc.await?.unwrap_err();
let err = err_vc.await?; let err = err_vc.await?;
...@@ -197,7 +199,7 @@ async fn errors_on_404() { ...@@ -197,7 +199,7 @@ async fn errors_on_404() {
#[tokio::test] #[tokio::test]
async fn client_cache() { async fn client_cache() {
// a simple fetch that should always succeed // a simple fetch that should always succeed
async fn simple_fetch(path: &str, config: ReqwestClientConfig) -> anyhow::Result<()> { async fn simple_fetch(path: &str, client: FetchClient) -> anyhow::Result<()> {
let mut server = mockito::Server::new_async().await; let mut server = mockito::Server::new_async().await;
let _resource_mock = server let _resource_mock = server
.mock("GET", &*format!("/{path}")) .mock("GET", &*format!("/{path}"))
...@@ -206,7 +208,11 @@ async fn client_cache() { ...@@ -206,7 +208,11 @@ async fn client_cache() {
.await; .await;
let url = RcStr::from(format!("{}/{}", server.url(), path)); let url = RcStr::from(format!("{}/{}", server.url(), path));
let response = match &*fetch(url.clone(), /* user_agent */ None, config.cell()).await? { let response = match &*client
.cell()
.fetch(url.clone(), /* user_agent */ None)
.await?
{
Ok(resp) => resp.await?, Ok(resp) => resp.await?,
Err(_err) => { Err(_err) => {
anyhow::bail!("fetch error") anyhow::bail!("fetch error")
...@@ -227,7 +233,7 @@ async fn client_cache() { ...@@ -227,7 +233,7 @@ async fn client_cache() {
simple_fetch( simple_fetch(
"/foo", "/foo",
ReqwestClientConfig { FetchClient {
tls_built_in_native_certs: false, tls_built_in_native_certs: false,
..Default::default() ..Default::default()
}, },
...@@ -239,7 +245,7 @@ async fn client_cache() { ...@@ -239,7 +245,7 @@ async fn client_cache() {
// the client is reused if the config is the same (by equality) // the client is reused if the config is the same (by equality)
simple_fetch( simple_fetch(
"/bar", "/bar",
ReqwestClientConfig { FetchClient {
tls_built_in_native_certs: false, tls_built_in_native_certs: false,
..Default::default() ..Default::default()
}, },
...@@ -251,7 +257,7 @@ async fn client_cache() { ...@@ -251,7 +257,7 @@ async fn client_cache() {
// the client is recreated if the config is different // the client is recreated if the config is different
simple_fetch( simple_fetch(
"/bar", "/bar",
ReqwestClientConfig { FetchClient {
tls_built_in_native_certs: true, tls_built_in_native_certs: true,
..Default::default() ..Default::default()
}, },
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment