Unverified Commit 9da305fe authored by OJ Kwon's avatar OJ Kwon Committed by GitHub

feat(turbopack): emit MODULE_FEATURE telemetry from turbopack (#52356)



### What?

closes WEB-1274. In effort to create feature parity to existing webpack/telemetry-plugin, this PR attempts to define a new struct `ModuleFeatureTelemetry` and emit it via resolve plugin, if module & its subpath matches to the existing plugin.

next-dev (--experimental-turbo) now will emit telemetry payload with other output, however this PR does not actually `record` telemetry via `telemetry.record` since original webpack telemetry records it in prod build only.
parent 87295505
...@@ -5,7 +5,10 @@ use next_api::route::{Endpoint, WrittenEndpoint}; ...@@ -5,7 +5,10 @@ use next_api::route::{Endpoint, WrittenEndpoint};
use turbo_tasks::Vc; use turbo_tasks::Vc;
use turbopack_binding::turbopack::core::error::PrettyPrintError; use turbopack_binding::turbopack::core::error::PrettyPrintError;
use super::utils::{get_issues, subscribe, NapiIssue, RootTask, TurbopackResult, VcArc}; use super::utils::{
get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, RootTask, TurbopackResult,
VcArc,
};
#[napi(object)] #[napi(object)]
#[derive(Default)] #[derive(Default)]
...@@ -71,12 +74,13 @@ pub async fn endpoint_write_to_disk( ...@@ -71,12 +74,13 @@ pub async fn endpoint_write_to_disk(
) -> napi::Result<TurbopackResult<NapiWrittenEndpoint>> { ) -> napi::Result<TurbopackResult<NapiWrittenEndpoint>> {
let turbo_tasks = endpoint.turbo_tasks().clone(); let turbo_tasks = endpoint.turbo_tasks().clone();
let endpoint = ***endpoint; let endpoint = ***endpoint;
let (written, issues) = turbo_tasks let (written, issues, diags) = turbo_tasks
.run_once(async move { .run_once(async move {
let write_to_disk = endpoint.write_to_disk(); let write_to_disk = endpoint.write_to_disk();
let issues = get_issues(write_to_disk).await?; let issues = get_issues(write_to_disk).await?;
let diags = get_diagnostics(write_to_disk).await?;
let written = write_to_disk.strongly_consistent().await?; let written = write_to_disk.strongly_consistent().await?;
Ok((written, issues)) Ok((written, issues, diags))
}) })
.await .await
.map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?; .map_err(|e| napi::Error::from_reason(PrettyPrintError(&e).to_string()))?;
...@@ -84,7 +88,7 @@ pub async fn endpoint_write_to_disk( ...@@ -84,7 +88,7 @@ pub async fn endpoint_write_to_disk(
Ok(TurbopackResult { Ok(TurbopackResult {
result: NapiWrittenEndpoint::from(&*written), result: NapiWrittenEndpoint::from(&*written),
issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(), issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
diagnostics: vec![], diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
}) })
} }
...@@ -103,16 +107,16 @@ pub fn endpoint_changed_subscribe( ...@@ -103,16 +107,16 @@ pub fn endpoint_changed_subscribe(
move || async move { move || async move {
let changed = endpoint.changed(); let changed = endpoint.changed();
let issues = get_issues(changed).await?; let issues = get_issues(changed).await?;
let diags = get_diagnostics(changed).await?;
changed.await?; changed.await?;
// TODO diagnostics Ok((issues, diags))
Ok(issues)
}, },
|ctx| { |ctx| {
let issues = ctx.value; let (issues, diags) = ctx.value;
Ok(vec![TurbopackResult { Ok(vec![TurbopackResult {
result: (), result: (),
issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(), issues: issues.iter().map(|i| NapiIssue::from(&**i)).collect(),
diagnostics: vec![], diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
}]) }])
}, },
) )
......
...@@ -14,7 +14,8 @@ use turbopack_binding::{ ...@@ -14,7 +14,8 @@ use turbopack_binding::{
use super::{ use super::{
endpoint::ExternalEndpoint, endpoint::ExternalEndpoint,
utils::{ utils::{
get_issues, serde_enum_to_string, subscribe, NapiDiagnostic, NapiIssue, RootTask, VcArc, get_diagnostics, get_issues, serde_enum_to_string, subscribe, NapiDiagnostic, NapiIssue,
RootTask, VcArc,
}, },
}; };
use crate::register; use crate::register;
...@@ -229,12 +230,16 @@ pub fn project_entrypoints_subscribe( ...@@ -229,12 +230,16 @@ pub fn project_entrypoints_subscribe(
move || async move { move || async move {
let entrypoints = project.entrypoints(); let entrypoints = project.entrypoints();
let issues = get_issues(entrypoints).await?; let issues = get_issues(entrypoints).await?;
let diags = get_diagnostics(entrypoints).await?;
let entrypoints = entrypoints.strongly_consistent().await?; let entrypoints = entrypoints.strongly_consistent().await?;
// TODO peek_issues and diagnostics // TODO peek_issues and diagnostics
Ok((entrypoints, issues)) Ok((entrypoints, issues, diags))
}, },
move |ctx| { move |ctx| {
let (entrypoints, issues) = ctx.value; let (entrypoints, issues, diags) = ctx.value;
Ok(vec![NapiEntrypoints { Ok(vec![NapiEntrypoints {
routes: entrypoints routes: entrypoints
.routes .routes
...@@ -264,7 +269,7 @@ pub fn project_entrypoints_subscribe( ...@@ -264,7 +269,7 @@ pub fn project_entrypoints_subscribe(
.iter() .iter()
.map(|issue| NapiIssue::from(&**issue)) .map(|issue| NapiIssue::from(&**issue))
.collect(), .collect(),
diagnostics: vec![], diagnostics: diags.iter().map(|d| NapiDiagnostic::from(d)).collect(),
}]) }])
}, },
) )
......
use std::{future::Future, ops::Deref, sync::Arc}; use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc};
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use napi::{ use napi::{
...@@ -7,10 +7,11 @@ use napi::{ ...@@ -7,10 +7,11 @@ use napi::{
JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status, JsFunction, JsObject, JsUnknown, NapiRaw, NapiValue, Status,
}; };
use serde::Serialize; use serde::Serialize;
use turbo_tasks::{unit, ReadRef, TaskId, TurboTasks, Vc}; use turbo_tasks::{unit, ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc};
use turbopack_binding::{ use turbopack_binding::{
turbo::{tasks_fs::FileContent, tasks_memory::MemoryBackend}, turbo::{tasks_fs::FileContent, tasks_memory::MemoryBackend},
turbopack::core::{ turbopack::core::{
diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic},
error::PrettyPrintError, error::PrettyPrintError,
issue::{IssueContextExt, PlainIssue, PlainIssueSource, PlainSource}, issue::{IssueContextExt, PlainIssue, PlainIssueSource, PlainSource},
source_pos::SourcePos, source_pos::SourcePos,
...@@ -85,6 +86,23 @@ pub async fn get_issues<T>(source: Vc<T>) -> Result<Vec<ReadRef<PlainIssue>>> { ...@@ -85,6 +86,23 @@ pub async fn get_issues<T>(source: Vc<T>) -> Result<Vec<ReadRef<PlainIssue>>> {
issues.get_plain_issues().await issues.get_plain_issues().await
} }
/// Collect [turbopack::core::diagnostics::Diagnostic] from given source,
/// returns [turbopack::core::diagnostics::PlainDiagnostic]
pub async fn get_diagnostics<T>(source: Vc<T>) -> Result<Vec<ReadRef<PlainDiagnostic>>> {
let captured_diags = source
.peek_diagnostics()
.await?
.strongly_consistent()
.await?;
captured_diags
.diagnostics
.iter()
.map(|d| d.into_plain())
.try_join()
.await
}
#[napi(object)] #[napi(object)]
pub struct NapiIssue { pub struct NapiIssue {
pub severity: String, pub severity: String,
...@@ -178,7 +196,21 @@ impl From<SourcePos> for NapiSourcePos { ...@@ -178,7 +196,21 @@ impl From<SourcePos> for NapiSourcePos {
} }
#[napi(object)] #[napi(object)]
pub struct NapiDiagnostic {} pub struct NapiDiagnostic {
pub category: String,
pub name: String,
pub payload: HashMap<String, String>,
}
impl NapiDiagnostic {
pub fn from(diagnostic: &PlainDiagnostic) -> Self {
Self {
category: diagnostic.category.clone(),
name: diagnostic.name.clone(),
payload: diagnostic.payload.clone(),
}
}
}
pub struct TurbopackResult<T: ToNapiValue> { pub struct TurbopackResult<T: ToNapiValue> {
pub result: T, pub result: T,
......
...@@ -36,6 +36,7 @@ mod next_route_matcher; ...@@ -36,6 +36,7 @@ mod next_route_matcher;
pub mod next_server; pub mod next_server;
pub mod next_server_component; pub mod next_server_component;
pub mod next_shared; pub mod next_shared;
pub mod next_telemetry;
mod page_loader; mod page_loader;
mod page_source; mod page_source;
pub mod pages_structure; pub mod pages_structure;
......
...@@ -46,7 +46,7 @@ use crate::{ ...@@ -46,7 +46,7 @@ use crate::{
get_next_client_resolved_map, mdx_import_source_file, get_next_client_resolved_map, mdx_import_source_file,
}, },
next_shared::{ next_shared::{
resolve::UnsupportedModulesResolvePlugin, resolve::{ModuleFeatureReportResolvePlugin, UnsupportedModulesResolvePlugin},
transforms::{ transforms::{
emotion::get_emotion_transform_plugin, get_relay_transform_plugin, emotion::get_emotion_transform_plugin, get_relay_transform_plugin,
styled_components::get_styled_components_transform_plugin, styled_components::get_styled_components_transform_plugin,
...@@ -145,9 +145,10 @@ pub async fn get_client_resolve_options_context( ...@@ -145,9 +145,10 @@ pub async fn get_client_resolve_options_context(
resolved_map: Some(next_client_resolved_map), resolved_map: Some(next_client_resolved_map),
browser: true, browser: true,
module: true, module: true,
plugins: vec![Vc::upcast(UnsupportedModulesResolvePlugin::new( plugins: vec![
project_path, Vc::upcast(ModuleFeatureReportResolvePlugin::new(project_path)),
))], Vc::upcast(UnsupportedModulesResolvePlugin::new(project_path)),
],
..Default::default() ..Default::default()
}; };
Ok(ResolveOptionsContext { Ok(ResolveOptionsContext {
......
...@@ -19,9 +19,13 @@ use turbopack_binding::{ ...@@ -19,9 +19,13 @@ use turbopack_binding::{
}; };
use crate::{ use crate::{
mode::NextMode, next_client::context::get_client_assets_path, next_config::NextConfig, mode::NextMode,
next_import_map::get_next_edge_import_map, next_server::context::ServerContextType, next_client::context::get_client_assets_path,
next_shared::resolve::UnsupportedModulesResolvePlugin, util::foreign_code_context_condition, next_config::NextConfig,
next_import_map::get_next_edge_import_map,
next_server::context::ServerContextType,
next_shared::resolve::{ModuleFeatureReportResolvePlugin, UnsupportedModulesResolvePlugin},
util::foreign_code_context_condition,
}; };
fn defines() -> CompileTimeDefines { fn defines() -> CompileTimeDefines {
...@@ -93,9 +97,10 @@ pub async fn get_edge_resolve_options_context( ...@@ -93,9 +97,10 @@ pub async fn get_edge_resolve_options_context(
import_map: Some(next_edge_import_map), import_map: Some(next_edge_import_map),
module: true, module: true,
browser: true, browser: true,
plugins: vec![Vc::upcast(UnsupportedModulesResolvePlugin::new( plugins: vec![
project_path, Vc::upcast(ModuleFeatureReportResolvePlugin::new(project_path)),
))], Vc::upcast(UnsupportedModulesResolvePlugin::new(project_path)),
],
..Default::default() ..Default::default()
}; };
......
...@@ -46,7 +46,7 @@ use crate::{ ...@@ -46,7 +46,7 @@ use crate::{
next_import_map::{get_next_server_import_map, mdx_import_source_file}, next_import_map::{get_next_server_import_map, mdx_import_source_file},
next_server::resolve::ExternalPredicate, next_server::resolve::ExternalPredicate,
next_shared::{ next_shared::{
resolve::UnsupportedModulesResolvePlugin, resolve::{ModuleFeatureReportResolvePlugin, UnsupportedModulesResolvePlugin},
transforms::{ transforms::{
emotion::get_emotion_transform_plugin, get_relay_transform_plugin, emotion::get_emotion_transform_plugin, get_relay_transform_plugin,
styled_components::get_styled_components_transform_plugin, styled_components::get_styled_components_transform_plugin,
...@@ -97,6 +97,7 @@ pub async fn get_server_resolve_options_context( ...@@ -97,6 +97,7 @@ pub async fn get_server_resolve_options_context(
get_next_server_import_map(project_path, ty, mode, next_config, execution_context); get_next_server_import_map(project_path, ty, mode, next_config, execution_context);
let foreign_code_context_condition = foreign_code_context_condition(next_config).await?; let foreign_code_context_condition = foreign_code_context_condition(next_config).await?;
let root_dir = project_path.root().resolve().await?; let root_dir = project_path.root().resolve().await?;
let module_feature_report_resolve_plugin = ModuleFeatureReportResolvePlugin::new(project_path);
let unsupported_modules_resolve_plugin = UnsupportedModulesResolvePlugin::new(project_path); let unsupported_modules_resolve_plugin = UnsupportedModulesResolvePlugin::new(project_path);
let server_component_externals_plugin = ExternalCjsModulesResolvePlugin::new( let server_component_externals_plugin = ExternalCjsModulesResolvePlugin::new(
project_path, project_path,
...@@ -118,6 +119,7 @@ pub async fn get_server_resolve_options_context( ...@@ -118,6 +119,7 @@ pub async fn get_server_resolve_options_context(
custom_conditions: vec![mode.node_env().to_string(), "node".to_string()], custom_conditions: vec![mode.node_env().to_string(), "node".to_string()],
import_map: Some(next_server_import_map), import_map: Some(next_server_import_map),
plugins: vec![ plugins: vec![
Vc::upcast(module_feature_report_resolve_plugin),
Vc::upcast(external_cjs_modules_plugin), Vc::upcast(external_cjs_modules_plugin),
Vc::upcast(unsupported_modules_resolve_plugin), Vc::upcast(unsupported_modules_resolve_plugin),
], ],
...@@ -146,6 +148,7 @@ pub async fn get_server_resolve_options_context( ...@@ -146,6 +148,7 @@ pub async fn get_server_resolve_options_context(
], ],
import_map: Some(next_server_import_map), import_map: Some(next_server_import_map),
plugins: vec![ plugins: vec![
Vc::upcast(module_feature_report_resolve_plugin),
Vc::upcast(server_component_externals_plugin), Vc::upcast(server_component_externals_plugin),
Vc::upcast(unsupported_modules_resolve_plugin), Vc::upcast(unsupported_modules_resolve_plugin),
], ],
...@@ -175,6 +178,7 @@ pub async fn get_server_resolve_options_context( ...@@ -175,6 +178,7 @@ pub async fn get_server_resolve_options_context(
], ],
import_map: Some(next_server_import_map), import_map: Some(next_server_import_map),
plugins: vec![ plugins: vec![
Vc::upcast(module_feature_report_resolve_plugin),
Vc::upcast(server_component_externals_plugin), Vc::upcast(server_component_externals_plugin),
Vc::upcast(unsupported_modules_resolve_plugin), Vc::upcast(unsupported_modules_resolve_plugin),
], ],
...@@ -197,6 +201,7 @@ pub async fn get_server_resolve_options_context( ...@@ -197,6 +201,7 @@ pub async fn get_server_resolve_options_context(
custom_conditions: vec![mode.node_env().to_string(), "node".to_string()], custom_conditions: vec![mode.node_env().to_string(), "node".to_string()],
import_map: Some(next_server_import_map), import_map: Some(next_server_import_map),
plugins: vec![ plugins: vec![
Vc::upcast(module_feature_report_resolve_plugin),
Vc::upcast(server_component_externals_plugin), Vc::upcast(server_component_externals_plugin),
Vc::upcast(unsupported_modules_resolve_plugin), Vc::upcast(unsupported_modules_resolve_plugin),
], ],
...@@ -218,7 +223,10 @@ pub async fn get_server_resolve_options_context( ...@@ -218,7 +223,10 @@ pub async fn get_server_resolve_options_context(
enable_node_externals: true, enable_node_externals: true,
module: true, module: true,
custom_conditions: vec![mode.node_env().to_string()], custom_conditions: vec![mode.node_env().to_string()],
plugins: vec![Vc::upcast(unsupported_modules_resolve_plugin)], plugins: vec![
Vc::upcast(module_feature_report_resolve_plugin),
Vc::upcast(unsupported_modules_resolve_plugin),
],
..Default::default() ..Default::default()
}; };
ResolveOptionsContext { ResolveOptionsContext {
......
use std::collections::HashSet; use std::collections::{HashMap, HashSet};
use anyhow::Result; use anyhow::Result;
use lazy_static::lazy_static; use lazy_static::lazy_static;
...@@ -7,6 +7,7 @@ use turbo_tasks_fs::glob::Glob; ...@@ -7,6 +7,7 @@ use turbo_tasks_fs::glob::Glob;
use turbopack_binding::{ use turbopack_binding::{
turbo::tasks_fs::FileSystemPath, turbo::tasks_fs::FileSystemPath,
turbopack::core::{ turbopack::core::{
diagnostics::DiagnosticExt,
issue::{unsupported_module::UnsupportedModuleIssue, IssueExt}, issue::{unsupported_module::UnsupportedModuleIssue, IssueExt},
resolve::{ resolve::{
parse::Request, parse::Request,
...@@ -17,9 +18,27 @@ use turbopack_binding::{ ...@@ -17,9 +18,27 @@ use turbopack_binding::{
}, },
}; };
use crate::next_telemetry::ModuleFeatureTelemetry;
lazy_static! { lazy_static! {
static ref UNSUPPORTED_PACKAGES: HashSet<&'static str> = ["@vercel/og"].into(); static ref UNSUPPORTED_PACKAGES: HashSet<&'static str> = ["@vercel/og"].into();
static ref UNSUPPORTED_PACKAGE_PATHS: HashSet<(&'static str, &'static str)> = [].into(); static ref UNSUPPORTED_PACKAGE_PATHS: HashSet<(&'static str, &'static str)> = [].into();
// Set of the features we want to track, following existing references in webpack/plugins/telemetry-plugin.
static ref FEATURE_MODULES: HashMap<&'static str, Vec<&'static str>> = HashMap::from([
(
"next",
vec![
"/image",
"/future/image",
"/legacy/image",
"/script",
"/dynamic",
"/font/google",
"/font/local"
]
),
("@next", vec!["/font/google", "/font/local"])
]);
} }
#[turbo_tasks::value] #[turbo_tasks::value]
...@@ -82,3 +101,56 @@ impl ResolvePlugin for UnsupportedModulesResolvePlugin { ...@@ -82,3 +101,56 @@ impl ResolvePlugin for UnsupportedModulesResolvePlugin {
Ok(ResolveResultOption::none()) Ok(ResolveResultOption::none())
} }
} }
/// A resolver plugin tracks the usage of certain import paths, emit
/// telemetry events if there is a match.
#[turbo_tasks::value]
pub(crate) struct ModuleFeatureReportResolvePlugin {
root: Vc<FileSystemPath>,
}
#[turbo_tasks::value_impl]
impl ModuleFeatureReportResolvePlugin {
#[turbo_tasks::function]
pub fn new(root: Vc<FileSystemPath>) -> Vc<Self> {
ModuleFeatureReportResolvePlugin { root }.cell()
}
}
#[turbo_tasks::value_impl]
impl ResolvePlugin for ModuleFeatureReportResolvePlugin {
#[turbo_tasks::function]
fn after_resolve_condition(&self) -> Vc<ResolvePluginCondition> {
ResolvePluginCondition::new(self.root.root(), Glob::new("**".to_string()))
}
#[turbo_tasks::function]
async fn after_resolve(
&self,
_fs_path: Vc<FileSystemPath>,
_context: Vc<FileSystemPath>,
request: Vc<Request>,
) -> Result<Vc<ResolveResultOption>> {
if let Request::Module {
module,
path,
query: _,
} = &*request.await?
{
let feature_module = FEATURE_MODULES.get(module.as_str());
if let Some(feature_module) = feature_module {
let sub_path = feature_module
.iter()
.find(|sub_path| path.is_match(sub_path));
if let Some(sub_path) = sub_path {
ModuleFeatureTelemetry::new(format!("{}{}", module, sub_path), 1)
.cell()
.emit();
}
}
}
Ok(ResolveResultOption::none())
}
}
use std::collections::HashMap;
use turbopack_binding::{
turbo::tasks::Vc,
turbopack::core::diagnostics::{Diagnostic, DiagnosticPayload},
};
/// A struct represent telemetry event for the feature usage,
/// referred as `importing` a certain module. (i.e importing @next/image)
#[turbo_tasks::value(shared)]
pub struct ModuleFeatureTelemetry {
pub event_name: String,
pub feature_name: String,
pub invocation_count: usize,
}
impl ModuleFeatureTelemetry {
pub fn new(feature_name: String, invocation_count: usize) -> Self {
ModuleFeatureTelemetry {
event_name: "EVENT_BUILD_FEATURE_USAGE".to_string(),
feature_name,
invocation_count,
}
}
}
#[turbo_tasks::value_impl]
impl Diagnostic for ModuleFeatureTelemetry {
#[turbo_tasks::function]
fn category(&self) -> Vc<String> {
Vc::cell("ModuleFeatureTelemetry_category_tbd".to_string())
}
#[turbo_tasks::function]
fn name(&self) -> Vc<String> {
Vc::cell(self.event_name.clone())
}
#[turbo_tasks::function]
fn payload(&self) -> Vc<DiagnosticPayload> {
Vc::cell(HashMap::from([(
self.feature_name.clone(),
self.invocation_count.to_string(),
)]))
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment