Unverified Commit 2f3cb719 authored by Jimmy Miller's avatar Jimmy Miller Committed by GitHub

[turbopack] Track webpack loader build dependency directories (#98838)

Track explicit directories while continuing to warn for unsupported
module and package inputs.

I will keep removing more and more of the unsupported cases with each PR
parent 5e097685
import { utilFn } from '../../utils/directory-build-dependency'
export default function Page() {
return <p>{utilFn()}</p>
}
{
"name": "build-dependency-package",
"version": "1.0.0",
"main": "one.js"
}
...@@ -5,13 +5,33 @@ const loader = async function (content) { ...@@ -5,13 +5,33 @@ const loader = async function (content) {
this.async() this.async()
if (this.resourcePath.endsWith('unsupported-build-dependency.ts')) { if (this.resourcePath.endsWith('unsupported-build-dependency.ts')) {
this.addBuildDependency(path.join(__dirname, 'utils')) this.addBuildDependency(path.join(__dirname, 'cyclic-build-dependency'))
this.addBuildDependency(
`${path.join(__dirname, 'utils', 'build-dependency.js')}${path.sep}`
)
return this.callback( return this.callback(
null, null,
`export const utilFn = () => 'unsupported build dependency';` `export const utilFn = () => 'unsupported build dependency';`
) )
} }
const directoryBuildDependency = this.resourcePath.endsWith(
'directory-build-dependency.ts'
)
if (directoryBuildDependency) {
const packageDirectory = path.join(__dirname, 'build-dependency-package')
const nestedEntry = path.join(packageDirectory, 'nested', 'value.js')
const directoryMarker = path.sep === '/' ? '\\' : '/'
this.addBuildDependency(`${packageDirectory}${directoryMarker}`)
const packageValue = fs
.readFileSync(nestedEntry, 'utf8')
.match(/'([^']+)'/)[1]
return this.callback(
null,
`export const utilFn = () => 'directory build dependency: ${packageValue}';`
)
}
if (!this.resourcePath.endsWith('file-to-transform.ts')) { if (!this.resourcePath.endsWith('file-to-transform.ts')) {
return this.callback(null, content) return this.callback(null, content)
} }
......
...@@ -59,15 +59,39 @@ describe('turbopack-loader-file-dependencies', () => { ...@@ -59,15 +59,39 @@ describe('turbopack-loader-file-dependencies', () => {
// @force-gate turbopack // @force-gate turbopack
it('warns for unsupported build dependency inputs', async () => { it('warns for unsupported build dependency inputs', async () => {
const outputIndex = next.cliOutput.length await next.symlink('cyclic-build-dependency', 'cyclic-build-dependency')
const $ = await next.render$('/unsupported') try {
expect($('p').text()).toContain('unsupported build dependency') const outputIndex = next.cliOutput.length
await retry(() => { const $ = await next.render$('/unsupported')
const output = next.cliOutput.slice(outputIndex) expect($('p').text()).toContain('unsupported build dependency')
expect(output).toContain('Unsupported webpack loader build dependency') await retry(() => {
expect(output).toContain('/utils') const output = next.cliOutput.slice(outputIndex)
expect(output).toContain('exact existing file') expect(output).toContain('Unsupported webpack loader build dependency')
expect(output).not.toMatch(/EISDIR|ELOOP/) expect(output).toContain('cyclic-build-dependency')
}) expect(output).toContain('build-dependency.js')
expect(output).not.toMatch(/EISDIR|ELOOP/)
})
} finally {
await next.deleteFile('cyclic-build-dependency')
}
})
// @force-gate turbopack
it('updates when a nested file in a build dependency directory changes', async () => {
const $ = await next.render$('/directory')
expect($('p').text()).toContain('directory build dependency: nested-one')
await next.patchFile(
'build-dependency-package/nested/value.js',
"module.exports = 'nested-two'",
async () => {
await retry(async () => {
const $2 = await next.render$('/directory')
expect($2('p').text()).toContain(
'directory build dependency: nested-two'
)
}, 10000)
}
)
}) })
}) })
use anyhow::{Result, bail}; use anyhow::{Result, bail};
use auto_hash_map::AutoMap;
use futures::try_join; use futures::try_join;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr; use turbo_rcstr::RcStr;
...@@ -212,7 +213,7 @@ async fn track_glob_internal( ...@@ -212,7 +213,7 @@ async fn track_glob_internal(
glob: Vc<Glob>, glob: Vc<Glob>,
include_dot_files: bool, include_dot_files: bool,
) -> Result<Vc<Completion>> { ) -> Result<Vc<Completion>> {
let dir = directory.read_dir().await?; let dir = read_dir_for_track_glob(directory.clone(), include_dot_files).await?;
let glob_value = glob.await?; let glob_value = glob.await?;
let fs = directory.fs().to_resolved().await?; let fs = directory.fs().to_resolved().await?;
let mut reads = Vec::new(); let mut reads = Vec::new();
...@@ -221,9 +222,6 @@ async fn track_glob_internal( ...@@ -221,9 +222,6 @@ async fn track_glob_internal(
match &*dir { match &*dir {
DirectoryContent::Entries(entries) => { DirectoryContent::Entries(entries) => {
for (segment, entry) in entries.iter() { for (segment, entry) in entries.iter() {
if !include_dot_files && segment.starts_with('.') {
continue;
}
// This is redundant with logic inside of `read_dir` but here we track it separately // This is redundant with logic inside of `read_dir` but here we track it separately
// so we don't follow symlinks. // so we don't follow symlinks.
let entry_path = if prefix.is_empty() { let entry_path = if prefix.is_empty() {
...@@ -276,6 +274,29 @@ async fn track_glob_internal( ...@@ -276,6 +274,29 @@ async fn track_glob_internal(
Ok(Completion::new()) Ok(Completion::new())
} }
// This task shields callers that want to ignore dot files from being invalidated when they change
#[turbo_tasks::function(fs)]
async fn read_dir_for_track_glob(
directory: FileSystemPath,
include_dot_files: bool,
) -> Result<Vc<DirectoryContent>> {
let read_dir = directory.read_dir();
if include_dot_files {
return Ok(read_dir);
}
Ok(match &*read_dir.await? {
DirectoryContent::Entries(entries) => DirectoryContent::new(
entries
.iter()
.filter(|(segment, _)| !segment.starts_with('.'))
.map(|(segment, entry)| (segment.clone(), entry.clone()))
.collect::<AutoMap<_, _>>(),
),
DirectoryContent::NotFound => read_dir,
})
}
#[cfg(test)] #[cfg(test)]
pub mod tests { pub mod tests {
...@@ -790,6 +811,27 @@ pub mod tests { ...@@ -790,6 +811,27 @@ pub mod tests {
.await?; .await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2)); assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Creating or deleting a dotfile should not invalidate the filtered glob.
read_strongly_consistent_and_apply_effects(
extract_effects_operation(write(root.join("dir/sub/.ignored")?, rcstr!("ignored"))),
|e| e,
)
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
.read_strongly_consistent()
.await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
read_strongly_consistent_and_apply_effects(
extract_effects_operation(delete(root.join("dir/sub/.ignored")?)),
|e| e,
)
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
.read_strongly_consistent()
.await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Delete a file that we should be tracking // Delete a file that we should be tracking
read_strongly_consistent_and_apply_effects( read_strongly_consistent_and_apply_effects(
extract_effects_operation(delete(root.join("dir/foo")?)), extract_effects_operation(delete(root.join("dir/foo")?)),
......
...@@ -13,6 +13,7 @@ export type IpcInfoMessage = ...@@ -13,6 +13,7 @@ export type IpcInfoMessage =
directories?: Array<[string, string]> directories?: Array<[string, string]>
filePaths?: string[] filePaths?: string[]
buildFilePaths?: string[] buildFilePaths?: string[]
buildDependencyRequests?: Array<[string, boolean]>
} }
| { | {
type: 'emittedError' type: 'emittedError'
......
...@@ -24,6 +24,7 @@ export type IpcInfoMessage = ...@@ -24,6 +24,7 @@ export type IpcInfoMessage =
directories?: Array<[string, string]> directories?: Array<[string, string]>
filePaths?: string[] filePaths?: string[]
buildFilePaths?: string[] buildFilePaths?: string[]
buildDependencyRequests?: Array<[string, boolean]>
} }
| { | {
type: 'emittedError' type: 'emittedError'
...@@ -199,7 +200,7 @@ const transform = ( ...@@ -199,7 +200,7 @@ const transform = (
: {} : {}
}, },
addBuildDependency(dependency: string) { addBuildDependency(dependency: string) {
buildDependencies.add(pathResolve(contextDir, dependency)) buildDependencies.add(dependency)
}, },
fs: { fs: {
readFile(p: string, optionsOrCb: any, maybeCb: any) { readFile(p: string, optionsOrCb: any, maybeCb: any) {
...@@ -558,7 +559,26 @@ const transform = ( ...@@ -558,7 +559,26 @@ const transform = (
...result.missingDependencies, ...result.missingDependencies,
], ],
directories: result.contextDependencies.map((dep) => [dep, '**']), directories: result.contextDependencies.map((dep) => [dep, '**']),
buildFilePaths: [...buildDependencies].sort(), buildDependencyRequests: [...buildDependencies]
.map((dependency) => {
// Webpack uses a trailing slash or backslash to identify directory build dependencies:
// https://github.com/webpack/webpack/blob/v5.98.0/lib/FileSystemInfo.js#L1741-L1748
const isDirectory = /[\\/]$/.test(dependency)
let request = isDirectory ? dependency.slice(0, -1) : dependency
if (path.isAbsolute(request)) {
request = path.relative(contextDir, request)
if (
!path.isAbsolute(request) &&
request.split(path.sep)[0] !== '..'
) {
request = `./${request}`
}
}
request =
path.sep === '/' ? request : request.replaceAll(path.sep, '/')
return [request, isDirectory] as [string, boolean]
})
.sort(([a], [b]) => a.localeCompare(b)),
}) })
if (err) { if (err) {
// Resolve loader paths to include in the error message using // Resolve loader paths to include in the error message using
......
...@@ -250,16 +250,21 @@ async fn build_files_changed( ...@@ -250,16 +250,21 @@ async fn build_files_changed(
) -> Result<Vc<Completion>> { ) -> Result<Vc<Completion>> {
for path in paths { for path in paths {
let entry_type = path.get_type().await?; let entry_type = path.get_type().await?;
match &*entry_type { let supported = match &*entry_type {
FileSystemEntryType::File | FileSystemEntryType::NotFound => { FileSystemEntryType::File => {
path.read().await?; path.read().await?;
true
}
FileSystemEntryType::NotFound => {
path.read().await?;
false
} }
FileSystemEntryType::Directory FileSystemEntryType::Directory
| FileSystemEntryType::Symlink | FileSystemEntryType::Symlink
| FileSystemEntryType::Other | FileSystemEntryType::Other
| FileSystemEntryType::Error => {} | FileSystemEntryType::Error => false,
} };
if !matches!(&*entry_type, FileSystemEntryType::File) { if !supported {
BuildDependencyIssue { BuildDependencyIssue {
source: IssueSource::from_source_only(source), source: IssueSource::from_source_only(source),
path, path,
...@@ -313,6 +318,47 @@ async fn loaders_changed( ...@@ -313,6 +318,47 @@ async fn loaders_changed(
Ok(Vc::<Completions>::cell(completions).completed()) Ok(Vc::<Completions>::cell(completions).completed())
} }
#[turbo_tasks::function]
async fn build_dependency_requests_changed(
cwd: FileSystemPath,
requests: Vec<(RcStr, bool)>,
source: ResolvedVc<Box<dyn Source>>,
) -> Result<Vc<Completion>> {
for (request, is_directory) in requests {
let path = cwd.join(&request)?;
let entry_type = path.get_type().await?;
let supported = match &*entry_type {
FileSystemEntryType::File if !is_directory => {
path.read().await?;
true
}
FileSystemEntryType::Directory if is_directory => {
path.track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
.await?;
true
}
FileSystemEntryType::NotFound => {
path.read().await?;
false
}
FileSystemEntryType::File
| FileSystemEntryType::Directory
| FileSystemEntryType::Symlink
| FileSystemEntryType::Other
| FileSystemEntryType::Error => false,
};
if !supported {
BuildDependencyIssue {
source: IssueSource::from_source_only(source),
path,
}
.resolved_cell()
.emit();
}
}
Ok(Completion::new())
}
#[turbo_tasks::value_impl] #[turbo_tasks::value_impl]
impl WebpackLoadersProcessedAsset { impl WebpackLoadersProcessedAsset {
#[turbo_tasks::function] #[turbo_tasks::function]
...@@ -533,6 +579,8 @@ pub enum InfoMessage { ...@@ -533,6 +579,8 @@ pub enum InfoMessage {
directories: Vec<(RcStr, RcStr)>, directories: Vec<(RcStr, RcStr)>,
#[serde(default)] #[serde(default)]
build_file_paths: Vec<RcStr>, build_file_paths: Vec<RcStr>,
#[serde(default)]
build_dependency_requests: Vec<(RcStr, bool)>,
}, },
EmittedError { EmittedError {
severity: IssueSeverity, severity: IssueSeverity,
...@@ -710,6 +758,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -710,6 +758,7 @@ impl EvaluateContext for WebpackLoaderContext {
file_paths, file_paths,
directories, directories,
build_file_paths, build_file_paths,
build_dependency_requests,
} => { } => {
// We only process these dependencies to help with tracking, so if it is disabled // We only process these dependencies to help with tracking, so if it is disabled
// dont bother. // dont bother.
...@@ -741,6 +790,15 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -741,6 +790,15 @@ impl EvaluateContext for WebpackLoaderContext {
.await?; .await?;
Ok::<_, anyhow::Error>(()) Ok::<_, anyhow::Error>(())
}; };
let build_dependency_request_subscriptions = async {
build_dependency_requests_changed(
self.cwd.clone(),
build_dependency_requests,
*self.context_source_for_issue,
)
.await?;
Ok::<_, anyhow::Error>(())
};
let directory_subscriptions = directories let directory_subscriptions = directories
.iter() .iter()
.map(async |(dir, glob)| { .map(async |(dir, glob)| {
...@@ -754,6 +812,7 @@ impl EvaluateContext for WebpackLoaderContext { ...@@ -754,6 +812,7 @@ impl EvaluateContext for WebpackLoaderContext {
env_subscriptions, env_subscriptions,
file_subscriptions, file_subscriptions,
build_file_subscriptions, build_file_subscriptions,
build_dependency_request_subscriptions,
directory_subscriptions directory_subscriptions
)?; )?;
} }
...@@ -1108,9 +1167,8 @@ impl Issue for BuildDependencyIssue { ...@@ -1108,9 +1167,8 @@ impl Issue for BuildDependencyIssue {
StyledString::Text(rcstr!("The path at ")), StyledString::Text(rcstr!("The path at ")),
StyledString::Code(self.path.to_string().into()), StyledString::Code(self.path.to_string().into()),
StyledString::Text( StyledString::Text(
" was passed to this.addBuildDependency, but it is not an exact existing file. \ " is not an exact existing file or explicit directory build dependency. \
Only exact existing file paths are supported; other inputs may require \ Unsupported inputs may require restarting the development server."
restarting the development server."
.into(), .into(),
), ),
]))) ])))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment