Unverified Commit 2f3cb719 authored by Jimmy Miller's avatar Jimmy Miller Committed by GitHub

[turbopack] Track webpack loader build dependency directories (#98838)

Track explicit directories while continuing to warn for unsupported
module and package inputs.

I will keep removing more and more of the unsupported cases with each PR
parent 5e097685
import { utilFn } from '../../utils/directory-build-dependency'
export default function Page() {
return <p>{utilFn()}</p>
}
{
"name": "build-dependency-package",
"version": "1.0.0",
"main": "one.js"
}
......@@ -5,13 +5,33 @@ const loader = async function (content) {
this.async()
if (this.resourcePath.endsWith('unsupported-build-dependency.ts')) {
this.addBuildDependency(path.join(__dirname, 'utils'))
this.addBuildDependency(path.join(__dirname, 'cyclic-build-dependency'))
this.addBuildDependency(
`${path.join(__dirname, 'utils', 'build-dependency.js')}${path.sep}`
)
return this.callback(
null,
`export const utilFn = () => 'unsupported build dependency';`
)
}
const directoryBuildDependency = this.resourcePath.endsWith(
'directory-build-dependency.ts'
)
if (directoryBuildDependency) {
const packageDirectory = path.join(__dirname, 'build-dependency-package')
const nestedEntry = path.join(packageDirectory, 'nested', 'value.js')
const directoryMarker = path.sep === '/' ? '\\' : '/'
this.addBuildDependency(`${packageDirectory}${directoryMarker}`)
const packageValue = fs
.readFileSync(nestedEntry, 'utf8')
.match(/'([^']+)'/)[1]
return this.callback(
null,
`export const utilFn = () => 'directory build dependency: ${packageValue}';`
)
}
if (!this.resourcePath.endsWith('file-to-transform.ts')) {
return this.callback(null, content)
}
......
......@@ -59,15 +59,39 @@ describe('turbopack-loader-file-dependencies', () => {
// @force-gate turbopack
it('warns for unsupported build dependency inputs', async () => {
const outputIndex = next.cliOutput.length
const $ = await next.render$('/unsupported')
expect($('p').text()).toContain('unsupported build dependency')
await retry(() => {
const output = next.cliOutput.slice(outputIndex)
expect(output).toContain('Unsupported webpack loader build dependency')
expect(output).toContain('/utils')
expect(output).toContain('exact existing file')
expect(output).not.toMatch(/EISDIR|ELOOP/)
})
await next.symlink('cyclic-build-dependency', 'cyclic-build-dependency')
try {
const outputIndex = next.cliOutput.length
const $ = await next.render$('/unsupported')
expect($('p').text()).toContain('unsupported build dependency')
await retry(() => {
const output = next.cliOutput.slice(outputIndex)
expect(output).toContain('Unsupported webpack loader build dependency')
expect(output).toContain('cyclic-build-dependency')
expect(output).toContain('build-dependency.js')
expect(output).not.toMatch(/EISDIR|ELOOP/)
})
} finally {
await next.deleteFile('cyclic-build-dependency')
}
})
// @force-gate turbopack
it('updates when a nested file in a build dependency directory changes', async () => {
const $ = await next.render$('/directory')
expect($('p').text()).toContain('directory build dependency: nested-one')
await next.patchFile(
'build-dependency-package/nested/value.js',
"module.exports = 'nested-two'",
async () => {
await retry(async () => {
const $2 = await next.render$('/directory')
expect($2('p').text()).toContain(
'directory build dependency: nested-two'
)
}, 10000)
}
)
})
})
use anyhow::{Result, bail};
use auto_hash_map::AutoMap;
use futures::try_join;
use rustc_hash::FxHashMap;
use turbo_rcstr::RcStr;
......@@ -212,7 +213,7 @@ async fn track_glob_internal(
glob: Vc<Glob>,
include_dot_files: bool,
) -> Result<Vc<Completion>> {
let dir = directory.read_dir().await?;
let dir = read_dir_for_track_glob(directory.clone(), include_dot_files).await?;
let glob_value = glob.await?;
let fs = directory.fs().to_resolved().await?;
let mut reads = Vec::new();
......@@ -221,9 +222,6 @@ async fn track_glob_internal(
match &*dir {
DirectoryContent::Entries(entries) => {
for (segment, entry) in entries.iter() {
if !include_dot_files && segment.starts_with('.') {
continue;
}
// This is redundant with logic inside of `read_dir` but here we track it separately
// so we don't follow symlinks.
let entry_path = if prefix.is_empty() {
......@@ -276,6 +274,29 @@ async fn track_glob_internal(
Ok(Completion::new())
}
// This task shields callers that want to ignore dot files from being invalidated when they change
#[turbo_tasks::function(fs)]
async fn read_dir_for_track_glob(
directory: FileSystemPath,
include_dot_files: bool,
) -> Result<Vc<DirectoryContent>> {
let read_dir = directory.read_dir();
if include_dot_files {
return Ok(read_dir);
}
Ok(match &*read_dir.await? {
DirectoryContent::Entries(entries) => DirectoryContent::new(
entries
.iter()
.filter(|(segment, _)| !segment.starts_with('.'))
.map(|(segment, entry)| (segment.clone(), entry.clone()))
.collect::<AutoMap<_, _>>(),
),
DirectoryContent::NotFound => read_dir,
})
}
#[cfg(test)]
pub mod tests {
......@@ -790,6 +811,27 @@ pub mod tests {
.await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Creating or deleting a dotfile should not invalidate the filtered glob.
read_strongly_consistent_and_apply_effects(
extract_effects_operation(write(root.join("dir/sub/.ignored")?, rcstr!("ignored"))),
|e| e,
)
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
.read_strongly_consistent()
.await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
read_strongly_consistent_and_apply_effects(
extract_effects_operation(delete(root.join("dir/sub/.ignored")?)),
|e| e,
)
.await?;
let read_dir2 = track_star_star_glob(dir.clone())
.read_strongly_consistent()
.await?;
assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
// Delete a file that we should be tracking
read_strongly_consistent_and_apply_effects(
extract_effects_operation(delete(root.join("dir/foo")?)),
......
......@@ -13,6 +13,7 @@ export type IpcInfoMessage =
directories?: Array<[string, string]>
filePaths?: string[]
buildFilePaths?: string[]
buildDependencyRequests?: Array<[string, boolean]>
}
| {
type: 'emittedError'
......
......@@ -24,6 +24,7 @@ export type IpcInfoMessage =
directories?: Array<[string, string]>
filePaths?: string[]
buildFilePaths?: string[]
buildDependencyRequests?: Array<[string, boolean]>
}
| {
type: 'emittedError'
......@@ -199,7 +200,7 @@ const transform = (
: {}
},
addBuildDependency(dependency: string) {
buildDependencies.add(pathResolve(contextDir, dependency))
buildDependencies.add(dependency)
},
fs: {
readFile(p: string, optionsOrCb: any, maybeCb: any) {
......@@ -558,7 +559,26 @@ const transform = (
...result.missingDependencies,
],
directories: result.contextDependencies.map((dep) => [dep, '**']),
buildFilePaths: [...buildDependencies].sort(),
buildDependencyRequests: [...buildDependencies]
.map((dependency) => {
// Webpack uses a trailing slash or backslash to identify directory build dependencies:
// https://github.com/webpack/webpack/blob/v5.98.0/lib/FileSystemInfo.js#L1741-L1748
const isDirectory = /[\\/]$/.test(dependency)
let request = isDirectory ? dependency.slice(0, -1) : dependency
if (path.isAbsolute(request)) {
request = path.relative(contextDir, request)
if (
!path.isAbsolute(request) &&
request.split(path.sep)[0] !== '..'
) {
request = `./${request}`
}
}
request =
path.sep === '/' ? request : request.replaceAll(path.sep, '/')
return [request, isDirectory] as [string, boolean]
})
.sort(([a], [b]) => a.localeCompare(b)),
})
if (err) {
// Resolve loader paths to include in the error message using
......
......@@ -250,16 +250,21 @@ async fn build_files_changed(
) -> Result<Vc<Completion>> {
for path in paths {
let entry_type = path.get_type().await?;
match &*entry_type {
FileSystemEntryType::File | FileSystemEntryType::NotFound => {
let supported = match &*entry_type {
FileSystemEntryType::File => {
path.read().await?;
true
}
FileSystemEntryType::NotFound => {
path.read().await?;
false
}
FileSystemEntryType::Directory
| FileSystemEntryType::Symlink
| FileSystemEntryType::Other
| FileSystemEntryType::Error => {}
}
if !matches!(&*entry_type, FileSystemEntryType::File) {
| FileSystemEntryType::Error => false,
};
if !supported {
BuildDependencyIssue {
source: IssueSource::from_source_only(source),
path,
......@@ -313,6 +318,47 @@ async fn loaders_changed(
Ok(Vc::<Completions>::cell(completions).completed())
}
#[turbo_tasks::function]
async fn build_dependency_requests_changed(
cwd: FileSystemPath,
requests: Vec<(RcStr, bool)>,
source: ResolvedVc<Box<dyn Source>>,
) -> Result<Vc<Completion>> {
for (request, is_directory) in requests {
let path = cwd.join(&request)?;
let entry_type = path.get_type().await?;
let supported = match &*entry_type {
FileSystemEntryType::File if !is_directory => {
path.read().await?;
true
}
FileSystemEntryType::Directory if is_directory => {
path.track_glob(Glob::new(rcstr!("**"), GlobOptions::default()), false)
.await?;
true
}
FileSystemEntryType::NotFound => {
path.read().await?;
false
}
FileSystemEntryType::File
| FileSystemEntryType::Directory
| FileSystemEntryType::Symlink
| FileSystemEntryType::Other
| FileSystemEntryType::Error => false,
};
if !supported {
BuildDependencyIssue {
source: IssueSource::from_source_only(source),
path,
}
.resolved_cell()
.emit();
}
}
Ok(Completion::new())
}
#[turbo_tasks::value_impl]
impl WebpackLoadersProcessedAsset {
#[turbo_tasks::function]
......@@ -533,6 +579,8 @@ pub enum InfoMessage {
directories: Vec<(RcStr, RcStr)>,
#[serde(default)]
build_file_paths: Vec<RcStr>,
#[serde(default)]
build_dependency_requests: Vec<(RcStr, bool)>,
},
EmittedError {
severity: IssueSeverity,
......@@ -710,6 +758,7 @@ impl EvaluateContext for WebpackLoaderContext {
file_paths,
directories,
build_file_paths,
build_dependency_requests,
} => {
// We only process these dependencies to help with tracking, so if it is disabled
// dont bother.
......@@ -741,6 +790,15 @@ impl EvaluateContext for WebpackLoaderContext {
.await?;
Ok::<_, anyhow::Error>(())
};
let build_dependency_request_subscriptions = async {
build_dependency_requests_changed(
self.cwd.clone(),
build_dependency_requests,
*self.context_source_for_issue,
)
.await?;
Ok::<_, anyhow::Error>(())
};
let directory_subscriptions = directories
.iter()
.map(async |(dir, glob)| {
......@@ -754,6 +812,7 @@ impl EvaluateContext for WebpackLoaderContext {
env_subscriptions,
file_subscriptions,
build_file_subscriptions,
build_dependency_request_subscriptions,
directory_subscriptions
)?;
}
......@@ -1108,9 +1167,8 @@ impl Issue for BuildDependencyIssue {
StyledString::Text(rcstr!("The path at ")),
StyledString::Code(self.path.to_string().into()),
StyledString::Text(
" was passed to this.addBuildDependency, but it is not an exact existing file. \
Only exact existing file paths are supported; other inputs may require \
restarting the development server."
" is not an exact existing file or explicit directory build dependency. \
Unsupported inputs may require restarting the development server."
.into(),
),
])))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment