Unverified Commit 45928cee authored by Mark Knichel's avatar Mark Knichel Committed by GitHub

Fix a bug in the webpack-sources patch that happens sometimes when using the webpack cache (#67928)

parent c175c24d
...@@ -188,7 +188,7 @@ export async function webpackBuildImpl( ...@@ -188,7 +188,7 @@ export async function webpackBuildImpl(
await runWebpackSpan.traceAsyncFn(async () => { await runWebpackSpan.traceAsyncFn(async () => {
if (config.experimental.webpackMemoryOptimizations) { if (config.experimental.webpackMemoryOptimizations) {
stringBufferUtils.disableDualStringBufferCaching() stringBufferUtils.disableDualStringBufferCaching()
stringBufferUtils.enableStringInterning() stringBufferUtils.enterStringInterningRange()
} }
// Run the server compilers first and then the client // Run the server compilers first and then the client
...@@ -261,7 +261,7 @@ export async function webpackBuildImpl( ...@@ -261,7 +261,7 @@ export async function webpackBuildImpl(
} }
if (config.experimental.webpackMemoryOptimizations) { if (config.experimental.webpackMemoryOptimizations) {
stringBufferUtils.disableStringInterning() stringBufferUtils.exitStringInterningRange()
} }
inputFileSystem?.purge?.() inputFileSystem?.purge?.()
......
...@@ -453,9 +453,9 @@ declare module 'next/dist/compiled/loader-utils3' ...@@ -453,9 +453,9 @@ declare module 'next/dist/compiled/loader-utils3'
declare module 'next/dist/compiled/webpack-sources3' { declare module 'next/dist/compiled/webpack-sources3' {
interface StringBufferUtils { interface StringBufferUtils {
disableDualStringBufferCaching: () => boolean disableDualStringBufferCaching: () => boolean
disableStringInterning: () => boolean
enableDualStringBufferCaching: () => boolean enableDualStringBufferCaching: () => boolean
enableStringInterning: () => boolean enterStringInterningRange: () => boolean
exitStringInterningRange: () => boolean
} }
export let stringBufferUtils: StringBufferUtils export let stringBufferUtils: StringBufferUtils
} }
......
diff --git a/lib/RawSource.js b/lib/RawSource.js diff --git a/lib/RawSource.js b/lib/RawSource.js
index 098d317..06e6b8f 100644 index 71f1341c43dd45dc82b633eabe0a1cfe99bbe25d..b56ea1a17ba81295826dc1c820d462cca8643bc1 100644
--- a/lib/RawSource.js --- a/lib/RawSource.js
+++ b/lib/RawSource.js +++ b/lib/RawSource.js
@@ -6,6 +6,10 @@ @@ -6,76 +6,91 @@
"use strict"; "use strict";
const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource"); const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource");
+const { +const {
+ internString, + internString,
+ isDualStringBufferCachingEnabled + isDualStringBufferCachingEnabled,
+} = require("./helpers/stringBufferUtils"); +} = require("./helpers/stringBufferUtils");
const Source = require("./Source"); const Source = require("./Source");
class RawSource extends Source { class RawSource extends Source {
@@ -17,8 +21,13 @@ class RawSource extends Source { - constructor(value, convertToString = false) {
} - super();
this._valueIsBuffer = !convertToString && isBuffer; - const isBuffer = Buffer.isBuffer(value);
- if (!isBuffer && typeof value !== "string") {
- throw new TypeError("argument 'value' must be either string of Buffer");
- }
- this._valueIsBuffer = !convertToString && isBuffer;
- this._value = convertToString && isBuffer ? undefined : value; - this._value = convertToString && isBuffer ? undefined : value;
- this._valueAsBuffer = isBuffer ? value : undefined;
- this._valueAsString = isBuffer ? undefined : value;
- }
+ constructor(value, convertToString = false) {
+ super();
+ const isBuffer = Buffer.isBuffer(value);
+ if (!isBuffer && typeof value !== "string") {
+ throw new TypeError("argument 'value' must be either string or Buffer");
+ }
+ this._valueIsBuffer = !convertToString && isBuffer;
+ const internedString =
+ typeof value === "string" ? internString(value) : undefined;
+ this._value = + this._value =
+ convertToString && isBuffer + convertToString && isBuffer
+ ? undefined + ? undefined
+ : typeof value === "string" + : typeof value === "string"
+ ? internString(value) + ? internedString
+ : value; + : value;
this._valueAsBuffer = isBuffer ? value : undefined; + this._valueAsBuffer = isBuffer ? value : undefined;
- this._valueAsString = isBuffer ? undefined : value; + this._valueAsString = isBuffer ? undefined : internedString;
+ this._valueAsString = isBuffer ? undefined : internString(value); + }
}
isBuffer() { - isBuffer() {
@@ -27,14 +36,22 @@ class RawSource extends Source { - return this._valueIsBuffer;
- }
+ isBuffer() {
+ return this._valueIsBuffer;
+ }
source() { - source() {
if (this._value === undefined) { - if (this._value === undefined) {
- this._value = this._valueAsBuffer.toString("utf-8"); - this._value = this._valueAsBuffer.toString("utf-8");
+ const value = internString(this._valueAsBuffer.toString("utf-8")); - }
- return this._value;
- }
+ source() {
+ if (this._value === undefined) {
+ const value = this._valueAsBuffer.toString("utf-8");
+ if (isDualStringBufferCachingEnabled()) { + if (isDualStringBufferCachingEnabled()) {
+ this._value = value; + this._value = internString(value);
+ } + }
+ return value; + return value;
} + }
return this._value; + return this._value;
} + }
buffer() { - buffer() {
if (this._valueAsBuffer === undefined) { - if (this._valueAsBuffer === undefined) {
- this._valueAsBuffer = Buffer.from(this._value, "utf-8"); - this._valueAsBuffer = Buffer.from(this._value, "utf-8");
- }
- return this._valueAsBuffer;
- }
+ buffer() {
+ if (this._valueAsBuffer === undefined) {
+ const value = Buffer.from(this._value, "utf-8"); + const value = Buffer.from(this._value, "utf-8");
+ if (isDualStringBufferCachingEnabled()) { + if (isDualStringBufferCachingEnabled()) {
+ this._valueAsBuffer = value; + this._valueAsBuffer = value;
+ } + }
+ return value; + return value;
} + }
return this._valueAsBuffer; + return this._valueAsBuffer;
} + }
@@ -51,17 +68,21 @@ class RawSource extends Source {
* @returns {void} - map(options) {
*/ - return null;
streamChunks(options, onChunk, onSource, onName) { - }
+ map(options) {
+ return null;
+ }
- /**
- * @param {object} options options
- * @param {function(string, number, number, number, number, number, number): void} onChunk called for each chunk of code
- * @param {function(number, string, string)} onSource called for each source
- * @param {function(number, string)} onName called for each name
- * @returns {void}
- */
- streamChunks(options, onChunk, onSource, onName) {
- if (this._value === undefined) { - if (this._value === undefined) {
+ if (this._value === undefined && isDualStringBufferCachingEnabled()) { - this._value = Buffer.from(this._valueAsBuffer, "utf-8");
this._value = Buffer.from(this._valueAsBuffer, "utf-8"); - }
}
- if (this._valueAsString === undefined) { - if (this._valueAsString === undefined) {
- this._valueAsString = - this._valueAsString =
- typeof this._value === "string"
- ? this._value
- : this._value.toString("utf-8");
- }
- return streamChunksOfRawSource(
- this._valueAsString,
- onChunk,
- onSource,
- onName,
- !!(options && options.finalSource)
- );
- }
+ /**
+ * @param {object} options options
+ * @param {function(string, number, number, number, number, number, number): void} onChunk called for each chunk of code
+ * @param {function(number, string, string)} onSource called for each source
+ * @param {function(number, string)} onName called for each name
+ * @returns {void}
+ */
+ streamChunks(options, onChunk, onSource, onName) {
+ let strValue = this._valueAsString; + let strValue = this._valueAsString;
+ if (strValue === undefined) { + if (strValue === undefined) {
+ strValue = + const value = this.source();
typeof this._value === "string" + strValue = typeof value === "string" ? value : value.toString("utf-8");
? this._value
- : this._value.toString("utf-8");
+ : internString(this._value.toString("utf-8"));
+ if (isDualStringBufferCachingEnabled()) { + if (isDualStringBufferCachingEnabled()) {
+ this._valueAsString = strValue; + this._valueAsString = internString(strValue);
+ } + }
} + }
return streamChunksOfRawSource( + return streamChunksOfRawSource(
- this._valueAsString,
+ strValue, + strValue,
onChunk, + onChunk,
onSource, + onSource,
onName, + onName,
@@ -70,11 +91,8 @@ class RawSource extends Source { + !!(options && options.finalSource)
} + );
+ }
updateHash(hash) { - updateHash(hash) {
- if (this._valueAsBuffer === undefined) { - if (this._valueAsBuffer === undefined) {
- this._valueAsBuffer = Buffer.from(this._value, "utf-8"); - this._valueAsBuffer = Buffer.from(this._value, "utf-8");
- } - }
hash.update("RawSource"); - hash.update("RawSource");
- hash.update(this._valueAsBuffer); - hash.update(this._valueAsBuffer);
- }
+ updateHash(hash) {
+ hash.update("RawSource");
+ hash.update(this.buffer()); + hash.update(this.buffer());
} + }
} }
module.exports = RawSource;
diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js
new file mode 100644 new file mode 100644
index 0000000..3b210f1 index 0000000000000000000000000000000000000000..3d8b85e61643412796e12e2aac6905c5e3a8d5b9
--- /dev/null --- /dev/null
+++ b/lib/helpers/stringBufferUtils.js +++ b/lib/helpers/stringBufferUtils.js
@@ -0,0 +1,107 @@ @@ -0,0 +1,120 @@
+/* +/*
+ MIT License http://www.opensource.org/licenses/mit-license.php + MIT License http://www.opensource.org/licenses/mit-license.php
+ Author Mark Knichel @mknichel + Author Mark Knichel @mknichel
...@@ -156,7 +217,12 @@ index 0000000..3b210f1 ...@@ -156,7 +217,12 @@ index 0000000..3b210f1
+ * of the same value if it has already been interned. + * of the same value if it has already been interned.
+ */ + */
+function internString(str) { +function internString(str) {
+ if (!isStringInterningEnabled() || !str || typeof str !== "string") { + if (
+ !isStringInterningEnabled() ||
+ !str ||
+ str.length < 128 ||
+ typeof str !== "string"
+ ) {
+ return str; + return str;
+ } + }
+ let internedString = interningStringMap.get(str); + let internedString = interningStringMap.get(str);
...@@ -174,25 +240,33 @@ index 0000000..3b210f1 ...@@ -174,25 +240,33 @@ index 0000000..3b210f1
+} +}
+ +
+/** +/**
+ * Enables a memory optimization to avoid repeat copies of the same string in + * Starts a memory optimization to avoid repeat copies of the same string in
+ * memory by caching a single reference to the string. This can reduce memory + * memory by caching a single reference to the string. This can reduce memory
+ * usage if the same string is repeated many times in the compiler, such as + * usage if the same string is repeated many times in the compiler, such as
+ * when Webpack layers are used with the same files. + * when Webpack layers are used with the same files.
+ * + *
+ * {@link exitStringInterningRange} should be called when string interning is
+ * no longer necessary to free up the memory used by the interned strings. If
+ * {@link enterStringInterningRange} has been called multiple times, then
+ * this method may not immediately free all the memory until
+ * {@link exitStringInterningRange} has been called to end all string
+ * interning ranges.
+ *
+ * @returns {void} + * @returns {void}
+ */ + */
+function enableStringInterning() { +function enterStringInterningRange() {
+ enableStringInterningRefCount++; + enableStringInterningRefCount++;
+} +}
+ +
+/** +/**
+ * Disables string interning. This should be called to free the memory used by + * Stops the current string interning range. Once all string interning ranges
+ * the interned strings after the compiler no longer needs to reuse the + * have been exited, this method will free all the memory used by the interned
+ * interned strings such as at the end of the compilation. + * strings. This method should be called once for each time that
+ * {@link enterStringInterningRange} was called.
+ * + *
+ * @returns {void} + * @returns {void}
+ */ + */
+function disableStringInterning() { +function exitStringInterningRange() {
+ if (--enableStringInterningRefCount <= 0) { + if (--enableStringInterningRefCount <= 0) {
+ interningStringMap.clear(); + interningStringMap.clear();
+ enableStringInterningRefCount = 0; + enableStringInterningRefCount = 0;
...@@ -201,16 +275,45 @@ index 0000000..3b210f1 ...@@ -201,16 +275,45 @@ index 0000000..3b210f1
+ +
+module.exports = { +module.exports = {
+ disableDualStringBufferCaching, + disableDualStringBufferCaching,
+ disableStringInterning,
+ enableDualStringBufferCaching, + enableDualStringBufferCaching,
+ enableStringInterning,
+ internString, + internString,
+ isDualStringBufferCachingEnabled + isDualStringBufferCachingEnabled,
+ enterStringInterningRange,
+ exitStringInterningRange,
+}; +};
diff --git a/lib/index.js b/lib/index.js diff --git a/lib/index.js b/lib/index.js
index 0c11c2f..86a7234 100644 index 0c11c2f4cf5387e249cf12fa0fcb746dfcfec539..9ca1b100d8250d4f3c41e5931f710efbbca21611 100644
--- a/lib/index.js --- a/lib/index.js
+++ b/lib/index.js +++ b/lib/index.js
@@ -4,17 +4,17 @@
*/
const defineExport = (name, fn) => {
- let value;
- Object.defineProperty(exports, name, {
- get: () => {
- if (fn !== undefined) {
- value = fn();
- fn = undefined;
- }
- return value;
- },
- configurable: true
- });
+ let value;
+ Object.defineProperty(exports, name, {
+ get: () => {
+ if (fn !== undefined) {
+ value = fn();
+ fn = undefined;
+ }
+ return value;
+ },
+ configurable: true,
+ });
};
defineExport("Source", () => require("./Source"));
@@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource")); @@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource"));
defineExport("PrefixSource", () => require("./PrefixSource")); defineExport("PrefixSource", () => require("./PrefixSource"));
defineExport("SizeOnlySource", () => require("./SizeOnlySource")); defineExport("SizeOnlySource", () => require("./SizeOnlySource"));
......
...@@ -22,7 +22,7 @@ overrides: ...@@ -22,7 +22,7 @@ overrides:
patchedDependencies: patchedDependencies:
webpack-sources@3.2.3: webpack-sources@3.2.3:
hash: exarmjd4pnde4auoeobjrmju54 hash: jbynf5dc46ambamq3wuyho6hkq
path: patches/webpack-sources@3.2.3.patch path: patches/webpack-sources@3.2.3.patch
importers: importers:
...@@ -1471,7 +1471,7 @@ importers: ...@@ -1471,7 +1471,7 @@ importers:
version: webpack-sources@1.4.3 version: webpack-sources@1.4.3
webpack-sources3: webpack-sources3:
specifier: npm:webpack-sources@3.2.3 specifier: npm:webpack-sources@3.2.3
version: webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54) version: webpack-sources@3.2.3(patch_hash=jbynf5dc46ambamq3wuyho6hkq)
ws: ws:
specifier: 8.2.3 specifier: 8.2.3
version: 8.2.3 version: 8.2.3
...@@ -31425,7 +31425,7 @@ snapshots: ...@@ -31425,7 +31425,7 @@ snapshots:
source-list-map: 2.0.1 source-list-map: 2.0.1
source-map: 0.6.1 source-map: 0.6.1
webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54): {} webpack-sources@3.2.3(patch_hash=jbynf5dc46ambamq3wuyho6hkq): {}
webpack-stats-plugin@1.1.0: {} webpack-stats-plugin@1.1.0: {}
...@@ -31454,7 +31454,7 @@ snapshots: ...@@ -31454,7 +31454,7 @@ snapshots:
tapable: 2.2.0 tapable: 2.2.0
terser-webpack-plugin: 5.3.10(@swc/core@1.6.13(@swc/helpers@0.5.12))(webpack@5.90.0(@swc/core@1.6.13(@swc/helpers@0.5.12))) terser-webpack-plugin: 5.3.10(@swc/core@1.6.13(@swc/helpers@0.5.12))(webpack@5.90.0(@swc/core@1.6.13(@swc/helpers@0.5.12)))
watchpack: 2.4.0 watchpack: 2.4.0
webpack-sources: 3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54) webpack-sources: 3.2.3(patch_hash=jbynf5dc46ambamq3wuyho6hkq)
transitivePeerDependencies: transitivePeerDependencies:
- '@swc/core' - '@swc/core'
- esbuild - esbuild
...@@ -31485,7 +31485,7 @@ snapshots: ...@@ -31485,7 +31485,7 @@ snapshots:
tapable: 2.2.0 tapable: 2.2.0
terser-webpack-plugin: 5.3.10(@swc/core@1.7.0-nightly-20240714.1(@swc/helpers@0.5.12))(webpack@5.90.0(@swc/core@1.7.0-nightly-20240714.1(@swc/helpers@0.5.12))) terser-webpack-plugin: 5.3.10(@swc/core@1.7.0-nightly-20240714.1(@swc/helpers@0.5.12))(webpack@5.90.0(@swc/core@1.7.0-nightly-20240714.1(@swc/helpers@0.5.12)))
watchpack: 2.4.0 watchpack: 2.4.0
webpack-sources: 3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54) webpack-sources: 3.2.3(patch_hash=jbynf5dc46ambamq3wuyho6hkq)
transitivePeerDependencies: transitivePeerDependencies:
- '@swc/core' - '@swc/core'
- esbuild - esbuild
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment