Unverified Commit d1601a14 authored by Niklas Mischkulnig's avatar Niklas Mischkulnig Committed by GitHub

Turbopack: use xxhash3_128 for content hashes (#90583)

Move to 128 bit hashes, so that we can include the untruncated hashes in the manifest in #88607

If anything, this is faster, surprisingly:

```
upgrade twox-hash 448a36b449
    pnpm next build --experimental-build-mode=compile  430.13s user, 73.72s system, 724% cpu, 1:09.53 total
    pnpm next build --experimental-build-mode=compile  430.05s user, 82.51s system, 736% cpu, 1:09.56 total
    pnpm next build --experimental-build-mode=compile  419.95s user, 89.50s system, 697% cpu, 1:13.03 total
    pnpm next build --experimental-build-mode=compile  425.95s user, 75.87s system, 748% cpu, 1:07.00 total

use Xxh3Hash128Hex for content hashes ffd6a87f09
    pnpm next build --experimental-build-mode=compile  397.42s user, 53.18s system, 670% cpu, 1:07.25 total
    pnpm next build --experimental-build-mode=compile  416.68s user, 66.32s system, 762% cpu, 1:03.32 total
    pnpm next build --experimental-build-mode=compile  412.63s user, 68.30s system, 774% cpu, 1:02.12 total
    pnpm next build --experimental-build-mode=compile  419.09s user, 68.57s system, 756% cpu, 1:04.50 total
```


https://github.com/Cyan4973/xxHash says:
| Hash Name     | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment |
| ---------     | ----- | ---------------- | ----- | --- | --- |
| __XXH3__ (SSE2) |  64 | 31.5 GB/s        | 133.1 | 10
| __XXH128__ (SSE2) | 128 | 29.6 GB/s      | 118.1 | 10
| __XXH64__     |    64 | 19.4 GB/s        |  71.0 | 10
| __XXH32__     |    32 |  9.7 GB/s        |  71.9 | 10
parent 17033ca9
......@@ -80,7 +80,7 @@ describe('next-image-proxy', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(local).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=90"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=90"`
)
} else {
expect(local).toMatchInlineSnapshot(
......
......@@ -50,7 +50,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(layout.attr('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=85"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=85"`
)
} else {
expect(layout.attr('src')).toMatchInlineSnapshot(
......@@ -60,7 +60,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(layout.attr('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=85 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=85 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=85 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=85 2x"`
)
} else {
expect(layout.attr('srcset')).toMatchInlineSnapshot(
......@@ -72,7 +72,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(page.attr('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=90"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=90"`
)
} else {
expect(page.attr('src')).toMatchInlineSnapshot(
......@@ -82,7 +82,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(page.attr('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=90 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=90 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=90 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=90 2x"`
)
} else {
expect(page.attr('srcset')).toMatchInlineSnapshot(
......@@ -94,7 +94,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(comp.attr('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=80"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=80"`
)
} else {
expect(comp.attr('src')).toMatchInlineSnapshot(
......@@ -104,7 +104,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(comp.attr('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=80 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=80 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=80 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=80 2x"`
)
} else {
expect(comp.attr('srcset')).toMatchInlineSnapshot(
......@@ -194,7 +194,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await layout.getAttribute('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=85"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=85"`
)
} else {
expect(await layout.getAttribute('src')).toMatchInlineSnapshot(
......@@ -204,7 +204,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await layout.getAttribute('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=85 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=85 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=85 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=85 2x"`
)
} else {
expect(await layout.getAttribute('srcset')).toMatchInlineSnapshot(
......@@ -216,7 +216,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await page.getAttribute('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=90"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=90"`
)
} else {
expect(await page.getAttribute('src')).toMatchInlineSnapshot(
......@@ -226,7 +226,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await page.getAttribute('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=90 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=90 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=90 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=90 2x"`
)
} else {
expect(await page.getAttribute('srcset')).toMatchInlineSnapshot(
......@@ -238,7 +238,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await comp.getAttribute('src')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=80"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=80"`
)
} else {
expect(await comp.getAttribute('src')).toMatchInlineSnapshot(
......@@ -248,7 +248,7 @@ describe('app dir - next-image', () => {
if (process.env.IS_TURBOPACK_TEST) {
expect(await comp.getAttribute('srcset')).toMatchInlineSnapshot(
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=640&q=80 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.55df2443.png&w=828&q=80 2x"`
`"/_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=640&q=80 1x, /_next/image?url=%2F_next%2Fstatic%2Fmedia%2Ftest.c5ae911e.png&w=828&q=80 2x"`
)
} else {
expect(await comp.getAttribute('srcset')).toMatchInlineSnapshot(
......
......@@ -564,8 +564,8 @@ module.exports = {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/chunks/HASH.css:
.red-text{color:red;background-image:url(../media/dark.656fe8ee.svg),url(../media/dark2.656fe8ee.svg)}
.blue-text{color:orange;background-image:url(../media/light.750c956a.svg);font-weight:bolder}
.red-text{color:red;background-image:url(../media/dark.3ca0d560.svg),url(../media/dark2.3ca0d560.svg)}
.blue-text{color:orange;background-image:url(../media/light.c5f50e38.svg);font-weight:bolder}
.blue-text{color:#00f}",
]
`)
......@@ -573,8 +573,8 @@ module.exports = {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/chunks/HASH.css:
.red-text{color:red;background-image:url(../media/dark.656fe8ee.svg),url(../media/dark2.656fe8ee.svg)}
.blue-text{color:orange;background-image:url(../media/light.750c956a.svg);font-weight:bolder}
.red-text{color:red;background-image:url(../media/dark.3ca0d560.svg),url(../media/dark2.3ca0d560.svg)}
.blue-text{color:orange;background-image:url(../media/light.c5f50e38.svg);font-weight:bolder}
.blue-text{color:#00f}",
]
`)
......@@ -640,8 +640,8 @@ describe('CSS URL via `file-loader` and asset prefix (1)', () => {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/chunks/HASH.css:
.red-text{color:red;background-image:url(../media/dark.656fe8ee.svg) url(../media/dark2.656fe8ee.svg)}
.blue-text{color:orange;background-image:url(../media/light.750c956a.svg);font-weight:bolder}
.red-text{color:red;background-image:url(../media/dark.3ca0d560.svg) url(../media/dark2.3ca0d560.svg)}
.blue-text{color:orange;background-image:url(../media/light.c5f50e38.svg);font-weight:bolder}
.blue-text{color:#00f}",
]
`)
......@@ -692,8 +692,8 @@ describe('CSS URL via `file-loader` and asset prefix (2)', () => {
expect(cssContent).toMatchInlineSnapshot(`
[
"/_next/static/chunks/HASH.css:
.red-text{color:red;background-image:url(../media/dark.656fe8ee.svg) url(../media/dark2.656fe8ee.svg)}
.blue-text{color:orange;background-image:url(../media/light.750c956a.svg);font-weight:bolder}
.red-text{color:red;background-image:url(../media/dark.3ca0d560.svg) url(../media/dark2.3ca0d560.svg)}
.blue-text{color:orange;background-image:url(../media/light.c5f50e38.svg);font-weight:bolder}
.blue-text{color:#00f}",
]
`)
......
......@@ -579,7 +579,7 @@ impl ChunkingContext for BrowserChunkingContext {
};
let hash = asset
.content()
.content_hash(HashAlgorithm::default())
.content_hash(HashAlgorithm::Xxh3Hash128Hex)
.await?;
let hash = hash.as_ref().context(
"chunk_path requires an asset with file content when content hashing is \
......
......@@ -234,7 +234,7 @@ impl FileHashVersion {
match asset_content {
AssetContent::File(file_vc) => {
let hash = file_vc
.content_hash(HashAlgorithm::default())
.content_hash(HashAlgorithm::Xxh3Hash128Hex)
.owned()
.await?
.context("file not found")?;
......
......@@ -41,7 +41,7 @@ impl OutputAsset for StaticOutputAsset {
async fn path(&self) -> Result<Vc<FileSystemPath>> {
let content = self.source.content();
let content_hash = content
.content_hash(HashAlgorithm::default())
.content_hash(HashAlgorithm::Xxh3Hash128Hex)
.owned()
.await?
.context(
......
/* [project]/turbopack/crates/turbopack-tests/tests/snapshot/css/embed-url/input/style.module.css [test] (css) */
.style-module__3oVw9q__module-style {
cursor: url("../static/image.7bbaf41b.png");
background-image: url("../static/image.7bbaf41b.png");
list-style-image: url("../static/image.7bbaf41b.png");
cursor: url("../static/image.de249277.png");
background-image: url("../static/image.de249277.png");
list-style-image: url("../static/image.de249277.png");
}
/* [project]/turbopack/crates/turbopack-tests/tests/snapshot/css/embed-url/input/style.css [test] (css) */
.style {
cursor: url("../static/image.7bbaf41b.png");
background-image: url("../static/image.7bbaf41b.png");
list-style-image: url("../static/image.7bbaf41b.png");
cursor: url("../static/image.de249277.png");
background-image: url("../static/image.de249277.png");
list-style-image: url("../static/image.de249277.png");
}
.style-unresolveable {
......
(globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push(["output/turbopack_crates_turbopack-tests_tests_snapshot_import-meta_url_input_3fece31c._.js",
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/import-meta/url/input/asset.txt (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/asset.6fdf3006.txt");}),
__turbopack_context__.q("/static/asset.76d5cf5c.txt");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/import-meta/url/input/mod.mjs [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
......
......@@ -5,7 +5,7 @@ module.exports = 'turbopack';
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/ignore-comments/input/vercel.cjs (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/vercel.fad5a703.cjs");}),
__turbopack_context__.q("/static/vercel.216e8e39.cjs");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/ignore-comments/input/vercel.cjs [test] (ecmascript, worker loader)", ((__turbopack_context__) => {
__turbopack_context__.v(function(Ctor, opts) {
......@@ -14,7 +14,7 @@ __turbopack_context__.v(function(Ctor, opts) {
}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/ignore-comments/input/ignore-worker.cjs (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/ignore-worker.4e0cf842.cjs");}),
__turbopack_context__.q("/static/ignore-worker.ce2a21fa.cjs");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/ignore-comments/input/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
......
(globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push(["output/turbopack_crates_turbopack-tests_tests_snapshot_imports_static_input_fb142aa8._.js",
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/static/input/vercel.svg (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/vercel.77adf0a9.svg");}),
__turbopack_context__.q("/static/vercel.b457dd5d.svg");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/imports/static/input/index.js [test] (ecmascript)", ((__turbopack_context__) => {
"use strict";
......
(globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push(["output/turbopack_crates_turbopack-tests_tests_snapshot_workers_basic_input_73fc86c5._.js",
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/workers/basic/input/worker.js (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/worker.adfa2c73.js");}),
__turbopack_context__.q("/static/worker.b90191a4.js");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/workers/basic/input/worker.js [test] (ecmascript, worker loader)", ((__turbopack_context__) => {
__turbopack_context__.v(function(Ctor, opts) {
......
(globalThis["TURBOPACK"] || (globalThis["TURBOPACK"] = [])).push(["output/turbopack_crates_turbopack-tests_tests_snapshot_workers_shared_input_3845375a._.js",
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/workers/shared/input/worker.js (static in ecmascript)", ((__turbopack_context__) => {
__turbopack_context__.q("/static/worker.b01ba0d1.js");}),
__turbopack_context__.q("/static/worker.63640b21.js");}),
"[project]/turbopack/crates/turbopack-tests/tests/snapshot/workers/shared/input/worker.js [test] (ecmascript, worker loader)", ((__turbopack_context__) => {
__turbopack_context__.v(function(Ctor, opts) {
......
......@@ -26,7 +26,7 @@ pub mod source;
pub async fn wasm_edge_var_name(asset: Vc<Box<dyn Asset>>) -> Result<Vc<RcStr>> {
let hash = asset
.content()
.content_hash(HashAlgorithm::default())
.content_hash(HashAlgorithm::Xxh3Hash128Hex)
.await?;
let hash = hash
.as_ref()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment