Unverified Commit 68cecd6a authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Update default error rate for client filter (#60542)

This updates our default error rate to be much more precise and reduce
false positives by increasing the default size of the client filter we
generate. We can afford to increase the default size as it compresses
extremely well and gives us more accurate navigations. This carries over
the failing test case from https://github.com/vercel/next.js/pull/59293
which showed one case of false positive in a smaller filter.

Closes: https://github.com/vercel/next.js/issues/47486

Closes NEXT-2070

---------
Co-authored-by: 's avatarCris Vergara <cris.vergara@bookofthemonth.com>
parent 9fd25b4b
......@@ -10,6 +10,9 @@ function murmurhash2(str: string) {
return h >>> 0
}
// default to 0.01% error rate as the filter compresses very well
const DEFAULT_ERROR_RATE = 0.0001
export class BloomFilter {
numItems: number
errorRate: number
......@@ -17,7 +20,7 @@ export class BloomFilter {
numHashes: number
bitArray: number[]
constructor(numItems: number, errorRate: number) {
constructor(numItems: number, errorRate: number = DEFAULT_ERROR_RATE) {
this.numItems = numItems
this.errorRate = errorRate
this.numBits = Math.ceil(
......@@ -27,7 +30,7 @@ export class BloomFilter {
this.bitArray = new Array(this.numBits).fill(0)
}
static from(items: string[], errorRate = 0.01) {
static from(items: string[], errorRate = DEFAULT_ERROR_RATE) {
const filter = new BloomFilter(items.length, errorRate)
for (const item of items) {
......@@ -46,7 +49,7 @@ export class BloomFilter {
}
if (process.env.NEXT_RUNTIME === 'nodejs') {
if (this.errorRate < 0.01) {
if (this.errorRate < DEFAULT_ERROR_RATE) {
const filterData = JSON.stringify(data)
const gzipSize = require('next/dist/compiled/gzip-size').sync(
filterData
......@@ -54,7 +57,7 @@ export class BloomFilter {
if (gzipSize > 1024) {
console.warn(
`Creating filter with error rate less than 1% (0.01) can increase the size dramatically proceed with caution. Received error rate ${this.errorRate} resulted in size ${filterData.length} bytes, ${gzipSize} bytes (gzip)`
`Creating filter with error rate less than 0.1% (0.001) can increase the size dramatically proceed with caution. Received error rate ${this.errorRate} resulted in size ${filterData.length} bytes, ${gzipSize} bytes (gzip)`
)
}
}
......
/* eslint-env jest */
import { createClientRouterFilter } from 'next/dist/lib/create-client-router-filter'
import { BloomFilter } from 'next/dist/shared/lib/bloom-filter'
describe('createClientRouterFilter', () => {
it('creates a filter that does not collide with wildly different path names', () => {
const { staticFilter, dynamicFilter } = createClientRouterFilter(
['/_not-found', '/a/[lang]/corporate', '/a/[lang]/gift'], // Routes are based on BOTM's app router migration project.
[]
)
const staticFilterInstance = new BloomFilter(
staticFilter.numItems,
staticFilter.errorRate
)
staticFilterInstance.import(staticFilter)
const dynamicFilterInstance = new BloomFilter(
dynamicFilter.numItems,
dynamicFilter.errorRate
)
dynamicFilterInstance.import(dynamicFilter)
expect(
staticFilterInstance.contains(
'/all-hardcovers/no-one-can-know-1511?category=current-features'
)
).toBe(false)
})
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment