Unverified Commit 33237020 authored by JJ Kasper's avatar JJ Kasper Committed by GitHub

Rework client router filter handling (#49741)

This simplifies the implementation for the client router filter and
makes the error rate configuring more accurate. Additional tests haven't
been added as this is still a matter of probability and existing tests
cover the client router behavior.

Example scenario of filter size/false positives with an error rate of
`0.01%`:

```sh
entries: 128 filter size: 4986 bytes, gzip size: 627 bytes
false positives with 100000 samples: 26 0.026%
```

x-ref: [slack
thread](https://vercel.slack.com/archives/C04K237UHCP/p1683129488543039)
parent c3f54ec4
...@@ -5,12 +5,10 @@ import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing- ...@@ -5,12 +5,10 @@ import { removeTrailingSlash } from '../shared/lib/router/utils/remove-trailing-
import { Redirect } from './load-custom-routes' import { Redirect } from './load-custom-routes'
import { tryToParsePath } from './try-to-parse-path' import { tryToParsePath } from './try-to-parse-path'
const POTENTIAL_ERROR_RATE = 0.01
export function createClientRouterFilter( export function createClientRouterFilter(
paths: string[], paths: string[],
redirects: Redirect[], redirects: Redirect[],
allowedErrorRate: number = POTENTIAL_ERROR_RATE allowedErrorRate?: number
): { ): {
staticFilter: ReturnType<BloomFilter['export']> staticFilter: ReturnType<BloomFilter['export']>
dynamicFilter: ReturnType<BloomFilter['export']> dynamicFilter: ReturnType<BloomFilter['export']>
......
...@@ -8,7 +8,7 @@ import { ...@@ -8,7 +8,7 @@ import {
IncrementalCacheValue, IncrementalCacheValue,
IncrementalCacheEntry, IncrementalCacheEntry,
} from '../../response-cache' } from '../../response-cache'
import { encode } from '../../../shared/lib/bloom-filter/base64-arraybuffer' import { encode } from '../../../shared/lib/base64-arraybuffer'
import { encodeText } from '../../stream-utils/encode-decode' import { encodeText } from '../../stream-utils/encode-decode'
import { import {
CACHE_ONE_YEAR, CACHE_ONE_YEAR,
......
...@@ -446,7 +446,7 @@ export function patchFetch({ ...@@ -446,7 +446,7 @@ export function patchFetch({
if (process.env.NEXT_RUNTIME === 'edge') { if (process.env.NEXT_RUNTIME === 'edge') {
const { decode } = const { decode } =
require('../../shared/lib/bloom-filter/base64-arraybuffer') as typeof import('../../shared/lib/bloom-filter/base64-arraybuffer') require('../../shared/lib/base64-arraybuffer') as typeof import('../../shared/lib/base64-arraybuffer')
decodedBody = decode(resData.body) decodedBody = decode(resData.body)
} else { } else {
decodedBody = Buffer.from(resData.body, 'base64').subarray() decodedBody = Buffer.from(resData.body, 'base64').subarray()
......
// minimal implementation MurmurHash2 hash function
function murmurhash2(str: string) {
let h = 0
for (let i = 0; i < str.length; i++) {
const c = str.charCodeAt(i)
h = Math.imul(h ^ c, 0x5bd1e995)
h ^= h >>> 13
h = Math.imul(h, 0x5bd1e995)
}
return h >>> 0
}
export class BloomFilter {
numItems: number
errorRate: number
numBits: number
numHashes: number
bitArray: number[]
constructor(numItems: number, errorRate: number) {
this.numItems = numItems
this.errorRate = errorRate
this.numBits = Math.ceil(
-(numItems * Math.log(errorRate)) / (Math.log(2) * Math.log(2))
)
this.numHashes = Math.ceil((this.numBits / numItems) * Math.log(2))
this.bitArray = new Array(this.numBits).fill(0)
}
static from(items: string[], errorRate = 0.01) {
const filter = new BloomFilter(items.length, errorRate)
for (const item of items) {
filter.add(item)
}
return filter
}
export() {
const data = {
numItems: this.numItems,
errorRate: this.errorRate,
numBits: this.numBits,
numHashes: this.numHashes,
bitArray: this.bitArray,
}
if (typeof window === 'undefined' && process.env.NEXT_RUNTIME !== 'edge') {
if (this.errorRate < 0.01) {
const filterData = JSON.stringify(data)
const gzipSize = require('next/dist/compiled/gzip-size').sync(
filterData
)
if (gzipSize > 1024) {
console.warn(
`Creating filter with error rate less than 1% (0.01) can increase the size dramatically proceed with caution. Received error rate ${this.errorRate} resulted in size ${filterData.length} bytes, ${gzipSize} bytes (gzip)`
)
}
}
}
return data
}
import(data: ReturnType<typeof this['export']>) {
this.numItems = data.numItems
this.errorRate = data.errorRate
this.numBits = data.numBits
this.numHashes = data.numHashes
this.bitArray = data.bitArray
}
add(item: string) {
const hashValues = this.getHashValues(item)
hashValues.forEach((hash) => {
this.bitArray[hash] = 1
})
}
contains(item: string) {
const hashValues = this.getHashValues(item)
return hashValues.every((hash) => this.bitArray[hash])
}
getHashValues(item: string) {
const hashValues = []
for (let i = 1; i <= this.numHashes; i++) {
const hash = murmurhash2(`${item}${i}`) % this.numBits
hashValues.push(hash)
}
return hashValues
}
}
/* file : base-filter.ts
MIT License
Copyright (c) 2017-2020 Thomas Minier & Arnaud Grall
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
import Hashing from './hashing'
import { getDefaultSeed } from './utils'
/**
* Exported prng type because it is not from seedrandom
* Orignal type can be found in: @types/seedrandom
*/
export interface prng {
(): number
int32(): number
quick(): number
}
function randomInt32() {
if (typeof window === 'undefined' && process.env.NEXT_RUNTIME === 'nodejs') {
return (require('crypto') as typeof import('crypto'))
.randomBytes(4)
.readUInt32BE(0)
}
return crypto.getRandomValues(new Uint32Array(1))[0]
}
function seedrandom() {
return {
int32: randomInt32,
quick: randomInt32,
}
}
/**
* A base class for implementing probailistic filters
* @author Thomas Minier
* @author Arnaud Grall
*/
export default abstract class BaseFilter {
public _seed: number
public _rng: prng
public _hashing: Hashing
constructor() {
this._seed = getDefaultSeed()
this._rng = seedrandom() as prng
this._hashing = new Hashing()
}
/**
* Get the seed used in this structure
*/
public get seed(): number {
return this._seed
}
/**
* Set the seed for this structure
* @param seed the new seed that will be used in this structure
*/
public set seed(seed: number) {
this._seed = seed
this._rng = seedrandom() as prng
}
/**
* Get a function used to draw random number
* @return A factory function used to draw random integer
*/
public get random(): prng {
return this._rng
}
/**
* Return a next random seeded int32 integer
* @returns
*/
public nextInt32(): number {
return this._rng.int32()
}
/**
* Save the current structure as a JSON
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
public saveAsJSON(): any {
throw new Error('not-implemented')
}
/**
* Load an Object from a provided JSON object
* @param json the JSON object to load
* @return Return the Object loaded from the provided JSON object
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unused-vars
public static fromJSON(json: JSON): any {
throw new Error(`not-implemented`)
}
}
import { encode, decode } from './base64-arraybuffer'
const bitsPerWord = 8
/**
* A memory-efficient Boolean array. Contains just the minimal operations needed for our Bloom filter implementation.
*
* @author David Leppik
*/
export default class BitSet {
public readonly size: number
// Uint32Array may be slightly faster due to memory alignment, but this avoids endianness when serializing
public array: Uint8Array
/**
* Constructor. All bits are initially set to false.
* @param size the number of bits that can be stored. (This is NOT required to be a multiple of 8.)
*/
constructor(size: number) {
const diff = bitsPerWord - (size % bitsPerWord)
this.size = size + ([0, 8].includes(diff) ? 0 : diff)
this.array = new Uint8Array(Math.ceil(this.size / bitsPerWord))
}
/**
* Returns the value of the bit at the given index
* @param index position of the bit, zero-indexed
*/
public has(index: number): boolean {
const wordIndex = Math.floor(index / bitsPerWord)
const mask = 1 << index % bitsPerWord
return (this.array[wordIndex] & mask) !== 0
}
/**
* Set the bit to true
* @param index position of the bit, zero-indexed
*/
public add(index: number) {
const wordIndex = Math.floor(index / bitsPerWord)
const mask = 1 << index % bitsPerWord
this.array[wordIndex] = this.array[wordIndex] | mask
}
/**
* Returns the maximum true bit.
*/
public max(): number {
for (let i = this.array.length - 1; i >= 0; i--) {
const bits = this.array[i]
if (bits) {
return BitSet.highBit(bits) + i * bitsPerWord
}
}
return 0
}
/**
* Returns the number of true bits.
*/
public bitCount(): number {
let result = 0
for (let i = 0; i < this.array.length; i++) {
result += BitSet.countBits(this.array[i]) // Assumes we never have bits set beyond the end
}
return result
}
/**
* Returns true if the size and contents are identical.
* @param other another BitSet
*/
public equals(other: BitSet): boolean {
if (other.size !== this.size) {
return false
}
for (let i = 0; i < this.array.length; i++) {
if (this.array[i] !== other.array[i]) {
return false
}
}
return true
}
/**
* Returns a JSON-encodable object readable by {@link import}.
*/
public export(): { size: number; content: string } {
return {
size: this.size,
content: encode(this.array),
}
}
/**
* Returns an object written by {@link export}.
* @param data an object written by {@link export}
*/
public static import(data: { size: number; content: string }): BitSet {
if (typeof data.size !== 'number') {
throw Error('BitSet missing size')
}
if (typeof data.content !== 'string') {
throw Error('BitSet: missing content')
}
const result = new BitSet(data.size)
const buffer = decode(data.content)
result.array = new Uint8Array(buffer)
return result
}
/**
* Returns the index of the maximum bit in the number, or -1 for 0
* @bits an unsigned 8-bit number
* ```js
* @example
* BitSet.highBit(0) // returns -1
* BitSet.highBit(5) // returns 2
* ```
*/
public static highBit(bits: number): number {
let result = bitsPerWord - 1
let mask = 1 << result
while (result >= 0 && (mask & bits) !== mask) {
mask >>>= 1
result--
}
return result
}
/**
* Returns the number of true bits in the number
* @bits an unsigned 8-bit number
* @example
* ```js
* BitSet.countBits(0) // returns 0
* BitSet.countBits(3) // returns 2
* ```
*/
public static countBits(bits: number): number {
let result = bits & 1
while (bits !== 0) {
bits = bits >>> 1
result += bits & 1
}
return result
}
}
/* file : formulas.ts
MIT License
Copyright (c) 2017-2020 Thomas Minier & Arnaud Grall
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
/**
* Various formulas used with Bloom Filters
* @namespace Formulas
* @private
*/
/**
* Compute the optimal size of a Bloom Filter
* @param length - The length of the set used to fill the filter
* @param errorRate - The targeted false positive rate
* @return The optimal size of a Bloom Filter
* @memberof Formulas
*/
export function optimalFilterSize(length: number, errorRate: number): number {
return Math.ceil(-((length * Math.log(errorRate)) / Math.pow(Math.log(2), 2)))
}
/**
* Compute the optimal number of hash functions to be used by a Bloom Filter
* @param size - The size of the filter
* @param length - The length of the set used to fill the filter
* @return The optimal number of hash functions to be used by a Bloom Filter
* @memberof Formulas
*/
export function optimalHashes(size: number, length: number): number {
return Math.ceil((size / length) * Math.log(2))
}
import fnv1a from '../fnv1a'
import { getDefaultSeed, numberToHex } from './utils'
/**
* @typedef {TwoHashes} Two hashes of the same value, as computed by {@link hashTwice}.
* @property {number} first - The result of the first hashing function applied to a value
* @property {number} second - The result of the second hashing function applied to a value
* @memberof Utils
*/
export interface TwoHashes {
first: number
second: number
}
/**
* Templated TwoHashes type
*/
export interface TwoHashesTemplated<T> {
first: T
second: T
}
/**
* TwoHashes type in number and int format
*/
export interface TwoHashesIntAndString {
int: TwoHashesTemplated<number>
string: TwoHashesTemplated<string>
}
/**
* Data type of an hashable value, must be string, ArrayBuffer or Buffer.
*/
export type HashableInput = string
export default class Hashing implements Hashing {
/**
* Apply enhanced Double Hashing to produce a n-hash
* @see {@link http://peterd.org/pcd-diss.pdf} s6.5.4
* @param n - The indice of the hash function we want to produce
* @param hashA - The result of the first hash function applied to a value.
* @param hashB - The result of the second hash function applied to a value.
* @param size - The size of the datastructures associated to the hash context (ex: the size of a Bloom Filter)
* @return The result of hash_n applied to a value.
* @memberof Hashing
* @author Thomas Minier
* @author Arnaud Grall
*/
public doubleHashing(
n: number,
hashA: number,
hashB: number,
size: number
): number {
return Math.abs((hashA + n * hashB + Math.floor((n ** 3 - n) / 6)) % size)
}
/**
* Generate a set of distinct indexes on interval [0, size) using the double hashing technique
* For generating efficiently distinct indexes we re-hash after detecting a cycle by changing slightly the seed.
* It has the effect of generating faster distinct indexes without loosing entirely the utility of the double hashing.
* For small number of indexes it will work perfectly. For a number close to the size, and size very large
* Advise: do not generate `size` indexes for a large interval. In practice, size should be equal
* to the number of hash functions used and is often low.
*
* @param element - The element to hash
* @param size - the range on which we can generate an index [0, size) = size
* @param number - The number of indexes desired
* @param seed - The seed used
* @return Array<number>
* @author Arnaud Grall
* @author Simon Woolf (SimonWoolf)
*/
public getDistinctIndexes(
element: HashableInput,
size: number,
number: number,
seed?: number
): Array<number> {
if (seed === undefined) {
seed = getDefaultSeed()
}
let n = 0
const indexes: Set<number> = new Set()
let hashes = this.hashTwice(element, seed)
// let cycle = 0
while (indexes.size < number) {
const ind = hashes.first % size
if (!indexes.has(ind)) {
indexes.add(ind)
}
hashes.first = (hashes.first + hashes.second) % size
hashes.second = (hashes.second + n) % size
n++
if (n > size) {
// Enhanced double hashing stops cycles of length less than `size` in the case where
// size is coprime with the second hash. But you still get cycles of length `size`.
// So if we reach there and haven't finished, append a prime to the input and
// rehash.
seed++
hashes = this.hashTwice(element, seed)
}
}
return [...indexes.values()]
}
/**
* Generate N indexes on range [0, size)
* It uses the double hashing technique to generate the indexes.
* It hash twice the value only once before generating the indexes.
* Warning: you can have a lot of modulo collisions.
* @param element - The element to hash
* @param size - The range on which we can generate the index, exclusive
* @param hashCount - The number of indexes we want
* @return An array of indexes on range [0, size)
*/
public getIndexes(
element: HashableInput,
size: number,
hashCount: number,
seed?: number
): Array<number> {
if (seed === undefined) {
seed = getDefaultSeed()
}
const arr = []
const hashes = this.hashTwice(element, seed)
for (let i = 0; i < hashCount; i++) {
arr.push(this.doubleHashing(i, hashes.first, hashes.second, size))
}
return arr
}
/**
* @internal
* Hash an element of type {@link HashableInput} into {@link Number}
* Can be overrided as long as you return a value of type {@link Number}
* Don't forget to use the seed when hashing, otherwise if some kind of randomness is in the process
* you may have inconsistent behaviors between 2 runs.
* @param element
* @param seed
* @returns A 64bits floating point {@link Number}
*/
public serialize(element: HashableInput, seed?: number) {
if (!seed) {
seed = getDefaultSeed()
}
return Number(fnv1a(element, { seed }))
}
/**
* (64-bits only) Hash a value into two values (in hex or integer format)
* @param value - The value to hash
* @param asInt - (optional) If True, the values will be returned as an integer. Otherwise, as hexadecimal values.
* @param seed the seed used for hashing
* @return The results of the hash functions applied to the value (in hex or integer)
* @author Arnaud Grall & Thomas Minier
*/
public hashTwice(value: HashableInput, seed?: number): TwoHashes {
if (seed === undefined) {
seed = getDefaultSeed()
}
return {
first: this.serialize(value, seed + 1),
second: this.serialize(value, seed + 2),
}
}
/**
* Hash twice an element into their HEX string representations
* @param value
* @param seed
* @returns TwoHashesTemplated<string>
*/
public hashTwiceAsString(
value: HashableInput,
seed?: number
): TwoHashesTemplated<string> {
const { first, second } = this.hashTwice(value, seed)
return {
first: numberToHex(first),
second: numberToHex(second),
}
}
/**
* (64-bits only) Same as hashTwice but return Numbers and String equivalent
* @param val the value to hash
* @param seed the seed to change when hashing
* @return TwoHashesIntAndString
* @author Arnaud Grall
*/
public hashTwiceIntAndString(
val: HashableInput,
seed?: number
): TwoHashesIntAndString {
if (seed === undefined) {
seed = getDefaultSeed()
}
const one = this.hashIntAndString(val, seed + 1)
const two = this.hashIntAndString(val, seed + 2)
return {
int: {
first: one.int,
second: two.int,
},
string: {
first: one.string,
second: two.string,
},
}
}
/**
* Hash an item as an unsigned int
* @param elem - Element to hash
* @param seed - The hash seed. If its is UINT32 make sure to set the length to 32
* @param length - The length of hashes (defaults to 32 bits)
* @return The hash value as an unsigned int
* @author Arnaud Grall
*/
public hashAsInt(elem: HashableInput, seed?: number): number {
if (seed === undefined) {
seed = getDefaultSeed()
}
return this.serialize(elem, seed)
}
/**
* Hash an item and return its number and HEX string representation
* @param elem - Element to hash
* @param seed - The hash seed. If its is UINT32 make sure to set the length to 32
* @param base - The base in which the string will be returned, default: 16
* @param length - The length of hashes (defaults to 32 bits)
* @return The item hased as an int and a string
* @author Arnaud Grall
*/
public hashIntAndString(elem: HashableInput, seed?: number) {
const hash = this.hashAsInt(elem, seed)
return { int: hash, string: numberToHex(hash) }
}
}
// original source: https://github.com/Callidon/bloom-filters
/* file : bloom-filter.ts
MIT License
Copyright (c) 2017 Thomas Minier & Arnaud Grall
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
import BaseFilter from './base-filter'
import BitSet from './bit-set'
import { optimalFilterSize, optimalHashes } from './formulas'
import { HashableInput } from './hashing'
/**
* A Bloom filter is a space-efficient probabilistic data structure, conceived by Burton Howard Bloom in 1970,
* that is used to test whether an element is a member of a set. False positive matches are possible, but false negatives are not.
*
* Reference: Bloom, B. H. (1970). Space/time trade-offs in hash coding with allowable errors. Communications of the ACM, 13(7), 422-426.
* @see {@link http://crystal.uta.edu/~mcguigan/cse6350/papers/Bloom.pdf} for more details about classic Bloom Filters.
* @author Thomas Minier
* @author Arnaud Grall
*/
export class BloomFilter extends BaseFilter {
public _size: number
public _nbHashes: number
public _filter: BitSet
/**
* Constructor
* @param size - The number of cells
* @param nbHashes - The number of hash functions used
*/
constructor(size: number, nbHashes: number) {
super()
if (nbHashes < 1) {
throw new Error(
`A BloomFilter cannot uses less than one hash function, while you tried to use ${nbHashes}.`
)
}
this._size = size
this._nbHashes = nbHashes
this._filter = new BitSet(size)
}
/**
* Create an optimal bloom filter providing the maximum of elements stored and the error rate desired
* @param nbItems - The maximum number of item to store
* @param errorRate - The error rate desired for a maximum of items inserted
* @return A new {@link BloomFilter}
*/
public static create(nbItems: number, errorRate: number): BloomFilter {
const size = optimalFilterSize(nbItems, errorRate)
const hashes = optimalHashes(size, nbItems)
return new this(size, hashes)
}
/**
* Build a new Bloom Filter from an existing iterable with a fixed error rate
* @param items - The iterable used to populate the filter
* @param errorRate - The error rate, i.e. 'false positive' rate, targeted by the filter
* @param seed - The random number seed (optional)
* @return A new Bloom Filter filled with the iterable's elements
* @example
* ```js
* // create a filter with a false positive rate of 0.1
* const filter = BloomFilter.from(['alice', 'bob', 'carl'], 0.1);
* ```
*/
public static from(
items: Iterable<HashableInput>,
errorRate: number,
seed?: number
): BloomFilter {
const array = Array.from(items)
const filter = BloomFilter.create(array.length, errorRate)
if (typeof seed === 'number') {
filter.seed = seed
}
array.forEach((element) => filter.add(element))
return filter
}
/**
* Get the optimal size of the filter
* @return The size of the filter
*/
get size(): number {
return this._size
}
/**
* Get the number of bits currently set in the filter
* @return The filter length
*/
public get length(): number {
return this._filter.bitCount()
}
/**
* Add an element to the filter
* @param element - The element to add
* @example
* ```js
* const filter = new BloomFilter(15, 0.1);
* filter.add('foo');
* ```
*/
public add(element: HashableInput): void {
const indexes = this._hashing.getIndexes(
element,
this._size,
this._nbHashes,
this.seed
)
for (let i = 0; i < indexes.length; i++) {
this._filter.add(indexes[i])
}
}
/**
* Test an element for membership
* @param element - The element to look for in the filter
* @return False if the element is definitively not in the filter, True is the element might be in the filter
* @example
* ```js
* const filter = new BloomFilter(15, 0.1);
* filter.add('foo');
* console.log(filter.has('foo')); // output: true
* console.log(filter.has('bar')); // output: false
* ```
*/
public has(element: HashableInput): boolean {
const indexes = this._hashing.getIndexes(
element,
this._size,
this._nbHashes,
this.seed
)
for (let i = 0; i < indexes.length; i++) {
if (!this._filter.has(indexes[i])) {
return false
}
}
return true
}
/**
* Get the current false positive rate (or error rate) of the filter
* @return The current false positive rate of the filter
* @example
* ```js
* const filter = new BloomFilter(15, 0.1);
* console.log(filter.rate()); // output: something around 0.1
* ```
*/
public rate(): number {
return Math.pow(1 - Math.exp(-this.length / this._size), this._nbHashes)
}
/**
* Check if another Bloom Filter is equal to this one
* @param other - The filter to compare to this one
* @return True if they are equal, false otherwise
*/
public equals(other: BloomFilter): boolean {
if (this._size !== other._size || this._nbHashes !== other._nbHashes) {
return false
}
return this._filter.equals(other._filter)
}
public export() {
return {
bitset: this._filter.export(),
hashes: this._nbHashes,
size: this._size,
}
}
public import(data: ReturnType<typeof this.export>) {
this._filter = BitSet.import(data.bitset)
}
}
/* file : utils.ts
MIT License
Copyright (c) 2017-2020 Thomas Minier & Arnaud Grall
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
/**
* Utilitaries functions
* @namespace Utils
* @private
*/
/* JSDOC typedef */
/**
* @typedef {TwoHashes} Two hashes of the same value, as computed by {@link hashTwice}.
* @property {number} first - The result of the first hashing function applied to a value
* @property {number} second - The result of the second hashing function applied to a value
* @memberof Utils
*/
export interface TwoHashes {
first: number
second: number
}
/**
* Templated TwoHashes type
*/
export interface TwoHashesTemplated<T> {
first: T
second: T
}
/**
* TwoHashes type in number and int format
*/
export interface TwoHashesIntAndString {
int: TwoHashesTemplated<number>
string: TwoHashesTemplated<string>
}
/**
* Data type of an hashable value, must be string, ArrayBuffer or Buffer.
*/
export type HashableInput = string | ArrayBuffer | Buffer
/**
* Return a number to its Hex format by padding zeroes if length mod 4 != 0
* @param elem the element to transform in HEX
* @returns the HEX number padded of zeroes
*/
export function numberToHex(elem: number): string {
let e = Number(elem).toString(16)
if (e.length % 4 !== 0) {
e = '0'.repeat(4 - (e.length % 4)) + e
}
return e
}
/**
* Generate a random int between two bounds (included)
* @param min - The lower bound
* @param max - The upper bound
* @param random - Function used to generate random floats
* @return A random int bewteen lower and upper bound (included)
* @memberof Utils
* @author Thomas Minier
*/
export function randomInt(
min: number,
max: number,
random?: () => number
): number {
if (random === undefined) {
random = Math.random
}
min = Math.ceil(min)
max = Math.floor(max)
const rn = random()
return Math.floor(rn * (max - min + 1)) + min
}
/**
* Return the default seed used in the package
* @return A seed as a floating point number
* @author Arnaud Grall
*/
export function getDefaultSeed(): number {
return 0x1234567890
}
...@@ -792,18 +792,18 @@ export default class Router implements BaseRouter { ...@@ -792,18 +792,18 @@ export default class Router implements BaseRouter {
const dynamicFilterData: typeof staticFilterData = process.env const dynamicFilterData: typeof staticFilterData = process.env
.__NEXT_CLIENT_ROUTER_D_FILTER as any .__NEXT_CLIENT_ROUTER_D_FILTER as any
if (staticFilterData?.hashes) { if (staticFilterData?.numHashes) {
this._bfl_s = new BloomFilter( this._bfl_s = new BloomFilter(
staticFilterData.size, staticFilterData.numItems,
staticFilterData.hashes staticFilterData.errorRate
) )
this._bfl_s.import(staticFilterData) this._bfl_s.import(staticFilterData)
} }
if (dynamicFilterData?.hashes) { if (dynamicFilterData?.numHashes) {
this._bfl_d = new BloomFilter( this._bfl_d = new BloomFilter(
dynamicFilterData.size, dynamicFilterData.numItems,
dynamicFilterData.hashes dynamicFilterData.errorRate
) )
this._bfl_d.import(dynamicFilterData) this._bfl_d.import(dynamicFilterData)
} }
...@@ -1082,8 +1082,8 @@ export default class Router implements BaseRouter { ...@@ -1082,8 +1082,8 @@ export default class Router implements BaseRouter {
) { ) {
matchesBflStatic = matchesBflStatic =
matchesBflStatic || matchesBflStatic ||
!!this._bfl_s?.has(asNoSlash) || !!this._bfl_s?.contains(asNoSlash) ||
!!this._bfl_s?.has(asNoSlashLocale) !!this._bfl_s?.contains(asNoSlashLocale)
for (const normalizedAS of [asNoSlash, asNoSlashLocale]) { for (const normalizedAS of [asNoSlash, asNoSlashLocale]) {
// if any sub-path of as matches a dynamic filter path // if any sub-path of as matches a dynamic filter path
...@@ -1095,7 +1095,7 @@ export default class Router implements BaseRouter { ...@@ -1095,7 +1095,7 @@ export default class Router implements BaseRouter {
i++ i++
) { ) {
const currentPart = curAsParts.slice(0, i).join('/') const currentPart = curAsParts.slice(0, i).join('/')
if (currentPart && this._bfl_d?.has(currentPart)) { if (currentPart && this._bfl_d?.contains(currentPart)) {
matchesBflDynamic = true matchesBflDynamic = true
break break
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment