diff --git a/README.md b/README.md index 659eacd..3003433 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ for example for many large filters. | Type | Members | | --- | --- | -| `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `buildFromBytes(Collection)` | +| `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `build(long[], int)`, `buildFromBytes(Collection)` | | `BinaryFuseFilter` (interface) | `contains(long)`, `contains(byte[])`, `contains(byte[], int, int)`, `contains(ByteBuffer)`, `fingerprintBits()`, `keyCount()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()`; static `wrap(ByteBuffer)`, `load(ByteBuffer)`, `load(byte[])` | | `WyHash` | `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)`, `hash64(long, long)` | | `InvalidFilterException` | thrown for corrupt or invalid serialized input | diff --git a/pom.xml b/pom.xml index 106763a..0e80fd4 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ scm:git:https://github.com/FastFilter/jbinfuse.git - scm:git:https://github.com/FastFilter/jbinfuse.git + https://github.com/FastFilter/jbinfuse scm:git:https://github.com/FastFilter/jbinfuse.git HEAD @@ -53,6 +53,9 @@ Sonatype Central Portal https://central.sonatype.com/repository/maven-snapshots/ + sonatype-central-portal Sonatype Central Portal diff --git a/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse16.java b/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse16.java index bdc45d4..b01f0b1 100644 --- a/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse16.java +++ b/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse16.java @@ -39,12 +39,28 @@ private BinaryFuse16(long seed, int keyCount, Fuse.Layout layout, short[] finger * @throws IllegalArgumentException if there are too many keys for one filter */ public static BinaryFuse16 build(long[] keys) { - Fuse.Layout layout = Fuse.Layout.forSize(keys.length, 16); + return build(keys, keys.length); + } + + /** + * Builds a filter containing the first {@code length} keys of the array, for example when + * the array was filled only partially. Otherwise identical to {@link #build(long[])}: the + * array is not modified and duplicate keys are allowed. + * + * @param keys the array holding the keys + * @param length the number of keys, from index 0 + * @return the filter + * @throws IndexOutOfBoundsException if {@code length} is negative or larger than the array + * @throws IllegalArgumentException if there are too many keys for one filter + */ + public static BinaryFuse16 build(long[] keys, int length) { + java.util.Objects.checkFromIndexSize(0, length, keys.length); + Fuse.Layout layout = Fuse.Layout.forSize(length, 16); short[] fp = new short[layout.arrayLength()]; - if (keys.length == 0) { + if (length == 0) { return new BinaryFuse16(0, 0, layout, fp); } - Fuse.Peeled p = Fuse.peel(keys, layout); + Fuse.Peeled p = Fuse.peel(keys, length, layout); final long[] order = p.reverseOrder(); final byte[] which = p.reverseH(); final int len = layout.segmentLength(); diff --git a/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse8.java b/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse8.java index 212d1e0..df00426 100644 --- a/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse8.java +++ b/src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse8.java @@ -38,12 +38,28 @@ private BinaryFuse8(long seed, int keyCount, Fuse.Layout layout, byte[] fingerpr * @throws IllegalArgumentException if there are too many keys for one filter */ public static BinaryFuse8 build(long[] keys) { - Fuse.Layout layout = Fuse.Layout.forSize(keys.length, 8); + return build(keys, keys.length); + } + + /** + * Builds a filter containing the first {@code length} keys of the array, for example when + * the array was filled only partially. Otherwise identical to {@link #build(long[])}: the + * array is not modified and duplicate keys are allowed. + * + * @param keys the array holding the keys + * @param length the number of keys, from index 0 + * @return the filter + * @throws IndexOutOfBoundsException if {@code length} is negative or larger than the array + * @throws IllegalArgumentException if there are too many keys for one filter + */ + public static BinaryFuse8 build(long[] keys, int length) { + java.util.Objects.checkFromIndexSize(0, length, keys.length); + Fuse.Layout layout = Fuse.Layout.forSize(length, 8); byte[] fp = new byte[layout.arrayLength()]; - if (keys.length == 0) { + if (length == 0) { return new BinaryFuse8(0, 0, layout, fp); } - Fuse.Peeled p = Fuse.peel(keys, layout); + Fuse.Peeled p = Fuse.peel(keys, length, layout); final long[] order = p.reverseOrder(); final byte[] which = p.reverseH(); final int len = layout.segmentLength(); diff --git a/src/main/java/io/github/fastfilter/jbinfuse/Fuse.java b/src/main/java/io/github/fastfilter/jbinfuse/Fuse.java index 8e4628b..8917bb2 100644 --- a/src/main/java/io/github/fastfilter/jbinfuse/Fuse.java +++ b/src/main/java/io/github/fastfilter/jbinfuse/Fuse.java @@ -66,7 +66,16 @@ static Layout forSize(int size, int bits) { // ------------------------------------------------------------- hashing - /** Hashes a 64-bit key with the filter seed. */ + /** + * Hashes a 64-bit key with the filter seed. This function is part of serialization format + * version 1: changing it would make existing filters return false negatives. + * + *

{@code WyHash.hash64(a, b)} multiplies {@code a ^ P0} by {@code b ^ P1}, so it has two + * degenerate inputs. The key {@code P0} hashes to the same value for every seed, which is + * harmless: a single key cannot make construction fail. A seed equal to {@code P1} would hash + * every key to the same value; the seeds come from a fixed sequence, and a test checks that its + * first {@link #MAX_ATTEMPTS} values are all different from {@code P1}. + */ static long hash(long key, long seed) { return WyHash.hash64(key, seed); } @@ -78,9 +87,9 @@ static long splitmix64(long state) { return z ^ (z >>> 31); } - private static final long SPLITMIX_GAMMA = 0x9e3779b97f4a7c15L; + static final long SPLITMIX_GAMMA = 0x9e3779b97f4a7c15L; /** Initial RNG state, as in the C reference. */ - private static final long RNG_START = 0x726b2b9d438b9d4dL; + static final long RNG_START = 0x726b2b9d438b9d4dL; // -------------------------------------------------------- construction @@ -91,12 +100,11 @@ static long splitmix64(long state) { record Peeled(long seed, int size, long[] reverseOrder, byte[] reverseH, int attempts) {} /** - * Runs the binary fuse construction (hash, bucket by segment, peel) over the keys. The input - * array is not modified. Keys are not sorted unless an attempt fails, in which case a sorted, - * deduplicated copy is used for the remaining attempts. + * Runs the binary fuse construction (hash, bucket by segment, peel) over the first {@code + * size} keys. The input array is not modified. Keys are not sorted unless an attempt fails, in + * which case a sorted, deduplicated copy is used for the remaining attempts. */ - static Peeled peel(long[] keys, Layout layout) { - int size = keys.length; + static Peeled peel(long[] keys, int size, Layout layout) { final int capacity = layout.arrayLength(); final int segmentLength = layout.segmentLength(); final int mask = layout.segmentLengthMask(); @@ -121,7 +129,7 @@ static Peeled peel(long[] keys, Layout layout) { final long seed = splitmix64(rng); if (attempt > 0) { if (!deduplicated) { - keys = sortedDistinct(keys); + keys = sortedDistinct(keys, size); deduplicated = true; size = keys.length; reverseOrder[size] = 1; @@ -232,9 +240,9 @@ static Peeled peel(long[] keys, Layout layout) { "binary fuse construction failed after " + MAX_ATTEMPTS + " attempts"); } - /** Returns a sorted copy of the keys without duplicates. */ - static long[] sortedDistinct(long[] keys) { - long[] copy = keys.clone(); + /** Returns a sorted copy of the first {@code size} keys, without duplicates. */ + static long[] sortedDistinct(long[] keys, int size) { + long[] copy = Arrays.copyOf(keys, size); Arrays.sort(copy); int n = 0; for (int i = 0; i < copy.length; i++) { diff --git a/src/test/java/io/github/fastfilter/jbinfuse/BinaryFuseFilterTest.java b/src/test/java/io/github/fastfilter/jbinfuse/BinaryFuseFilterTest.java index 110f9a1..bdaa425 100644 --- a/src/test/java/io/github/fastfilter/jbinfuse/BinaryFuseFilterTest.java +++ b/src/test/java/io/github/fastfilter/jbinfuse/BinaryFuseFilterTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -446,12 +447,12 @@ void serializeRejectsSmallOrReadOnlyBuffers() { void fewConstructionAttempts() { int worst = 0; for (int n = 1; n <= 3000; n++) { - worst = Math.max(worst, Fuse.peel(randomKeys(n, ~n), Fuse.Layout.forSize(n, 8)).attempts()); + worst = Math.max(worst, Fuse.peel(randomKeys(n, ~n), n, Fuse.Layout.forSize(n, 8)).attempts()); } for (int n : new int[] {10_000, 100_000, 1_000_000}) { for (int s = 0; s < 3; s++) { worst = Math.max(worst, - Fuse.peel(randomKeys(n, n + s), Fuse.Layout.forSize(n, 8)).attempts()); + Fuse.peel(randomKeys(n, n + s), n, Fuse.Layout.forSize(n, 8)).attempts()); } } assertTrue(worst <= 4, "needed " + worst + " attempts"); @@ -483,4 +484,44 @@ void viewDoesNotDependOnBufferState() { buf.position(100).limit(200).order(ByteOrder.LITTLE_ENDIAN); assertSameAnswers(f, view, 50_000, 3); } + + @Test + void buildFromPrefixOfArray() { + long[] keys = randomKeys(10_000, 7); + for (int length : new int[] {0, 1, 2, 999, 10_000}) { + long[] prefix = java.util.Arrays.copyOf(keys, length); + long[] before = keys.clone(); + assertArrayEquals( + BinaryFuse8.build(prefix).toByteArray(), BinaryFuse8.build(keys, length).toByteArray()); + assertArrayEquals( + BinaryFuse16.build(prefix).toByteArray(), BinaryFuse16.build(keys, length).toByteArray()); + assertArrayEquals(before, keys); + } + assertThrows(IndexOutOfBoundsException.class, () -> BinaryFuse8.build(keys, -1)); + assertThrows(IndexOutOfBoundsException.class, () -> BinaryFuse16.build(keys, keys.length + 1)); + } + + @Test + void seedSequenceAvoidsDegenerateSeed() { + // WyHash.hash64(key, seed) maps every key to the same value if seed == P1. + long p1 = 0x8bb84b93962eacc9L; + long rng = Fuse.RNG_START; + for (int attempt = 0; attempt < Fuse.MAX_ATTEMPTS; attempt++) { + rng += Fuse.SPLITMIX_GAMMA; + assertNotEquals(p1, Fuse.splitmix64(rng)); + } + } + + @Test + void degenerateKeyIsHarmless() { + // WyHash.hash64(P0, seed) does not depend on the seed. + long p0 = 0x2d358dccaa6c78a5L; + long[] keys = randomKeys(1000, 3); + keys[500] = p0; + for (BinaryFuseFilter f : new BinaryFuseFilter[] {BinaryFuse8.build(keys), BinaryFuse16.build(keys)}) { + for (long key : keys) { + assertTrue(f.contains(key)); + } + } + } }