Add build(long[], int); document hash edge cases; POM fixes

README.md

@@ -140,7 +140,7 @@ for example for many large filters.
 
 | Type | Members |
 | --- | --- |
-| `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `buildFromBytes(Collection<byte[]>)` |
+| `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `build(long[], int)`, `buildFromBytes(Collection<byte[]>)` |
 | `BinaryFuseFilter` (interface) | `contains(long)`, `contains(byte[])`, `contains(byte[], int, int)`, `contains(ByteBuffer)`, `fingerprintBits()`, `keyCount()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()`; static `wrap(ByteBuffer)`, `load(ByteBuffer)`, `load(byte[])` |
 | `WyHash` | `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)`, `hash64(long, long)` |
 | `InvalidFilterException` | thrown for corrupt or invalid serialized input |

pom.xml

@@ -37,7 +37,7 @@
 
   <scm>
     <connection>scm:git:https://github.com/FastFilter/jbinfuse.git</connection>
-    <url>scm:git:https://github.com/FastFilter/jbinfuse.git</url>
+    <url>https://github.com/FastFilter/jbinfuse</url>
     <developerConnection>scm:git:https://github.com/FastFilter/jbinfuse.git</developerConnection>
     <tag>HEAD</tag>
   </scm>
@@ -53,6 +53,9 @@
       <name>Sonatype Central Portal</name>
       <url>https://central.sonatype.com/repository/maven-snapshots/</url>
     </snapshotRepository>
+    <!-- Releases are deployed through Njord (see .github/workflows/release.yml), which uses this
+         repository's id to find the Central Portal publisher in the settings. Following Njord's
+         convention, the URL is where released artifacts end up: Maven Central. -->
     <repository>
       <id>sonatype-central-portal</id>
       <name>Sonatype Central Portal</name>

src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse16.java

@@ -39,12 +39,28 @@ private BinaryFuse16(long seed, int keyCount, Fuse.Layout layout, short[] finger
    * @throws IllegalArgumentException if there are too many keys for one filter
    */
   public static BinaryFuse16 build(long[] keys) {
-    Fuse.Layout layout = Fuse.Layout.forSize(keys.length, 16);
+    return build(keys, keys.length);
+  }
+
+  /**
+   * Builds a filter containing the first {@code length} keys of the array, for example when
+   * the array was filled only partially. Otherwise identical to {@link #build(long[])}: the
+   * array is not modified and duplicate keys are allowed.
+   *
+   * @param keys the array holding the keys
+   * @param length the number of keys, from index 0
+   * @return the filter
+   * @throws IndexOutOfBoundsException if {@code length} is negative or larger than the array
+   * @throws IllegalArgumentException if there are too many keys for one filter
+   */
+  public static BinaryFuse16 build(long[] keys, int length) {
+    java.util.Objects.checkFromIndexSize(0, length, keys.length);
+    Fuse.Layout layout = Fuse.Layout.forSize(length, 16);
     short[] fp = new short[layout.arrayLength()];
-    if (keys.length == 0) {
+    if (length == 0) {
       return new BinaryFuse16(0, 0, layout, fp);
     }
-    Fuse.Peeled p = Fuse.peel(keys, layout);
+    Fuse.Peeled p = Fuse.peel(keys, length, layout);
     final long[] order = p.reverseOrder();
     final byte[] which = p.reverseH();
     final int len = layout.segmentLength();

src/main/java/io/github/fastfilter/jbinfuse/BinaryFuse8.java

@@ -38,12 +38,28 @@ private BinaryFuse8(long seed, int keyCount, Fuse.Layout layout, byte[] fingerpr
    * @throws IllegalArgumentException if there are too many keys for one filter
    */
   public static BinaryFuse8 build(long[] keys) {
-    Fuse.Layout layout = Fuse.Layout.forSize(keys.length, 8);
+    return build(keys, keys.length);
+  }
+
+  /**
+   * Builds a filter containing the first {@code length} keys of the array, for example when
+   * the array was filled only partially. Otherwise identical to {@link #build(long[])}: the
+   * array is not modified and duplicate keys are allowed.
+   *
+   * @param keys the array holding the keys
+   * @param length the number of keys, from index 0
+   * @return the filter
+   * @throws IndexOutOfBoundsException if {@code length} is negative or larger than the array
+   * @throws IllegalArgumentException if there are too many keys for one filter
+   */
+  public static BinaryFuse8 build(long[] keys, int length) {
+    java.util.Objects.checkFromIndexSize(0, length, keys.length);
+    Fuse.Layout layout = Fuse.Layout.forSize(length, 8);
     byte[] fp = new byte[layout.arrayLength()];
-    if (keys.length == 0) {
+    if (length == 0) {
       return new BinaryFuse8(0, 0, layout, fp);
     }
-    Fuse.Peeled p = Fuse.peel(keys, layout);
+    Fuse.Peeled p = Fuse.peel(keys, length, layout);
     final long[] order = p.reverseOrder();
     final byte[] which = p.reverseH();
     final int len = layout.segmentLength();

src/main/java/io/github/fastfilter/jbinfuse/Fuse.java

@@ -66,7 +66,16 @@ static Layout forSize(int size, int bits) {
 
   // ------------------------------------------------------------- hashing
 
-  /** Hashes a 64-bit key with the filter seed. */
+  /**
+   * Hashes a 64-bit key with the filter seed. This function is part of serialization format
+   * version 1: changing it would make existing filters return false negatives.
+   *
+   * <p>{@code WyHash.hash64(a, b)} multiplies {@code a ^ P0} by {@code b ^ P1}, so it has two
+   * degenerate inputs. The key {@code P0} hashes to the same value for every seed, which is
+   * harmless: a single key cannot make construction fail. A seed equal to {@code P1} would hash
+   * every key to the same value; the seeds come from a fixed sequence, and a test checks that its
+   * first {@link #MAX_ATTEMPTS} values are all different from {@code P1}.
+   */
   static long hash(long key, long seed) {
     return WyHash.hash64(key, seed);
   }
@@ -78,9 +87,9 @@ static long splitmix64(long state) {
     return z ^ (z >>> 31);
   }
 
-  private static final long SPLITMIX_GAMMA = 0x9e3779b97f4a7c15L;
+  static final long SPLITMIX_GAMMA = 0x9e3779b97f4a7c15L;
   /** Initial RNG state, as in the C reference. */
-  private static final long RNG_START = 0x726b2b9d438b9d4dL;
+  static final long RNG_START = 0x726b2b9d438b9d4dL;
 
   // -------------------------------------------------------- construction
 
@@ -91,12 +100,11 @@ static long splitmix64(long state) {
   record Peeled(long seed, int size, long[] reverseOrder, byte[] reverseH, int attempts) {}
 
   /**
-   * Runs the binary fuse construction (hash, bucket by segment, peel) over the keys. The input
-   * array is not modified. Keys are not sorted unless an attempt fails, in which case a sorted,
-   * deduplicated copy is used for the remaining attempts.
+   * Runs the binary fuse construction (hash, bucket by segment, peel) over the first {@code
+   * size} keys. The input array is not modified. Keys are not sorted unless an attempt fails, in
+   * which case a sorted, deduplicated copy is used for the remaining attempts.
    */
-  static Peeled peel(long[] keys, Layout layout) {
-    int size = keys.length;
+  static Peeled peel(long[] keys, int size, Layout layout) {
     final int capacity = layout.arrayLength();
     final int segmentLength = layout.segmentLength();
     final int mask = layout.segmentLengthMask();
@@ -121,7 +129,7 @@ static Peeled peel(long[] keys, Layout layout) {
       final long seed = splitmix64(rng);
       if (attempt > 0) {
         if (!deduplicated) {
-          keys = sortedDistinct(keys);
+          keys = sortedDistinct(keys, size);
           deduplicated = true;
           size = keys.length;
           reverseOrder[size] = 1;
@@ -232,9 +240,9 @@ static Peeled peel(long[] keys, Layout layout) {
         "binary fuse construction failed after " + MAX_ATTEMPTS + " attempts");
   }
 
-  /** Returns a sorted copy of the keys without duplicates. */
-  static long[] sortedDistinct(long[] keys) {
-    long[] copy = keys.clone();
+  /** Returns a sorted copy of the first {@code size} keys, without duplicates. */
+  static long[] sortedDistinct(long[] keys, int size) {
+    long[] copy = Arrays.copyOf(keys, size);
     Arrays.sort(copy);
     int n = 0;
     for (int i = 0; i < copy.length; i++) {

src/test/java/io/github/fastfilter/jbinfuse/BinaryFuseFilterTest.java

@@ -3,6 +3,7 @@
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -446,12 +447,12 @@ void serializeRejectsSmallOrReadOnlyBuffers() {
   void fewConstructionAttempts() {
     int worst = 0;
     for (int n = 1; n <= 3000; n++) {
-      worst = Math.max(worst, Fuse.peel(randomKeys(n, ~n), Fuse.Layout.forSize(n, 8)).attempts());
+      worst = Math.max(worst, Fuse.peel(randomKeys(n, ~n), n, Fuse.Layout.forSize(n, 8)).attempts());
     }
     for (int n : new int[] {10_000, 100_000, 1_000_000}) {
       for (int s = 0; s < 3; s++) {
         worst = Math.max(worst,
-            Fuse.peel(randomKeys(n, n + s), Fuse.Layout.forSize(n, 8)).attempts());
+            Fuse.peel(randomKeys(n, n + s), n, Fuse.Layout.forSize(n, 8)).attempts());
       }
     }
     assertTrue(worst <= 4, "needed " + worst + " attempts");
@@ -483,4 +484,44 @@ void viewDoesNotDependOnBufferState() {
     buf.position(100).limit(200).order(ByteOrder.LITTLE_ENDIAN);
     assertSameAnswers(f, view, 50_000, 3);
   }
+
+  @Test
+  void buildFromPrefixOfArray() {
+    long[] keys = randomKeys(10_000, 7);
+    for (int length : new int[] {0, 1, 2, 999, 10_000}) {
+      long[] prefix = java.util.Arrays.copyOf(keys, length);
+      long[] before = keys.clone();
+      assertArrayEquals(
+          BinaryFuse8.build(prefix).toByteArray(), BinaryFuse8.build(keys, length).toByteArray());
+      assertArrayEquals(
+          BinaryFuse16.build(prefix).toByteArray(), BinaryFuse16.build(keys, length).toByteArray());
+      assertArrayEquals(before, keys);
+    }
+    assertThrows(IndexOutOfBoundsException.class, () -> BinaryFuse8.build(keys, -1));
+    assertThrows(IndexOutOfBoundsException.class, () -> BinaryFuse16.build(keys, keys.length + 1));
+  }
+
+  @Test
+  void seedSequenceAvoidsDegenerateSeed() {
+    // WyHash.hash64(key, seed) maps every key to the same value if seed == P1.
+    long p1 = 0x8bb84b93962eacc9L;
+    long rng = Fuse.RNG_START;
+    for (int attempt = 0; attempt < Fuse.MAX_ATTEMPTS; attempt++) {
+      rng += Fuse.SPLITMIX_GAMMA;
+      assertNotEquals(p1, Fuse.splitmix64(rng));
+    }
+  }
+
+  @Test
+  void degenerateKeyIsHarmless() {
+    // WyHash.hash64(P0, seed) does not depend on the seed.
+    long p0 = 0x2d358dccaa6c78a5L;
+    long[] keys = randomKeys(1000, 3);
+    keys[500] = p0;
+    for (BinaryFuseFilter f : new BinaryFuseFilter[] {BinaryFuse8.build(keys), BinaryFuse16.build(keys)}) {
+      for (long key : keys) {
+        assertTrue(f.contains(key));
+      }
+    }
+  }
 }