Add zero-copy views of constant maps over a ByteBuffer

README.md

@@ -208,11 +208,38 @@ VerifiedConstMap fromPython = VerifiedConstMap.load(Files.readAllBytes(Path.of("
 - `load` validates the magic bytes, the segment parameters and the FNV-1a checksum, and throws
   `InvalidFilterException` otherwise, including for a file of another kind of map.
 
-The constant maps are always loaded onto the heap; there is no in-place view as for filters.
+### Querying a constant map in place
+
+As with filters, `wrap(ByteBuffer)` returns a view that queries a serialized map in place,
+without copying it, from a heap, direct or memory-mapped buffer. It reads every format above,
+written by any of the four implementations:
+
+```java
+try (FileChannel ch = FileChannel.open(Path.of("built-by-go.vmap"), StandardOpenOption.READ)) {
+  MappedByteBuffer mapped = ch.map(FileChannel.MapMode.READ_ONLY, 0, ch.size());
+  VerifiedConstMap view = VerifiedConstMap.wrap(mapped);   // constant time, no copy
+  view.get("banana");
+}
+```
+
+- Views give the same answers as on-heap maps, including `getMany` and `paired()` (which
+  returns a new on-heap map). This is tested on the files written by rsconstmap and
+  fastconstmap, at every alignment, in heap, direct and memory-mapped buffers.
+- There is no alignment requirement. A Go or Rust `ConstMap` file (`CMAP0001`), whose 28-byte
+  header leaves the array 4 bytes off an 8-byte boundary, can be wrapped too, at any offset;
+  fastconstmap's `from_buffer` refuses those files.
+- `wrap` validates the header in constant time (magic bytes, segment parameters consistent
+  with the array length, and a buffer long enough for the whole map) but does not verify the
+  checksum, which would read the whole map; `load` does. On corrupt bytes, a view gives wrong
+  answers but never reads outside the map.
+- A view serializes to the bytes it wraps, unchanged (so a `CMAP0003` file from fastconstmap
+  stays one, and keeps its key count). On-heap maps write the formats of Go and Rust.
+- The view is valid as long as the buffer's bytes are not modified. The buffer's position,
+  limit and byte order are not changed.
 
 ### Constant map performance
 
-JMH (`ConstMapBenchmark`) on an Apple M4 Max with JDK 27, ns per lookup of a present key, in
+JMH (`ConstMapBenchmark`, on-heap maps) on an Apple M4 Max with JDK 27, ns per lookup of a present key, in
 random order, with keys like `key-123456-1e3779b9`:
 
 | | 1 million keys | 10 million keys |
@@ -234,9 +261,9 @@ straight from their chars; other strings are first encoded to UTF-8, which costs
 | --- | --- |
 | `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `build(long[], int)`, `buildFromBytes(Collection<byte[]>)` |
 | `BinaryFuseFilter` (interface) | `contains(long)`, `contains(byte[])`, `contains(byte[], int, int)`, `contains(ByteBuffer)`, `fingerprintBits()`, `keyCount()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()`; static `wrap(ByteBuffer)`, `load(ByteBuffer)`, `load(byte[])` |
-| `ConstMap` | static `build(List<String>, long[])`, `buildFromBytes(List<byte[]>, long[])`, `buildFromHashes(long[], long[])`, `load(ByteBuffer)`, `load(byte[])`; `get(String)`, `get(byte[])`, `get(byte[], int, int)`, `get(ByteBuffer)`, `getByHash(long)`, `getMany(List<String>)`, `getMany(List<String>, long[])`, `arrayLength()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()` |
-| `VerifiedConstMap` | as `ConstMap`, plus `containsKey(String)`, `paired()` and the constant `NOT_FOUND` |
-| `PairedVerifiedConstMap` | as `ConstMap`, plus `containsKey(String)` |
+| `ConstMap` (interface) | static `build(List<String>, long[])`, `buildFromBytes(List<byte[]>, long[])`, `buildFromHashes(long[], long[])`, `load(ByteBuffer)`, `load(byte[])`, `wrap(ByteBuffer)`; `get(String)`, `get(byte[])`, `get(byte[], int, int)`, `get(ByteBuffer)`, `getByHash(long)`, `getMany(List<String>)`, `getMany(List<String>, long[])`, `arrayLength()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()` |
+| `VerifiedConstMap` (interface) | as `ConstMap`, plus `containsKey(String)`, `paired()` and the constant `NOT_FOUND` |
+| `PairedVerifiedConstMap` (interface) | as `ConstMap`, plus `containsKey(String)` |
 | `XxHash64` | `hash(String)`, `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)` |
 | `WyHash` | `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)`, `hash64(long, long)` |
 | `InvalidFilterException` | thrown for corrupt or invalid serialized input |

benchmarks/src/main/java/io/github/fastfilter/jbinfuse/bench/ConstMapBenchmark.java

@@ -3,6 +3,12 @@
 import io.github.fastfilter.jbinfuse.ConstMap;
 import io.github.fastfilter.jbinfuse.PairedVerifiedConstMap;
 import io.github.fastfilter.jbinfuse.VerifiedConstMap;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.SplittableRandom;
@@ -41,14 +47,21 @@ public class ConstMapBenchmark {
   @Param({"const", "verified", "paired"})
   String kind;
 
+  /**
+   * Where the map lives: on the heap, or a view ({@code wrap}) of its serialized bytes in a
+   * direct buffer or a memory-mapped temporary file.
+   */
+  @Param({"heap", "direct", "mapped"})
+  String storage;
+
   ConstMap cm;
   VerifiedConstMap vm;
   PairedVerifiedConstMap pm;
   List<String> queries;
   long[] out;
 
   @Setup
-  public void setup() {
+  public void setup() throws IOException {
     List<String> keys = new ArrayList<>(size);
     long[] values = new long[size];
     for (int i = 0; i < size; i++) {
@@ -60,6 +73,30 @@ public void setup() {
       case "verified" -> vm = VerifiedConstMap.build(keys, values);
       default -> pm = PairedVerifiedConstMap.build(keys, values);
     }
+    if (!storage.equals("heap")) {
+      byte[] bytes = switch (kind) {
+        case "const" -> cm.toByteArray();
+        case "verified" -> vm.toByteArray();
+        default -> pm.toByteArray();
+      };
+      ByteBuffer buf;
+      if (storage.equals("direct")) {
+        buf = ByteBuffer.allocateDirect(bytes.length);
+        buf.put(bytes).flip();
+      } else {
+        Path file = Files.createTempFile("constmap", ".bin");
+        file.toFile().deleteOnExit();
+        Files.write(file, bytes);
+        try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) {
+          buf = ch.map(FileChannel.MapMode.READ_ONLY, 0, ch.size());
+        }
+      }
+      switch (kind) {
+        case "const" -> cm = ConstMap.wrap(buf);
+        case "verified" -> vm = VerifiedConstMap.wrap(buf);
+        default -> pm = PairedVerifiedConstMap.wrap(buf);
+      }
+    }
     SplittableRandom r = new SplittableRandom(size);
     queries = new ArrayList<>(QUERIES);
     for (int i = 0; i < QUERIES; i++) {

src/main/java/io/github/fastfilter/jbinfuse/BufferConstMap.java

@@ -0,0 +1,90 @@
+package io.github.fastfilter.jbinfuse;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+/** A view of a serialized {@link ConstMap}, queried in place. Created by {@link ConstMap#wrap}. */
+final class BufferConstMap implements ConstMap {
+  private final ByteBuffer serialized;
+  private final ByteBuffer data;
+  private final long seed;
+  private final int segmentLength;
+  private final int segmentLengthMask;
+  private final long segmentCountLength;
+  private final int arrayLength;
+
+  private BufferConstMap(ByteBuffer buffer, ConstMaps.Header h) {
+    ConstMaps.Params p = h.params();
+    this.serialized = ConstMaps.slice(buffer, h.start(), h.totalBytes());
+    this.data = ConstMaps.slice(buffer, h.arrayStart(), 8 * p.arrayLength());
+    this.seed = p.seed();
+    this.segmentLength = p.segmentLength();
+    this.segmentLengthMask = p.segmentLengthMask();
+    this.segmentCountLength = p.segmentCountLength();
+    this.arrayLength = p.arrayLength();
+  }
+
+  static BufferConstMap wrap(ByteBuffer buffer) {
+    return new BufferConstMap(buffer, ConstMaps.readHeader(buffer, ConstMaps.Kind.CONST, 1));
+  }
+
+  @Override
+  public long getByHash(long keyHash) {
+    if (arrayLength == 0) {
+      return 0;
+    }
+    long hash = ConstMaps.mixsplit(keyHash, seed);
+    int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength);
+    int h1 = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask);
+    int h2 = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask);
+    final ByteBuffer d = data;
+    return d.getLong(8 * h0) ^ d.getLong(8 * h1) ^ d.getLong(8 * h2);
+  }
+
+  @Override
+  public void getMany(List<String> keys, long[] dst) {
+    int n = keys.size();
+    Objects.checkFromIndexSize(0, n, dst.length);
+    if (arrayLength == 0) {
+      Arrays.fill(dst, 0, n, 0);
+      return;
+    }
+    final ByteBuffer d = data;
+    final int[] h = new int[3 * ConstMaps.BLOCK];
+    for (int start = 0; start < n; start += ConstMaps.BLOCK) {
+      int m = Math.min(ConstMaps.BLOCK, n - start);
+      for (int j = 0; j < m; j++) {
+        long hash = ConstMaps.mixsplit(XxHash64.hash(keys.get(start + j)), seed);
+        int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength);
+        h[3 * j] = 8 * h0;
+        h[3 * j + 1] = 8 * ((h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask));
+        h[3 * j + 2] = 8 * ((h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask));
+      }
+      for (int j = 0; j < m; j++) {
+        dst[start + j] = d.getLong(h[3 * j]) ^ d.getLong(h[3 * j + 1]) ^ d.getLong(h[3 * j + 2]);
+      }
+    }
+  }
+
+  @Override
+  public int arrayLength() {
+    return arrayLength;
+  }
+
+  @Override
+  public int serializedSize() {
+    return serialized.capacity();
+  }
+
+  @Override
+  public void serialize(ByteBuffer out) {
+    ConstMaps.copy(serialized, out);
+  }
+
+  @Override
+  public String toString() {
+    return "ConstMapView{words=" + arrayLength + ", bytes=" + serializedSize() + "}";
+  }
+}

src/main/java/io/github/fastfilter/jbinfuse/BufferPairedVerifiedConstMap.java

@@ -0,0 +1,110 @@
+package io.github.fastfilter.jbinfuse;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A view of a serialized {@link PairedVerifiedConstMap}, queried in place. Created by
+ * {@link PairedVerifiedConstMap#wrap}.
+ */
+final class BufferPairedVerifiedConstMap implements PairedVerifiedConstMap {
+  private final ByteBuffer serialized;
+  /** The value for position i at byte 16i, its check word at byte 16i + 8. */
+  private final ByteBuffer slots;
+  private final long seed;
+  private final int segmentLength;
+  private final int segmentLengthMask;
+  private final long segmentCountLength;
+  private final int arrayLength;
+
+  private BufferPairedVerifiedConstMap(ByteBuffer buffer, ConstMaps.Header h) {
+    ConstMaps.Params p = h.params();
+    this.serialized = ConstMaps.slice(buffer, h.start(), h.totalBytes());
+    this.slots = ConstMaps.slice(buffer, h.arrayStart(), 16 * p.arrayLength());
+    this.seed = p.seed();
+    this.segmentLength = p.segmentLength();
+    this.segmentLengthMask = p.segmentLengthMask();
+    this.segmentCountLength = p.segmentCountLength();
+    this.arrayLength = p.arrayLength();
+  }
+
+  static BufferPairedVerifiedConstMap wrap(ByteBuffer buffer) {
+    return new BufferPairedVerifiedConstMap(buffer,
+        ConstMaps.readHeader(buffer, ConstMaps.Kind.PAIRED, 2));
+  }
+
+  @Override
+  public long getByHash(long keyHash) {
+    if (arrayLength == 0) {
+      return ConstMaps.NOT_FOUND;
+    }
+    long hash = ConstMaps.mixsplit(keyHash, seed);
+    int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength);
+    int h1 = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask);
+    int h2 = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask);
+    final ByteBuffer s = slots;
+    int a = 16 * h0;
+    int b = 16 * h1;
+    int c = 16 * h2;
+    if ((s.getLong(a + 8) ^ s.getLong(b + 8) ^ s.getLong(c + 8))
+        != ConstMaps.fingerprint(hash)) {
+      return ConstMaps.NOT_FOUND;
+    }
+    return s.getLong(a) ^ s.getLong(b) ^ s.getLong(c);
+  }
+
+  @Override
+  public void getMany(List<String> keys, long[] dst) {
+    int n = keys.size();
+    Objects.checkFromIndexSize(0, n, dst.length);
+    if (arrayLength == 0) {
+      Arrays.fill(dst, 0, n, ConstMaps.NOT_FOUND);
+      return;
+    }
+    final ByteBuffer s = slots;
+    final int[] h = new int[3 * ConstMaps.BLOCK];
+    final long[] fp = new long[ConstMaps.BLOCK];
+    for (int start = 0; start < n; start += ConstMaps.BLOCK) {
+      int m = Math.min(ConstMaps.BLOCK, n - start);
+      for (int j = 0; j < m; j++) {
+        long hash = ConstMaps.mixsplit(XxHash64.hash(keys.get(start + j)), seed);
+        int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength);
+        h[3 * j] = 16 * h0;
+        h[3 * j + 1] = 16 * ((h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask));
+        h[3 * j + 2] = 16 * ((h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask));
+        fp[j] = ConstMaps.fingerprint(hash);
+      }
+      for (int j = 0; j < m; j++) {
+        int a = h[3 * j];
+        int b = h[3 * j + 1];
+        int c = h[3 * j + 2];
+        long v = s.getLong(a) ^ s.getLong(b) ^ s.getLong(c);
+        dst[start + j] = (s.getLong(a + 8) ^ s.getLong(b + 8) ^ s.getLong(c + 8)) == fp[j]
+            ? v : ConstMaps.NOT_FOUND;
+      }
+    }
+  }
+
+  @Override
+  public int arrayLength() {
+    return arrayLength;
+  }
+
+  @Override
+  public int serializedSize() {
+    return serialized.capacity();
+  }
+
+  @Override
+  public void serialize(ByteBuffer out) {
+    ConstMaps.copy(serialized, out);
+  }
+
+  @Override
+  public String toString() {
+    return "PairedVerifiedConstMapView{positions=" + arrayLength + ", bytes=" + serializedSize()
+        + "}";
+  }
+}