From c30faa22f5dbc821437cbfbfe1c89d781dc12a42 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 3 Nov 2026 23:08:37 -0500 Subject: [PATCH] Add constant maps compatible with constmap, rsconstmap and fastconstmap ConstMap, VerifiedConstMap and PairedVerifiedConstMap are immutable maps from string or byte-array keys to 64-bit values built with the binary fuse construction, ported from github.com/lemire/constmap (Go). They share the key hash (XXH64), mixing, seed sequence, layout and file formats with constmap, rsconstmap (Rust) and fastconstmap (C/Python): - for the same input they write the same bytes as Go and Rust (CMAP0001, VMAP0001, PMAP0001), checked against the interop files of the constmap repository and against the Go package for sizes from 0 to 1,234,567 keys (tools/constmap_vectors); - they read fastconstmap's CMAP0003 and ignore its key counts, and name the legacy fastconstmap 0.9 formats (XXH3) instead of misreading them; - load validates magic, segment parameters and the FNV-1a checksum. Also adds XxHash64, a dependency-free XXH64 checked against vectors from the reference C implementation (tools/xxh64_vectors.c), with an allocation-free path for ASCII strings, plus batched getMany lookups and a JMH benchmark. --- README.md | 104 ++- .../jbinfuse/bench/ConstMapBenchmark.java | 106 +++ pom.xml | 2 +- .../github/fastfilter/jbinfuse/ConstMap.java | 287 +++++++ .../github/fastfilter/jbinfuse/ConstMaps.java | 557 +++++++++++++ .../jbinfuse/InvalidFilterException.java | 6 +- .../jbinfuse/PairedVerifiedConstMap.java | 298 +++++++ .../fastfilter/jbinfuse/VerifiedConstMap.java | 329 ++++++++ .../github/fastfilter/jbinfuse/XxHash64.java | 321 ++++++++ .../fastfilter/jbinfuse/package-info.java | 7 + .../fastfilter/jbinfuse/ConstMapTest.java | 430 ++++++++++ .../fastfilter/jbinfuse/XxHash64Test.java | 115 +++ .../resources/constmap/fastconstmap-0.9.cmap | Bin 0 -> 2600 bytes .../resources/constmap/fastconstmap-0.9.vmap | Bin 0 -> 5160 bytes src/test/resources/constmap/fastconstmap.cmap | Bin 0 -> 2600 bytes src/test/resources/constmap/fastconstmap.pmap | Bin 0 -> 5160 bytes src/test/resources/constmap/fastconstmap.vmap | Bin 0 -> 5160 bytes src/test/resources/constmap/rust.cmap | Bin 0 -> 2596 bytes src/test/resources/constmap/rust.pmap | Bin 0 -> 5160 bytes src/test/resources/constmap/rust.vmap | Bin 0 -> 5160 bytes src/test/resources/constmap_vectors.txt | 163 ++++ src/test/resources/xxh64_vectors.txt | 748 ++++++++++++++++++ tools/constmap_vectors/go.mod | 7 + tools/constmap_vectors/go.sum | 4 + tools/constmap_vectors/main.go | 68 ++ tools/xxh64_vectors.c | 45 ++ 26 files changed, 3590 insertions(+), 7 deletions(-) create mode 100644 benchmarks/src/main/java/io/github/fastfilter/jbinfuse/bench/ConstMapBenchmark.java create mode 100644 src/main/java/io/github/fastfilter/jbinfuse/ConstMap.java create mode 100644 src/main/java/io/github/fastfilter/jbinfuse/ConstMaps.java create mode 100644 src/main/java/io/github/fastfilter/jbinfuse/PairedVerifiedConstMap.java create mode 100644 src/main/java/io/github/fastfilter/jbinfuse/VerifiedConstMap.java create mode 100644 src/main/java/io/github/fastfilter/jbinfuse/XxHash64.java create mode 100644 src/test/java/io/github/fastfilter/jbinfuse/ConstMapTest.java create mode 100644 src/test/java/io/github/fastfilter/jbinfuse/XxHash64Test.java create mode 100644 src/test/resources/constmap/fastconstmap-0.9.cmap create mode 100644 src/test/resources/constmap/fastconstmap-0.9.vmap create mode 100644 src/test/resources/constmap/fastconstmap.cmap create mode 100644 src/test/resources/constmap/fastconstmap.pmap create mode 100644 src/test/resources/constmap/fastconstmap.vmap create mode 100644 src/test/resources/constmap/rust.cmap create mode 100644 src/test/resources/constmap/rust.pmap create mode 100644 src/test/resources/constmap/rust.vmap create mode 100644 src/test/resources/constmap_vectors.txt create mode 100644 src/test/resources/xxh64_vectors.txt create mode 100644 tools/constmap_vectors/go.mod create mode 100644 tools/constmap_vectors/go.sum create mode 100644 tools/constmap_vectors/main.go create mode 100644 tools/xxh64_vectors.c diff --git a/README.md b/README.md index 3f43463..ee91593 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ filters for the same false-positive rate, and a query reads exactly three memory `BinaryFuse16`), the fastest; and **in-place views** of a serialized filter in a `ByteBuffer` (heap, direct, or a memory-mapped file), with near-zero heap use and instant open. See [Choosing a query mode](#choosing-a-query-mode). +- **Constant maps** from string or byte-array keys to 64-bit values, built on the same + construction (about 9 or 18 bytes per key), whose files are interchangeable with the Go, + Rust and C/Python implementations. See [Constant maps](#constant-maps). ## Maven @@ -139,16 +142,106 @@ misses on 4 KiB page-cache pages, whereas a large Java heap typically uses 2 MiB huge pages.) Choose `wrap` when heap memory or open time matters more than those nanoseconds, for example for many large filters. +## Constant maps + +The binary fuse construction also gives an immutable map from keys to 64-bit values that does +not store the keys: a lookup is one hash, three array reads and two XORs. + +```java +import io.github.fastfilter.jbinfuse.ConstMap; +import io.github.fastfilter.jbinfuse.VerifiedConstMap; + +List keys = List.of("apple", "banana", "cherry"); +long[] values = {100, 200, 300}; + +ConstMap cm = ConstMap.build(keys, values); // about 9 bytes per key +cm.get("banana"); // 200 +cm.get("grape"); // an arbitrary value! + +VerifiedConstMap vm = VerifiedConstMap.build(keys, values); // about 18 bytes per key +vm.get("banana"); // 200 +vm.get("grape"); // VerifiedConstMap.NOT_FOUND (-1) +vm.containsKey("grape"); // false + +long[] many = vm.getMany(List.of("apple", "grape")); // {100, NOT_FOUND} +``` + +| | `ConstMap` | `VerifiedConstMap` | `PairedVerifiedConstMap` | +| --- | --- | --- | --- | +| bytes per key | about 9 | about 18 | about 18 | +| missing key | arbitrary value | `NOT_FOUND` | `NOT_FOUND` | +| cache lines per lookup | 3 | 6 (3 for most misses) | 3 | +| best when | you only look up keys in the map | most lookups miss | most lookups hit | + +Keys must be distinct. String keys are hashed through their UTF-8 encoding with XXH64 +(`XxHash64`), so `get("abc")` and `get("abc".getBytes(UTF_8))` are the same lookup; there are +also `buildFromBytes(List, long[])`, `get(byte[], int, int)`, `get(ByteBuffer)`, and +`buildFromHashes`/`getByHash` taking XXH64 hashes you computed yourself. A verified map reports +a missing key as present with probability about 2-64; a key whose value is +`NOT_FOUND` cannot be told apart from a missing one. `vm.paired()` converts a +`VerifiedConstMap` to a `PairedVerifiedConstMap`. + +`getMany` hashes a block of keys before reading the array, so that the processor can overlap +the memory accesses of several lookups; it returns exactly what a loop over `get` would. + +### Interoperability with constmap, rsconstmap and fastconstmap + +These classes are a Java implementation of [constmap](https://github.com/lemire/constmap) (Go), +also implemented by [rsconstmap](https://github.com/lemire/rsconstmap) (Rust) and +[fastconstmap](https://github.com/lemire/fastconstmap) (C, with Python bindings). All four +share the key hash, the mixing, the seed sequence, the layout and the file formats, so a map +saved by any of them loads in the others: + +```java +Files.write(Path.of("map.vmap"), vm.toByteArray()); // Go, Rust, Python read it +VerifiedConstMap fromPython = VerifiedConstMap.load(Files.readAllBytes(Path.of("built-by-python.vmap"))); +``` + +- For the same keys and values, the Java classes write exactly the bytes that Go and Rust + write: `CMAP0001`, `VMAP0001` and `PMAP0001`. This is tested against the files of the other + implementations and against the Go package for sizes from 0 to 1,234,567 keys. +- `ConstMap.load` also reads fastconstmap's `CMAP0003` (the same format with a key count, + which is ignored). fastconstmap's key count in the padding word of the other two formats is + ignored too. +- Files from fastconstmap 0.9 and earlier (`CMAP0002`, `VCMP0002`) used another hash (XXH3) + and are refused with a message saying so; rebuild them with fastconstmap 0.10 or later. +- `load` validates the magic bytes, the segment parameters and the FNV-1a checksum, and throws + `InvalidFilterException` otherwise, including for a file of another kind of map. + +The constant maps are always loaded onto the heap; there is no in-place view as for filters. + +### Constant map performance + +JMH (`ConstMapBenchmark`) on an Apple M4 Max with JDK 27, ns per lookup of a present key, in +random order, with keys like `key-123456-1e3779b9`: + +| | 1 million keys | 10 million keys | +| --- | ---: | ---: | +| `ConstMap.get(String)` | 16.7 | 33.0 | +| `ConstMap.getMany` | 16.2 | 24.8 | +| `VerifiedConstMap.get(String)` | 25.8 | 65.4 | +| `VerifiedConstMap.getMany` | 20.1 | 38.1 | +| `PairedVerifiedConstMap.get(String)` | 23.1 | 44.9 | +| `PairedVerifiedConstMap.getMany` | 20.7 | 28.9 | + +About 12 ns of a lookup at one million keys is hashing the string. ASCII strings are hashed +straight from their chars; other strings are first encoded to UTF-8, which costs more. +`get(byte[])` with a ready byte array takes about 11.8 ns at one million keys. + ## API summary | Type | Members | | --- | --- | | `BinaryFuse8`, `BinaryFuse16` | `build(long[])`, `build(long[], int)`, `buildFromBytes(Collection)` | | `BinaryFuseFilter` (interface) | `contains(long)`, `contains(byte[])`, `contains(byte[], int, int)`, `contains(ByteBuffer)`, `fingerprintBits()`, `keyCount()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()`; static `wrap(ByteBuffer)`, `load(ByteBuffer)`, `load(byte[])` | +| `ConstMap` | static `build(List, long[])`, `buildFromBytes(List, long[])`, `buildFromHashes(long[], long[])`, `load(ByteBuffer)`, `load(byte[])`; `get(String)`, `get(byte[])`, `get(byte[], int, int)`, `get(ByteBuffer)`, `getByHash(long)`, `getMany(List)`, `getMany(List, long[])`, `arrayLength()`, `serializedSize()`, `serialize(ByteBuffer)`, `toByteArray()` | +| `VerifiedConstMap` | as `ConstMap`, plus `containsKey(String)`, `paired()` and the constant `NOT_FOUND` | +| `PairedVerifiedConstMap` | as `ConstMap`, plus `containsKey(String)` | +| `XxHash64` | `hash(String)`, `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)` | | `WyHash` | `hash(byte[])`, `hash(byte[], int, int)`, `hash(byte[], int, int, long)`, `hash(ByteBuffer)`, `hash(ByteBuffer, int, int, long)`, `hash64(long, long)` | | `InvalidFilterException` | thrown for corrupt or invalid serialized input | -All filters are immutable and safe for concurrent queries. +All filters and maps are immutable and safe for concurrent queries. ## Size and false-positive rate @@ -247,7 +340,12 @@ of the library will keep reading version 1. ``` The wyhash test vectors in `src/test/resources/wyhash_vectors.txt` were generated from the -reference C implementation by `tools/wyhash_vectors.c` (see the comment at the top of that file). +reference C implementation by `tools/wyhash_vectors.c` (see the comment at the top of that file), +and the XXH64 vectors in `src/test/resources/xxh64_vectors.txt` by `tools/xxh64_vectors.c`. +`src/test/resources/constmap_vectors.txt` holds the size and checksum of maps built by the Go +constmap package, written by `tools/constmap_vectors` (`go run .` in that directory), and +`src/test/resources/constmap/` holds the interoperability files of the constmap repository, +written by rsconstmap and fastconstmap. ## Release procedure @@ -270,7 +368,7 @@ The workflow uses the repository secrets `GPG_PRIVATE_KEY`, `GPG_PASSPHRASE`, `M ## License Apache License 2.0; see [LICENSE](LICENSE). The wyhash algorithm is by Wang Yi and is in the -public domain. +public domain. The XXH64 algorithm is by Yann Collet. ## Reference diff --git a/benchmarks/src/main/java/io/github/fastfilter/jbinfuse/bench/ConstMapBenchmark.java b/benchmarks/src/main/java/io/github/fastfilter/jbinfuse/bench/ConstMapBenchmark.java new file mode 100644 index 0000000..833e49f --- /dev/null +++ b/benchmarks/src/main/java/io/github/fastfilter/jbinfuse/bench/ConstMapBenchmark.java @@ -0,0 +1,106 @@ +package io.github.fastfilter.jbinfuse.bench; + +import io.github.fastfilter.jbinfuse.ConstMap; +import io.github.fastfilter.jbinfuse.PairedVerifiedConstMap; +import io.github.fastfilter.jbinfuse.VerifiedConstMap; +import java.util.ArrayList; +import java.util.List; +import java.util.SplittableRandom; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Average time of one lookup of a present string key, in random order, one at a time with + * {@code get(String)} or in batches with {@code getMany}. The query strings are allocated in + * query order, so that reading them does not add scattered loads that belong to the caller + * rather than to the map. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(1) +@State(Scope.Benchmark) +public class ConstMapBenchmark { + static final int QUERIES = 1 << 16; + + @Param({"1000000", "10000000"}) + int size; + + @Param({"const", "verified", "paired"}) + String kind; + + ConstMap cm; + VerifiedConstMap vm; + PairedVerifiedConstMap pm; + List queries; + long[] out; + + @Setup + public void setup() { + List keys = new ArrayList<>(size); + long[] values = new long[size]; + for (int i = 0; i < size; i++) { + keys.add("key-" + i + "-" + Integer.toHexString(i * 0x9E3779B9)); + values[i] = i; + } + switch (kind) { + case "const" -> cm = ConstMap.build(keys, values); + case "verified" -> vm = VerifiedConstMap.build(keys, values); + default -> pm = PairedVerifiedConstMap.build(keys, values); + } + SplittableRandom r = new SplittableRandom(size); + queries = new ArrayList<>(QUERIES); + for (int i = 0; i < QUERIES; i++) { + // A copy of the chars, not new String(key), which would share the key's array. + queries.add(new String(keys.get(r.nextInt(size)).toCharArray())); + } + out = new long[QUERIES]; + } + + @Benchmark + @OperationsPerInvocation(QUERIES) + public long get() { + long sum = 0; + switch (kind) { + case "const" -> { + for (String q : queries) { + sum += cm.get(q); + } + } + case "verified" -> { + for (String q : queries) { + sum += vm.get(q); + } + } + default -> { + for (String q : queries) { + sum += pm.get(q); + } + } + } + return sum; + } + + @Benchmark + @OperationsPerInvocation(QUERIES) + public long[] getMany() { + switch (kind) { + case "const" -> cm.getMany(queries, out); + case "verified" -> vm.getMany(queries, out); + default -> pm.getMany(queries, out); + } + return out; + } +} diff --git a/pom.xml b/pom.xml index c64525c..5d7fe37 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ jar jbinfuse - Small, fast binary fuse filters (8-bit and 16-bit) for Java, queryable off-heap from serialized bytes. + Small, fast binary fuse filters (8-bit and 16-bit) for Java, queryable off-heap from serialized bytes, and constant maps from keys to 64-bit values compatible with constmap (Go), rsconstmap (Rust) and fastconstmap (C/Python). https://github.com/FastFilter/jbinfuse diff --git a/src/main/java/io/github/fastfilter/jbinfuse/ConstMap.java b/src/main/java/io/github/fastfilter/jbinfuse/ConstMap.java new file mode 100644 index 0000000..cf0f733 --- /dev/null +++ b/src/main/java/io/github/fastfilter/jbinfuse/ConstMap.java @@ -0,0 +1,287 @@ +package io.github.fastfilter.jbinfuse; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Objects; + +/** + * An immutable map from keys (strings or byte sequences) to 64-bit values, in about 9 bytes + * per key, that does not store the keys. A lookup is one XXH64 hash, three array reads and two + * XORs. + * + *

Looking up a key that was not in the map returns an arbitrary value: use + * {@link VerifiedConstMap} or {@link PairedVerifiedConstMap} if you need to detect missing + * keys. + * + *

This is a Java implementation of constmap + * (Go). Its serialized form is shared with constmap, with + * rsconstmap (Rust) and with + * fastconstmap (C and Python): a map + * built by any of them loads in the others. For the same keys and values, this class writes + * exactly the bytes that the Go and Rust implementations write. + * + *

String keys are hashed through their UTF-8 encoding, so {@code get("abc")} and + * {@code get("abc".getBytes(UTF_8))} are the same lookup. Values are 64-bit words; read them as + * unsigned if you need values above {@link Long#MAX_VALUE}. + * + *

Instances are immutable and thread-safe. + */ +public final class ConstMap { + private final ConstMaps.Params params; + private final long seed; + private final int segmentLength; + private final int segmentLengthMask; + private final long segmentCountLength; + private final long[] data; + + private ConstMap(ConstMaps.Params params, long[] data) { + this.params = params; + this.seed = params.seed(); + this.segmentLength = params.segmentLength(); + this.segmentLengthMask = params.segmentLengthMask(); + this.segmentCountLength = params.segmentCountLength(); + this.data = data; + } + + /** + * Builds a map from string keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static ConstMap build(List keys, long[] values) { + ConstMaps.checkLengths(keys.size(), values.length); + return buildFromHashes(ConstMaps.hashStrings(keys), values); + } + + /** + * Builds a map from byte-array keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static ConstMap buildFromBytes(List keys, long[] values) { + ConstMaps.checkLengths(keys.size(), values.length); + return buildFromHashes(ConstMaps.hashBytes(keys), values); + } + + /** + * Builds a map from the XXH64 hashes of the keys ({@link XxHash64#hash(byte[])}) and their + * values, for callers that hash keys as they stream them. Neither array is modified. + * + * @param keyHashes the XXH64 hashes (seed 0) of the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two hashes are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static ConstMap buildFromHashes(long[] keyHashes, long[] values) { + ConstMaps.checkLengths(keyHashes.length, values.length); + if (keyHashes.length == 0) { + return new ConstMap(ConstMaps.Params.EMPTY, new long[0]); + } + ConstMaps.Peeled p = ConstMaps.peel(keyHashes); + long[] data = new long[p.params().arrayLength()]; + ConstMaps.assign(p, values, data, null); + return new ConstMap(p.params(), data); + } + + /** + * Returns the value of a key whose XXH64 hash (seed 0) is given. Equivalent to the other + * {@code get} methods on the key itself. + * + * @param keyHash the XXH64 hash of the key + * @return the value if the key is in the map, an arbitrary value otherwise + */ + public long getByHash(long keyHash) { + if (data.length == 0) { + return 0; + } + long hash = ConstMaps.mixsplit(keyHash, seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + int h1 = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask); + int h2 = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask); + return data[h0] ^ data[h1] ^ data[h2]; + } + + /** + * Returns the value of a string key (hashed through its UTF-8 encoding). + * + * @param key the key + * @return the value if the key is in the map, an arbitrary value otherwise + */ + public long get(String key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of a byte-array key. + * + * @param key the key + * @return the value if the key is in the map, an arbitrary value otherwise + */ + public long get(byte[] key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of the key made of {@code len} bytes of {@code key} from {@code off}. + * + * @param key the array holding the key + * @param off the index of its first byte + * @param len its length + * @return the value if the key is in the map, an arbitrary value otherwise + * @throws IndexOutOfBoundsException if the range is not within the array + */ + public long get(byte[] key, int off, int len) { + return getByHash(XxHash64.hash(key, off, len)); + } + + /** + * Returns the value of the key made of the buffer's bytes from its position to its limit. + * The buffer is not modified. + * + * @param key the key + * @return the value if the key is in the map, an arbitrary value otherwise + */ + public long get(ByteBuffer key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Looks up many string keys at once: {@code dst[i]} receives the value of + * {@code keys.get(i)}, exactly as {@link #get(String)} would return it. Hashing every key + * of a block before reading the array lets the processor overlap the memory accesses of + * several lookups, which helps when the map is larger than the cache. + * + * @param keys the keys + * @param dst where the values go; only the first {@code keys.size()} entries are written + * @throws IndexOutOfBoundsException if {@code dst} is shorter than {@code keys} + */ + public void getMany(List keys, long[] dst) { + int n = keys.size(); + Objects.checkFromIndexSize(0, n, dst.length); + if (data.length == 0) { + java.util.Arrays.fill(dst, 0, n, 0); + return; + } + final long[] d = data; + final int[] h = new int[3 * BLOCK]; + for (int start = 0; start < n; start += BLOCK) { + int m = Math.min(BLOCK, n - start); + for (int j = 0; j < m; j++) { + long hash = ConstMaps.mixsplit(XxHash64.hash(keys.get(start + j)), seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + h[3 * j] = h0; + h[3 * j + 1] = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask); + h[3 * j + 2] = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask); + } + for (int j = 0; j < m; j++) { + dst[start + j] = d[h[3 * j]] ^ d[h[3 * j + 1]] ^ d[h[3 * j + 2]]; + } + } + } + + /** Keys per block in {@link #getMany}. */ + static final int BLOCK = 8; + + /** + * Looks up many string keys at once, as {@link #getMany(List, long[])} does, into a new + * array. + * + * @param keys the keys + * @return the values, one per key + */ + public long[] getMany(List keys) { + long[] dst = new long[keys.size()]; + getMany(keys, dst); + return dst; + } + + /** + * Returns the number of 64-bit words in the map's array: about 1.125 per key for large + * maps, more for small ones. + * + * @return the array length + */ + public int arrayLength() { + return data.length; + } + + /** + * Returns the number of bytes written by {@link #serialize(ByteBuffer)}: a 28-byte header, + * the array, and an 8-byte checksum. + * + * @return the serialized size in bytes + */ + public int serializedSize() { + return ConstMaps.CMAP_HEADER + 8 * data.length + 8; + } + + /** + * Writes the map at the buffer's current position, in the {@code CMAP0001} format shared + * with the Go, Rust and C implementations, and advances the position by + * {@link #serializedSize()}. The buffer's byte order is ignored and left unchanged. + * + * @param out the destination + * @throws java.nio.BufferOverflowException if fewer than {@link #serializedSize()} bytes + * remain + * @throws java.nio.ReadOnlyBufferException if the buffer is read-only + */ + public void serialize(ByteBuffer out) { + ConstMaps.write(out, ConstMaps.MAGIC_CMAP, params, data.length, false, data); + } + + /** + * Returns the serialized form of the map, as written by {@link #serialize(ByteBuffer)}. + * + * @return a new array of {@link #serializedSize()} bytes + */ + public byte[] toByteArray() { + byte[] bytes = new byte[serializedSize()]; + serialize(ByteBuffer.wrap(bytes)); + return bytes; + } + + /** + * Reads a serialized map starting at the buffer's position into a new on-heap map, after + * validating its header and checksum. Accepts both the {@code CMAP0001} format (Go, Rust, + * this class) and fastconstmap's {@code CMAP0003}, which adds a key count that is ignored. + * The buffer's position is not changed. + * + * @param buffer a buffer holding a serialized map at its position + * @return the map + * @throws InvalidFilterException if the input is not a valid {@code ConstMap} (including a + * {@code VerifiedConstMap} or {@code PairedVerifiedConstMap}, or a file from + * fastconstmap 0.9 or earlier, which used another hash), or is corrupt + */ + public static ConstMap load(ByteBuffer buffer) { + ConstMaps.Header h = ConstMaps.read(buffer, ConstMaps.Kind.CONST, 1); + int n = h.params().arrayLength(); + return new ConstMap(h.params(), ConstMaps.readWords(buffer, h.start(), n)); + } + + /** + * Reads a serialized map, as {@link #load(ByteBuffer)} does. + * + * @param bytes the serialized map + * @return the map + * @throws InvalidFilterException if the input is invalid or corrupt + */ + public static ConstMap load(byte[] bytes) { + return load(ByteBuffer.wrap(bytes)); + } + + @Override + public String toString() { + return "ConstMap{words=" + data.length + ", bytes=" + serializedSize() + "}"; + } +} diff --git a/src/main/java/io/github/fastfilter/jbinfuse/ConstMaps.java b/src/main/java/io/github/fastfilter/jbinfuse/ConstMaps.java new file mode 100644 index 0000000..096ab33 --- /dev/null +++ b/src/main/java/io/github/fastfilter/jbinfuse/ConstMaps.java @@ -0,0 +1,557 @@ +package io.github.fastfilter.jbinfuse; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Construction and serialization shared by {@link ConstMap}, {@link VerifiedConstMap} and + * {@link PairedVerifiedConstMap}. + * + *

Everything here follows constmap (Go) + * exactly: the key hash (XXH64, seed 0), the mixing ({@code murmur64(hash + seed)}), the seed + * sequence (splitmix64 from 1), the segment parameters, the retry schedule and the peeling + * order. For the same keys and values the three maps therefore hold the same words as the Go, + * Rust (rsconstmap) and C/Python + * (fastconstmap) implementations, and + * serialize to the same bytes as Go and Rust. + */ +final class ConstMaps { + private ConstMaps() {} + + /** Returned by verified lookups for keys that are not in the map. */ + static final long NOT_FOUND = -1L; + + static final int MAX_SEGMENT_LENGTH = 262144; + static final int MAX_ITERATIONS = 100; + + /** + * Largest array length we build or load: a paired map holds two words per position in one + * Java array. + */ + static final int MAX_ARRAY_LENGTH = (Integer.MAX_VALUE - 16) / 2; + + // --------------------------------------------------------------------------------------- + // Hashing. + + static long murmur64(long h) { + h ^= h >>> 33; + h *= 0xff51afd7ed558ccdL; + h ^= h >>> 33; + h *= 0xc4ceb9fe1a85ec53L; + h ^= h >>> 33; + return h; + } + + static long mixsplit(long key, long seed) { + return murmur64(key + seed); + } + + /** The verification word derived from a mixed hash. */ + static long fingerprint(long hash) { + return hash ^ (hash >>> 32); + } + + static long[] hashStrings(List keys) { + long[] h = new long[keys.size()]; + int i = 0; + for (String k : keys) { + h[i++] = XxHash64.hash(k); + } + return h; + } + + static long[] hashBytes(List keys) { + long[] h = new long[keys.size()]; + int i = 0; + for (byte[] k : keys) { + h[i++] = XxHash64.hash(k); + } + return h; + } + + // --------------------------------------------------------------------------------------- + // Layout. + + /** + * The parameters of a map. Positions are {@code h0 = mulhi(hash, segmentCount * + * segmentLength)}, {@code h1 = (h0 + segmentLength) ^ ((hash >>> 18) & mask)} and {@code h2 = + * (h0 + 2 * segmentLength) ^ (hash & mask)}, all below {@code arrayLength = (segmentCount + + * 2) * segmentLength}. + */ + record Params(long seed, int segmentLength, int segmentCount, int arrayLength) { + static final Params EMPTY = new Params(0, 0, 0, 0); + + int segmentLengthMask() { + return segmentLength - 1; + } + + long segmentCountLength() { + return (long) segmentCount * segmentLength; + } + } + + static int calculateSegmentLength(int size) { + if (size == 0) { + return 4; + } + return 1 << (int) Math.floor(Math.log(size) / Math.log(3.33) + 2.25); + } + + static double calculateSizeFactor(int size) { + return Math.max(1.125, 0.875 + 0.25 * Math.log(1000000) / Math.log(size)); + } + + // --------------------------------------------------------------------------------------- + // Construction. + + /** + * The outcome of peeling: the seed and parameters that worked, and for each stack entry + * (to be assigned in reverse) the mixed hash, which of its three positions it owns, and the + * index of the key in the caller's arrays. + */ + record Peeled(Params params, long[] order, byte[] which, int[] keyIndex) {} + + static void checkLengths(int keys, int values) { + if (keys != values) { + throw new IllegalArgumentException( + "keys and values must have equal length (" + keys + " vs " + values + ")"); + } + } + + /** + * Finds a seed and an assignment order for keys with the given XXH64 hashes. A port of the + * construction loop of constmap's {@code New}; where the Go code recovers each key's value + * through a map from mixed hash to value, this keeps the XOR of the key indices in each + * cell next to the XOR of their hashes, so a singleton cell names its key directly. The + * peeling, and so the result, is the same. + * + * @param hashes the XXH64 key hashes; must not be empty + */ + static Peeled peel(long[] hashes) { + final int size = hashes.length; + int segmentLength = Math.min(calculateSegmentLength(size), MAX_SEGMENT_LENGTH); + long cap0 = 0; + if (size > 1) { + cap0 = Math.round(size * calculateSizeFactor(size)); + } + long totalSegmentCount = Math.max(3, (cap0 + segmentLength - 1) / segmentLength); + long arrayLengthLong = totalSegmentCount * segmentLength; + if (arrayLengthLong > MAX_ARRAY_LENGTH) { + throw new IllegalArgumentException("too many keys for one map: " + size); + } + final int capacity = (int) arrayLengthLong; + int segmentCount = (int) totalSegmentCount - 2; + + long rngcounter = 1; + rngcounter += 0x9E3779B97F4A7C15L; + long seed = splitmix64Value(rngcounter); + + final int[] alone = new int[capacity]; + final byte[] t2count = new byte[capacity]; + final long[] t2hash = new long[capacity]; + final int[] t2index = new int[capacity]; + final byte[] reverseH = new byte[size]; + final long[] reverseOrder = new long[size + 1]; + final int[] reverseIndex = new int[size]; + final int[] bucketIndex = new int[size]; + reverseOrder[size] = 1; + + for (int iterations = 0; ; iterations++) { + if (iterations > MAX_ITERATIONS) { + throw new IllegalArgumentException("failed to construct map, possible duplicate keys"); + } + if (size > 4 && size < 1_000_000) { + switch (iterations % 4) { + case 2 -> { + segmentLength /= 2; + segmentCount = segmentCount * 2 + 2; + } + case 3 -> { + segmentLength *= 2; + segmentCount = segmentCount / 2 - 1; + } + default -> { } + } + } + final int segLen = segmentLength; + final int mask = segLen - 1; + final long scl = (long) segmentCount * segLen; + + int blockBits = 1; + while ((1L << blockBits) < segmentCount) { + blockBits++; + } + final int[] startPos = new int[1 << blockBits]; + for (int i = 0; i < startPos.length; i++) { + startPos[i] = (int) (((long) i * size) >>> blockBits); + } + final int blockMask = (1 << blockBits) - 1; + for (int k = 0; k < size; k++) { + long hash = mixsplit(hashes[k], seed); + int segmentIndex = (int) (hash >>> (64 - blockBits)); + while (reverseOrder[startPos[segmentIndex]] != 0) { + segmentIndex = (segmentIndex + 1) & blockMask; + } + reverseOrder[startPos[segmentIndex]] = hash; + bucketIndex[startPos[segmentIndex]] = k; + startPos[segmentIndex]++; + } + + boolean hasError = false; + for (int i = 0; i < size; i++) { + long hash = reverseOrder[i]; + int key = bucketIndex[i]; + int i1 = (int) Math.unsignedMultiplyHigh(hash, scl); + int i2 = (i1 + segLen) ^ ((int) (hash >>> 18) & mask); + int i3 = (i1 + 2 * segLen) ^ ((int) hash & mask); + t2count[i1] += 4; + t2hash[i1] ^= hash; + t2index[i1] ^= key; + t2count[i2] = (byte) ((t2count[i2] + 4) ^ 1); + t2hash[i2] ^= hash; + t2index[i2] ^= key; + t2count[i3] = (byte) ((t2count[i3] + 4) ^ 2); + t2hash[i3] ^= hash; + t2index[i3] ^= key; + if ((t2hash[i1] & t2hash[i2] & t2hash[i3]) == 0) { + if ((t2hash[i1] == 0 && t2count[i1] == 8) + || (t2hash[i2] == 0 && t2count[i2] == 8) + || (t2hash[i3] == 0 && t2count[i3] == 8)) { + throw new IllegalArgumentException("duplicate key hash detected"); + } + } + if ((t2count[i1] & 0xFF) < 4 || (t2count[i2] & 0xFF) < 4 + || (t2count[i3] & 0xFF) < 4) { + hasError = true; + } + } + + int stacksize = 0; + if (!hasError) { + int qsize = 0; + for (int i = 0; i < capacity; i++) { + alone[qsize] = i; + if (((t2count[i] & 0xFF) >>> 2) == 1) { + qsize++; + } + } + final int segLenToMinusSegLenX2 = segLen ^ (-(2 * segLen)); + while (qsize > 0) { + qsize--; + int index = alone[qsize]; + if (((t2count[index] & 0xFF) >>> 2) == 1) { + long hash = t2hash[index]; + int found = t2count[index] & 3; + reverseH[stacksize] = (byte) found; + reverseOrder[stacksize] = hash; + int key = t2index[index]; + reverseIndex[stacksize] = key; + stacksize++; + + int h01 = (int) (hash >>> 18) & mask; + int h02 = (int) hash & mask; + int is0 = -(((found - 1) & 0xFF) >>> 7); + int is1 = -(found & 1); + int is2 = -(found >>> 1); + int other1 = index + (segLen ^ (segLenToMinusSegLenX2 & is2)); + int other2 = index - (segLen ^ (segLenToMinusSegLenX2 & is0)); + other1 ^= (h01 & ~is2) ^ (h02 & ~is0); + other2 ^= (h01 & ~is0) ^ (h02 & ~is1); + int f1 = (is0 & 1) | (is1 & 2); + int f2 = (is0 & 2) | (is2 & 1); + + alone[qsize] = other1; + if (((t2count[other1] & 0xFF) >>> 2) == 2) { + qsize++; + } + t2count[other1] = (byte) ((t2count[other1] - 4) ^ f1); + t2hash[other1] ^= hash; + t2index[other1] ^= key; + + alone[qsize] = other2; + if (((t2count[other2] & 0xFF) >>> 2) == 2) { + qsize++; + } + t2count[other2] = (byte) ((t2count[other2] - 4) ^ f2); + t2hash[other2] ^= hash; + t2index[other2] ^= key; + } + } + } + if (stacksize == size) { + Params params = new Params(seed, segLen, segmentCount, capacity); + return new Peeled(params, Arrays.copyOf(reverseOrder, size), reverseH, reverseIndex); + } + Arrays.fill(reverseOrder, 0, size, 0); + Arrays.fill(t2count, (byte) 0); + Arrays.fill(t2hash, 0); + Arrays.fill(t2index, 0); + rngcounter += 0x9E3779B97F4A7C15L; + seed = splitmix64Value(rngcounter); + } + } + + /** The output of splitmix64 for an already-advanced state. */ + private static long splitmix64Value(long z) { + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9L; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EBL; + return z ^ (z >>> 31); + } + + /** + * Fills {@code data} (and, if not null, {@code checks}) so that the three positions of each + * key XOR to its value (and to its fingerprint). + */ + static void assign(Peeled p, long[] values, long[] data, long[] checks) { + final Params params = p.params(); + final int segLen = params.segmentLength(); + final int mask = params.segmentLengthMask(); + final long scl = params.segmentCountLength(); + final long[] order = p.order(); + final byte[] which = p.which(); + final int[] keyIndex = p.keyIndex(); + for (int i = order.length - 1; i >= 0; i--) { + long hash = order[i]; + int h0 = (int) Math.unsignedMultiplyHigh(hash, scl); + int h1 = (h0 + segLen) ^ ((int) (hash >>> 18) & mask); + int h2 = (h0 + 2 * segLen) ^ ((int) hash & mask); + int a; + int b; + int c; + switch (which[i]) { + case 0 -> { + a = h0; + b = h1; + c = h2; + } + case 1 -> { + a = h1; + b = h2; + c = h0; + } + default -> { + a = h2; + b = h0; + c = h1; + } + } + data[a] = values[keyIndex[i]] ^ data[b] ^ data[c]; + if (checks != null) { + checks[a] = fingerprint(hash) ^ checks[b] ^ checks[c]; + } + } + } + + // --------------------------------------------------------------------------------------- + // Serialization. + // + // All three formats are little-endian and end with the 64-bit FNV-1a hash of every byte + // before it: + // + // ConstMap "CMAP0001" seed:8 segmentLength:4 segmentCount:4 dataLength:4 + // data:8*dataLength fnv:8 (28-byte header) + // fastconstmap writes "CMAP0003": the same with a 4-byte key count + // after dataLength (32-byte header). Both are read. + // VerifiedConstMap "VMAP0001" seed:8 segmentLength:4 segmentCount:4 dataLength:4 + // pad:4 data:8*dataLength checks:8*dataLength fnv:8 + // PairedVerified... "PMAP0001" seed:8 segmentLength:4 segmentCount:4 slotCount:4 + // pad:4 (value:8 check:8)*slotCount fnv:8 + // + // fastconstmap stores its key count in the pad word; every reader ignores it. + + static final long FNV_OFFSET = 0xcbf29ce484222325L; + static final long FNV_PRIME = 0x100000001b3L; + + static final long MAGIC_CMAP = magic("CMAP0001"); + static final long MAGIC_CMAP_COUNTED = magic("CMAP0003"); + static final long MAGIC_VMAP = magic("VMAP0001"); + static final long MAGIC_PMAP = magic("PMAP0001"); + static final long MAGIC_LEGACY_CMAP = magic("CMAP0002"); + static final long MAGIC_LEGACY_VMAP = magic("VCMP0002"); + + static final int CMAP_HEADER = 28; + static final int VMAP_HEADER = 32; + + /** The kinds of map, for naming a file handed to the wrong reader. */ + enum Kind { + CONST("ConstMap"), + VERIFIED("VerifiedConstMap"), + PAIRED("PairedVerifiedConstMap"); + + final String className; + + Kind(String className) { + this.className = className; + } + } + + private static long magic(String s) { + byte[] b = s.getBytes(StandardCharsets.US_ASCII); + return ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).getLong(); + } + + static long fnv(long h, long word) { + for (int i = 0; i < 8; i++) { + h ^= word & 0xFF; + h *= FNV_PRIME; + word >>>= 8; + } + return h; + } + + static long fnv32(long h, int word) { + for (int i = 0; i < 4; i++) { + h ^= word & 0xFF; + h *= FNV_PRIME; + word >>>= 8; + } + return h; + } + + /** A little-endian view of {@code out} at its position, after checking the space. */ + static ByteBuffer writer(ByteBuffer out, long size) { + Objects.requireNonNull(out); + if (out.isReadOnly()) { + throw new java.nio.ReadOnlyBufferException(); + } + if (out.remaining() < size) { + throw new java.nio.BufferOverflowException(); + } + return out.duplicate().order(ByteOrder.LITTLE_ENDIAN); + } + + /** + * Writes a header and the given arrays (each one after the other) and the checksum, and + * advances {@code out}'s position. + */ + static void write(ByteBuffer out, long magic, Params p, int count, boolean pad, + long[]... arrays) { + long size = (pad ? VMAP_HEADER : CMAP_HEADER) + 8L; + for (long[] a : arrays) { + size += 8L * a.length; + } + ByteBuffer w = writer(out, size); + long h = FNV_OFFSET; + w.putLong(magic); + h = fnv(h, magic); + w.putLong(p.seed()); + h = fnv(h, p.seed()); + w.putInt(p.segmentLength()); + h = fnv32(h, p.segmentLength()); + w.putInt(p.segmentCount()); + h = fnv32(h, p.segmentCount()); + w.putInt(count); + h = fnv32(h, count); + if (pad) { + w.putInt(0); + h = fnv32(h, 0); + } + for (long[] a : arrays) { + for (long v : a) { + w.putLong(v); + h = fnv(h, v); + } + } + w.putLong(h); + out.position(w.position()); + } + + /** A parsed header: the parameters, the header size and where the arrays start. */ + record Header(Params params, int headerSize, int start) {} + + /** + * Reads and validates the header of a map of the given kind at the buffer's position, and + * verifies the trailing checksum over {@code wordsPerPosition} words per array position. + */ + static Header read(ByteBuffer buffer, Kind kind, int wordsPerPosition) { + ByteBuffer b = buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN); + final int start = b.position(); + if (b.remaining() < 8) { + throw new InvalidFilterException("truncated input: no room for the magic bytes"); + } + long magic = b.getLong(start); + boolean counted = false; + Kind found; + if (magic == MAGIC_CMAP) { + found = Kind.CONST; + } else if (magic == MAGIC_CMAP_COUNTED) { + found = Kind.CONST; + counted = true; + } else if (magic == MAGIC_VMAP) { + found = Kind.VERIFIED; + } else if (magic == MAGIC_PMAP) { + found = Kind.PAIRED; + } else if (magic == MAGIC_LEGACY_CMAP || magic == MAGIC_LEGACY_VMAP) { + throw new InvalidFilterException("this file was written by fastconstmap 0.9 or earlier" + + " with a different key hash (XXH3); rebuild it from its keys with fastconstmap" + + " 0.10 or later, which writes the shared format"); + } else { + throw new InvalidFilterException("invalid magic bytes: not a constant map"); + } + if (found != kind) { + throw new InvalidFilterException("this is a " + found.className + ", not a " + + kind.className + "; load it with " + found.className + ".load"); + } + final int headerSize = (kind == Kind.CONST && !counted) ? CMAP_HEADER : VMAP_HEADER; + if (b.remaining() < headerSize) { + throw new InvalidFilterException("truncated input: " + b.remaining() + + " bytes, the header alone is " + headerSize); + } + long seed = b.getLong(start + 8); + int segmentLength = b.getInt(start + 16); + int segmentCount = b.getInt(start + 20); + long count = b.getInt(start + 24) & 0xFFFFFFFFL; + if (count > MAX_ARRAY_LENGTH) { + throw new InvalidFilterException("map too large for Java arrays: " + count + " positions"); + } + long total = headerSize + 8L * wordsPerPosition * count + 8; + if (b.remaining() < total) { + throw new InvalidFilterException("truncated input: " + b.remaining() + + " bytes, expected " + total); + } + Params params; + if (count == 0) { + params = Params.EMPTY; + } else { + if (segmentLength <= 0 || (segmentLength & (segmentLength - 1)) != 0) { + throw new InvalidFilterException( + "segment length " + Integer.toUnsignedString(segmentLength) + " is not a power of two"); + } + if (segmentCount <= 0) { + throw new InvalidFilterException("invalid segment count " + + Integer.toUnsignedString(segmentCount)); + } + if (((long) segmentCount + 2) * segmentLength != count) { + throw new InvalidFilterException(count + " positions but the segment parameters describe " + + ((long) segmentCount + 2) * segmentLength); + } + params = new Params(seed, segmentLength, segmentCount, (int) count); + } + long h = FNV_OFFSET; + int end = (int) (start + total - 8); + int i = start; + for (; i + 8 <= end; i += 8) { + h = fnv(h, b.getLong(i)); + } + if (i < end) { + h = fnv32(h, b.getInt(i)); + } + long stored = b.getLong(end); + if (stored != h) { + throw new InvalidFilterException(String.format( + "checksum mismatch (stored %016x, computed %016x)", stored, h)); + } + return new Header(params, headerSize, start + headerSize); + } + + /** Reads {@code n} little-endian words starting at absolute index {@code from}. */ + static long[] readWords(ByteBuffer buffer, int from, int n) { + long[] words = new long[n]; + buffer.duplicate().order(ByteOrder.LITTLE_ENDIAN).position(from).asLongBuffer().get(words); + return words; + } +} diff --git a/src/main/java/io/github/fastfilter/jbinfuse/InvalidFilterException.java b/src/main/java/io/github/fastfilter/jbinfuse/InvalidFilterException.java index 1feb02e..cd03483 100644 --- a/src/main/java/io/github/fastfilter/jbinfuse/InvalidFilterException.java +++ b/src/main/java/io/github/fastfilter/jbinfuse/InvalidFilterException.java @@ -1,9 +1,9 @@ package io.github.fastfilter.jbinfuse; /** - * Thrown when serialized bytes are not a valid binary fuse filter: wrong magic bytes, an - * unsupported format version or variant, inconsistent parameters, truncated input or a - * checksum mismatch. + * Thrown when serialized bytes are not a valid binary fuse filter or constant map: wrong magic + * bytes, an unsupported format version or variant, inconsistent parameters, truncated input or + * a checksum mismatch. */ public final class InvalidFilterException extends IllegalArgumentException { private static final long serialVersionUID = 1L; diff --git a/src/main/java/io/github/fastfilter/jbinfuse/PairedVerifiedConstMap.java b/src/main/java/io/github/fastfilter/jbinfuse/PairedVerifiedConstMap.java new file mode 100644 index 0000000..0cf170a --- /dev/null +++ b/src/main/java/io/github/fastfilter/jbinfuse/PairedVerifiedConstMap.java @@ -0,0 +1,298 @@ +package io.github.fastfilter.jbinfuse; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * A {@link VerifiedConstMap} with each value stored next to its check word, so that a lookup + * touches three cache lines instead of six. Same answers, same size (about 18 bytes per key); + * faster when most keys looked up are present, slower when most are missing (the verified + * layout then reads only its check array, half the size of the map). + * + *

This is a Java implementation of the {@code PairedVerifiedConstMap} of + * constmap (Go). Its serialized form + * ({@code PMAP0001}) is shared with constmap, rsconstmap (Rust) and fastconstmap (C and + * Python), and for the same keys and values this class writes exactly the bytes that the Go + * and Rust implementations write. See {@link ConstMap} for how keys are hashed. + * + *

Instances are immutable and thread-safe. + */ +public final class PairedVerifiedConstMap { + private final ConstMaps.Params params; + private final long seed; + private final int segmentLength; + private final int segmentLengthMask; + private final long segmentCountLength; + /** The value for position i at slots[2i], its check word at slots[2i+1]. */ + private final long[] slots; + + private PairedVerifiedConstMap(ConstMaps.Params params, long[] slots) { + this.params = params; + this.seed = params.seed(); + this.segmentLength = params.segmentLength(); + this.segmentLengthMask = params.segmentLengthMask(); + this.segmentCountLength = params.segmentCountLength(); + this.slots = slots; + } + + static PairedVerifiedConstMap of(ConstMaps.Params params, long[] slots) { + return new PairedVerifiedConstMap(params, slots); + } + + /** + * Builds a map from string keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static PairedVerifiedConstMap build(List keys, long[] values) { + return VerifiedConstMap.build(keys, values).paired(); + } + + /** + * Builds a map from byte-array keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static PairedVerifiedConstMap buildFromBytes(List keys, long[] values) { + return VerifiedConstMap.buildFromBytes(keys, values).paired(); + } + + /** + * Builds a map from the XXH64 hashes of the keys ({@link XxHash64#hash(byte[])}) and their + * values. Neither array is modified. + * + * @param keyHashes the XXH64 hashes (seed 0) of the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two hashes are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static PairedVerifiedConstMap buildFromHashes(long[] keyHashes, long[] values) { + return VerifiedConstMap.buildFromHashes(keyHashes, values).paired(); + } + + /** + * Returns the value of a key whose XXH64 hash (seed 0) is given. Equivalent to the other + * {@code get} methods on the key itself. + * + * @param keyHash the XXH64 hash of the key + * @return the value, or {@link VerifiedConstMap#NOT_FOUND} if the key is not in the map + */ + public long getByHash(long keyHash) { + if (slots.length == 0) { + return ConstMaps.NOT_FOUND; + } + long hash = ConstMaps.mixsplit(keyHash, seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + int h1 = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask); + int h2 = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask); + final long[] s = slots; + int i0 = 2 * h0; + int i1 = 2 * h1; + int i2 = 2 * h2; + if ((s[i0 + 1] ^ s[i1 + 1] ^ s[i2 + 1]) != ConstMaps.fingerprint(hash)) { + return ConstMaps.NOT_FOUND; + } + return s[i0] ^ s[i1] ^ s[i2]; + } + + /** + * Returns the value of a string key (hashed through its UTF-8 encoding). + * + * @param key the key + * @return the value, or {@link VerifiedConstMap#NOT_FOUND} if the key is not in the map + */ + public long get(String key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of a byte-array key. + * + * @param key the key + * @return the value, or {@link VerifiedConstMap#NOT_FOUND} if the key is not in the map + */ + public long get(byte[] key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of the key made of {@code len} bytes of {@code key} from {@code off}. + * + * @param key the array holding the key + * @param off the index of its first byte + * @param len its length + * @return the value, or {@link VerifiedConstMap#NOT_FOUND} if the key is not in the map + * @throws IndexOutOfBoundsException if the range is not within the array + */ + public long get(byte[] key, int off, int len) { + return getByHash(XxHash64.hash(key, off, len)); + } + + /** + * Returns the value of the key made of the buffer's bytes from its position to its limit. + * The buffer is not modified. + * + * @param key the key + * @return the value, or {@link VerifiedConstMap#NOT_FOUND} if the key is not in the map + */ + public long get(ByteBuffer key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns whether a string key is in the map, that is, whether {@link #get(String)} does not + * return {@link VerifiedConstMap#NOT_FOUND}. + * + * @param key the key + * @return true if the key is in the map (or, with probability about 2-64, a + * false positive), false if it is not or if its value is + * {@link VerifiedConstMap#NOT_FOUND} + */ + public boolean containsKey(String key) { + return get(key) != ConstMaps.NOT_FOUND; + } + + /** + * Looks up many string keys at once: {@code dst[i]} receives the value of + * {@code keys.get(i)}, exactly as {@link #get(String)} would return it. Hashing every key + * of a block before reading the array lets the processor overlap the memory accesses of + * several lookups. + * + * @param keys the keys + * @param dst where the values go; only the first {@code keys.size()} entries are written + * @throws IndexOutOfBoundsException if {@code dst} is shorter than {@code keys} + */ + public void getMany(List keys, long[] dst) { + int n = keys.size(); + Objects.checkFromIndexSize(0, n, dst.length); + if (slots.length == 0) { + Arrays.fill(dst, 0, n, ConstMaps.NOT_FOUND); + return; + } + final long[] s = slots; + final int[] h = new int[3 * ConstMap.BLOCK]; + final long[] fp = new long[ConstMap.BLOCK]; + for (int start = 0; start < n; start += ConstMap.BLOCK) { + int m = Math.min(ConstMap.BLOCK, n - start); + for (int j = 0; j < m; j++) { + long hash = ConstMaps.mixsplit(XxHash64.hash(keys.get(start + j)), seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + h[3 * j] = 2 * h0; + h[3 * j + 1] = 2 * ((h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask)); + h[3 * j + 2] = 2 * ((h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask)); + fp[j] = ConstMaps.fingerprint(hash); + } + for (int j = 0; j < m; j++) { + int a = h[3 * j]; + int b = h[3 * j + 1]; + int c = h[3 * j + 2]; + long v = s[a] ^ s[b] ^ s[c]; + dst[start + j] = (s[a + 1] ^ s[b + 1] ^ s[c + 1]) == fp[j] ? v : ConstMaps.NOT_FOUND; + } + } + } + + /** + * Looks up many string keys at once, as {@link #getMany(List, long[])} does, into a new + * array. + * + * @param keys the keys + * @return the values, one per key, {@link VerifiedConstMap#NOT_FOUND} for missing keys + */ + public long[] getMany(List keys) { + long[] dst = new long[keys.size()]; + getMany(keys, dst); + return dst; + } + + /** + * Returns the number of positions (value and check word pairs) in the map: about 1.125 per + * key for large maps, more for small ones. + * + * @return the number of positions + */ + public int arrayLength() { + return slots.length / 2; + } + + /** + * Returns the number of bytes written by {@link #serialize(ByteBuffer)}: a 32-byte header, + * the slots, and an 8-byte checksum. + * + * @return the serialized size in bytes + */ + public int serializedSize() { + return ConstMaps.VMAP_HEADER + 8 * slots.length + 8; + } + + /** + * Writes the map at the buffer's current position, in the {@code PMAP0001} format shared + * with the Go, Rust and C implementations, and advances the position by + * {@link #serializedSize()}. The buffer's byte order is ignored and left unchanged. + * + * @param out the destination + * @throws java.nio.BufferOverflowException if fewer than {@link #serializedSize()} bytes + * remain + * @throws java.nio.ReadOnlyBufferException if the buffer is read-only + */ + public void serialize(ByteBuffer out) { + ConstMaps.write(out, ConstMaps.MAGIC_PMAP, params, slots.length / 2, true, slots); + } + + /** + * Returns the serialized form of the map, as written by {@link #serialize(ByteBuffer)}. + * + * @return a new array of {@link #serializedSize()} bytes + */ + public byte[] toByteArray() { + byte[] bytes = new byte[serializedSize()]; + serialize(ByteBuffer.wrap(bytes)); + return bytes; + } + + /** + * Reads a serialized map ({@code PMAP0001}) starting at the buffer's position into a new + * on-heap map, after validating its header and checksum. The buffer's position is not + * changed. + * + * @param buffer a buffer holding a serialized map at its position + * @return the map + * @throws InvalidFilterException if the input is not a valid {@code PairedVerifiedConstMap} + * or is corrupt + */ + public static PairedVerifiedConstMap load(ByteBuffer buffer) { + ConstMaps.Header h = ConstMaps.read(buffer, ConstMaps.Kind.PAIRED, 2); + int n = h.params().arrayLength(); + return new PairedVerifiedConstMap(h.params(), ConstMaps.readWords(buffer, h.start(), 2 * n)); + } + + /** + * Reads a serialized map, as {@link #load(ByteBuffer)} does. + * + * @param bytes the serialized map + * @return the map + * @throws InvalidFilterException if the input is invalid or corrupt + */ + public static PairedVerifiedConstMap load(byte[] bytes) { + return load(ByteBuffer.wrap(bytes)); + } + + @Override + public String toString() { + return "PairedVerifiedConstMap{positions=" + arrayLength() + ", bytes=" + serializedSize() + + "}"; + } +} diff --git a/src/main/java/io/github/fastfilter/jbinfuse/VerifiedConstMap.java b/src/main/java/io/github/fastfilter/jbinfuse/VerifiedConstMap.java new file mode 100644 index 0000000..87f6a25 --- /dev/null +++ b/src/main/java/io/github/fastfilter/jbinfuse/VerifiedConstMap.java @@ -0,0 +1,329 @@ +package io.github.fastfilter.jbinfuse; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * An immutable map from keys (strings or byte sequences) to 64-bit values that detects + * missing keys, in about 18 bytes per key, without storing the keys. Next to the array of + * values of a {@link ConstMap} it keeps an array of 64-bit check words: a lookup returns + * {@link #NOT_FOUND} for a key that was not in the map, except with probability about + * 2-64. + * + *

A lookup of a present key reads three check words and three values, in two arrays. A + * lookup of a missing key usually stops after the check words, so this layout is the better + * one when most keys looked up are missing; {@link PairedVerifiedConstMap} stores each value + * next to its check word and is faster when most are present. {@link #paired()} converts. + * + *

This is a Java implementation of the {@code VerifiedConstMap} of + * constmap (Go). Its serialized form + * ({@code VMAP0001}) is shared with constmap, rsconstmap (Rust) and fastconstmap (C and + * Python), and for the same keys and values this class writes exactly the bytes that the Go + * and Rust implementations write. See {@link ConstMap} for how keys are hashed. + * + *

Instances are immutable and thread-safe. + */ +public final class VerifiedConstMap { + /** + * The value returned for a key that is not in the map: all 64 bits set ({@code -1}, or + * 264-1 read as unsigned). A key mapped to this value cannot be told apart from a + * missing key. + */ + public static final long NOT_FOUND = ConstMaps.NOT_FOUND; + + private final ConstMaps.Params params; + private final long seed; + private final int segmentLength; + private final int segmentLengthMask; + private final long segmentCountLength; + private final long[] data; + private final long[] checks; + + private VerifiedConstMap(ConstMaps.Params params, long[] data, long[] checks) { + this.params = params; + this.seed = params.seed(); + this.segmentLength = params.segmentLength(); + this.segmentLengthMask = params.segmentLengthMask(); + this.segmentCountLength = params.segmentCountLength(); + this.data = data; + this.checks = checks; + } + + /** + * Builds a map from string keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static VerifiedConstMap build(List keys, long[] values) { + ConstMaps.checkLengths(keys.size(), values.length); + return buildFromHashes(ConstMaps.hashStrings(keys), values); + } + + /** + * Builds a map from byte-array keys and their values: {@code keys.get(i)} maps to + * {@code values[i]}. Keys must be distinct. Construction is deterministic. + * + * @param keys the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two keys are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static VerifiedConstMap buildFromBytes(List keys, long[] values) { + ConstMaps.checkLengths(keys.size(), values.length); + return buildFromHashes(ConstMaps.hashBytes(keys), values); + } + + /** + * Builds a map from the XXH64 hashes of the keys ({@link XxHash64#hash(byte[])}) and their + * values. Neither array is modified. + * + * @param keyHashes the XXH64 hashes (seed 0) of the keys, distinct + * @param values the values, one per key + * @return the map + * @throws IllegalArgumentException if the lengths differ, if two hashes are equal (detected + * with high probability), or if there are too many keys for one map + */ + public static VerifiedConstMap buildFromHashes(long[] keyHashes, long[] values) { + ConstMaps.checkLengths(keyHashes.length, values.length); + if (keyHashes.length == 0) { + return new VerifiedConstMap(ConstMaps.Params.EMPTY, new long[0], new long[0]); + } + ConstMaps.Peeled p = ConstMaps.peel(keyHashes); + int n = p.params().arrayLength(); + long[] data = new long[n]; + long[] checks = new long[n]; + ConstMaps.assign(p, values, data, checks); + return new VerifiedConstMap(p.params(), data, checks); + } + + /** + * Returns the value of a key whose XXH64 hash (seed 0) is given. Equivalent to the other + * {@code get} methods on the key itself. + * + * @param keyHash the XXH64 hash of the key + * @return the value, or {@link #NOT_FOUND} if the key is not in the map + */ + public long getByHash(long keyHash) { + if (data.length == 0) { + return NOT_FOUND; + } + long hash = ConstMaps.mixsplit(keyHash, seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + int h1 = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask); + int h2 = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask); + if ((checks[h0] ^ checks[h1] ^ checks[h2]) != ConstMaps.fingerprint(hash)) { + return NOT_FOUND; + } + return data[h0] ^ data[h1] ^ data[h2]; + } + + /** + * Returns the value of a string key (hashed through its UTF-8 encoding). + * + * @param key the key + * @return the value, or {@link #NOT_FOUND} if the key is not in the map + */ + public long get(String key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of a byte-array key. + * + * @param key the key + * @return the value, or {@link #NOT_FOUND} if the key is not in the map + */ + public long get(byte[] key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns the value of the key made of {@code len} bytes of {@code key} from {@code off}. + * + * @param key the array holding the key + * @param off the index of its first byte + * @param len its length + * @return the value, or {@link #NOT_FOUND} if the key is not in the map + * @throws IndexOutOfBoundsException if the range is not within the array + */ + public long get(byte[] key, int off, int len) { + return getByHash(XxHash64.hash(key, off, len)); + } + + /** + * Returns the value of the key made of the buffer's bytes from its position to its limit. + * The buffer is not modified. + * + * @param key the key + * @return the value, or {@link #NOT_FOUND} if the key is not in the map + */ + public long get(ByteBuffer key) { + return getByHash(XxHash64.hash(key)); + } + + /** + * Returns whether a string key is in the map, that is, whether {@link #get(String)} does not + * return {@link #NOT_FOUND}. + * + * @param key the key + * @return true if the key is in the map (or, with probability about 2-64, a + * false positive), false if it is not or if its value is {@link #NOT_FOUND} + */ + public boolean containsKey(String key) { + return get(key) != NOT_FOUND; + } + + /** + * Looks up many string keys at once: {@code dst[i]} receives the value of + * {@code keys.get(i)}, exactly as {@link #get(String)} would return it. Hashing every key + * of a block before reading the arrays lets the processor overlap the memory accesses of + * several lookups. + * + * @param keys the keys + * @param dst where the values go; only the first {@code keys.size()} entries are written + * @throws IndexOutOfBoundsException if {@code dst} is shorter than {@code keys} + */ + public void getMany(List keys, long[] dst) { + int n = keys.size(); + Objects.checkFromIndexSize(0, n, dst.length); + if (data.length == 0) { + Arrays.fill(dst, 0, n, NOT_FOUND); + return; + } + final long[] d = data; + final long[] c = checks; + final int[] h = new int[3 * ConstMap.BLOCK]; + final long[] fp = new long[ConstMap.BLOCK]; + for (int start = 0; start < n; start += ConstMap.BLOCK) { + int m = Math.min(ConstMap.BLOCK, n - start); + for (int j = 0; j < m; j++) { + long hash = ConstMaps.mixsplit(XxHash64.hash(keys.get(start + j)), seed); + int h0 = (int) Math.unsignedMultiplyHigh(hash, segmentCountLength); + h[3 * j] = h0; + h[3 * j + 1] = (h0 + segmentLength) ^ ((int) (hash >>> 18) & segmentLengthMask); + h[3 * j + 2] = (h0 + 2 * segmentLength) ^ ((int) hash & segmentLengthMask); + fp[j] = ConstMaps.fingerprint(hash); + } + for (int j = 0; j < m; j++) { + int a = h[3 * j]; + int b = h[3 * j + 1]; + int e = h[3 * j + 2]; + long v = d[a] ^ d[b] ^ d[e]; + dst[start + j] = (c[a] ^ c[b] ^ c[e]) == fp[j] ? v : NOT_FOUND; + } + } + } + + /** + * Looks up many string keys at once, as {@link #getMany(List, long[])} does, into a new + * array. + * + * @param keys the keys + * @return the values, one per key, {@link #NOT_FOUND} for missing keys + */ + public long[] getMany(List keys) { + long[] dst = new long[keys.size()]; + getMany(keys, dst); + return dst; + } + + /** + * Returns the same map in the paired layout, as a copy. + * + * @return a {@link PairedVerifiedConstMap} giving the same answers + */ + public PairedVerifiedConstMap paired() { + long[] slots = new long[2 * data.length]; + for (int i = 0; i < data.length; i++) { + slots[2 * i] = data[i]; + slots[2 * i + 1] = checks[i]; + } + return PairedVerifiedConstMap.of(params, slots); + } + + /** + * Returns the number of positions in each of the map's two arrays: about 1.125 per key for + * large maps, more for small ones. + * + * @return the array length + */ + public int arrayLength() { + return data.length; + } + + /** + * Returns the number of bytes written by {@link #serialize(ByteBuffer)}: a 32-byte header, + * the two arrays, and an 8-byte checksum. + * + * @return the serialized size in bytes + */ + public int serializedSize() { + return ConstMaps.VMAP_HEADER + 16 * data.length + 8; + } + + /** + * Writes the map at the buffer's current position, in the {@code VMAP0001} format shared + * with the Go, Rust and C implementations, and advances the position by + * {@link #serializedSize()}. The buffer's byte order is ignored and left unchanged. + * + * @param out the destination + * @throws java.nio.BufferOverflowException if fewer than {@link #serializedSize()} bytes + * remain + * @throws java.nio.ReadOnlyBufferException if the buffer is read-only + */ + public void serialize(ByteBuffer out) { + ConstMaps.write(out, ConstMaps.MAGIC_VMAP, params, data.length, true, data, checks); + } + + /** + * Returns the serialized form of the map, as written by {@link #serialize(ByteBuffer)}. + * + * @return a new array of {@link #serializedSize()} bytes + */ + public byte[] toByteArray() { + byte[] bytes = new byte[serializedSize()]; + serialize(ByteBuffer.wrap(bytes)); + return bytes; + } + + /** + * Reads a serialized map ({@code VMAP0001}) starting at the buffer's position into a new + * on-heap map, after validating its header and checksum. The buffer's position is not + * changed. + * + * @param buffer a buffer holding a serialized map at its position + * @return the map + * @throws InvalidFilterException if the input is not a valid {@code VerifiedConstMap} or is + * corrupt + */ + public static VerifiedConstMap load(ByteBuffer buffer) { + ConstMaps.Header h = ConstMaps.read(buffer, ConstMaps.Kind.VERIFIED, 2); + int n = h.params().arrayLength(); + return new VerifiedConstMap(h.params(), ConstMaps.readWords(buffer, h.start(), n), + ConstMaps.readWords(buffer, h.start() + 8 * n, n)); + } + + /** + * Reads a serialized map, as {@link #load(ByteBuffer)} does. + * + * @param bytes the serialized map + * @return the map + * @throws InvalidFilterException if the input is invalid or corrupt + */ + public static VerifiedConstMap load(byte[] bytes) { + return load(ByteBuffer.wrap(bytes)); + } + + @Override + public String toString() { + return "VerifiedConstMap{positions=" + data.length + ", bytes=" + serializedSize() + "}"; + } +} diff --git a/src/main/java/io/github/fastfilter/jbinfuse/XxHash64.java b/src/main/java/io/github/fastfilter/jbinfuse/XxHash64.java new file mode 100644 index 0000000..24ec26a --- /dev/null +++ b/src/main/java/io/github/fastfilter/jbinfuse/XxHash64.java @@ -0,0 +1,321 @@ +package io.github.fastfilter.jbinfuse; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** + * A Java implementation of XXH64, the 64-bit hash of xxHash by Yann Collet + * (github.com/Cyan4973/xxHash). + * + *

The results are bit-for-bit identical to the reference C function + * {@code XXH64(input, length, seed)}, and so to Go's {@code xxhash.Sum64} (seed 0). This is + * the key hash of the constant maps ({@link ConstMap}, {@link VerifiedConstMap}, + * {@link PairedVerifiedConstMap}), which must agree with the Go, Rust and C implementations + * for their files to be interchangeable. Bytes are always read as little-endian, regardless + * of the platform or of the byte order of a {@link ByteBuffer}. + * + *

This class is stateless and thread-safe. + */ +public final class XxHash64 { + private static final long P1 = 0x9E3779B185EBCA87L; + private static final long P2 = 0xC2B2AE3D27D4EB4FL; + private static final long P3 = 0x165667B19E3779F9L; + private static final long P4 = 0x85EBCA77C2B2AE63L; + private static final long P5 = 0x27D4EB2F165667C5L; + + private static final VarHandle LONG_LE = + MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN); + private static final VarHandle INT_LE = + MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN); + + private XxHash64() {} + + /** + * Hashes the UTF-8 encoding of a string, with seed 0. This is how the constant maps hash + * {@code String} keys, and equals Go's {@code xxhash.Sum64String(s)}. + * + * @param s the string to hash + * @return the 64-bit hash + */ + public static long hash(String s) { + final int len = s.length(); + // Fast path for ASCII strings, whose UTF-8 encoding is one byte per char: hash the chars + // directly rather than allocate the encoding. Any other char sends us to the encoder. + int p = 0; + long h; + if (len >= 32) { + long v1 = P1 + P2; + long v2 = P2; + long v3 = 0; + long v4 = -P1; + final int limit = len - 32; + do { + long a = ascii8(s, p); + long b = ascii8(s, p + 8); + long c = ascii8(s, p + 16); + long d = ascii8(s, p + 24); + if (((a | b | c | d) & 0x8080808080808080L) != 0) { + return hashEncoded(s); + } + v1 = round(v1, a); + v2 = round(v2, b); + v3 = round(v3, c); + v4 = round(v4, d); + p += 32; + } while (p <= limit); + h = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + Long.rotateLeft(v3, 12) + + Long.rotateLeft(v4, 18); + h = mergeRound(h, v1); + h = mergeRound(h, v2); + h = mergeRound(h, v3); + h = mergeRound(h, v4); + } else { + h = P5; + } + h += len; + while (p + 8 <= len) { + long a = ascii8(s, p); + if ((a & 0x8080808080808080L) != 0) { + return hashEncoded(s); + } + h ^= round(0, a); + h = Long.rotateLeft(h, 27) * P1 + P4; + p += 8; + } + if (p + 4 <= len) { + long a = asciiWord(s, p, 4); + if ((a & 0x80808080L) != 0) { + return hashEncoded(s); + } + h ^= a * P1; + h = Long.rotateLeft(h, 23) * P2 + P3; + p += 4; + } + while (p < len) { + char c = s.charAt(p); + if (c >= 0x80) { + return hashEncoded(s); + } + h ^= c * P5; + h = Long.rotateLeft(h, 11) * P1; + p++; + } + return avalanche(h); + } + + /** + * The little-endian word of {@code n} chars from {@code p}, each char that is not ASCII + * replaced by 0x80 so that it shows in the caller's {@code 0x80} mask. + */ + private static long asciiWord(String s, int p, int n) { + long w = 0; + for (int i = n - 1; i >= 0; i--) { + char c = s.charAt(p + i); + w = (w << 8) | (c < 0x80 ? c : 0x80); + } + return w; + } + + private static long ascii8(String s, int p) { + return asciiWord(s, p, 8); + } + + private static long hashEncoded(String s) { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + return hash(b, 0, b.length, 0); + } + + /** + * Hashes a whole byte array with seed 0. + * + * @param data the bytes to hash + * @return the 64-bit hash + */ + public static long hash(byte[] data) { + return hash(data, 0, data.length, 0); + } + + /** + * Hashes {@code len} bytes of {@code data} starting at {@code off}, with seed 0. + * + * @param data the array holding the bytes + * @param off the index of the first byte to hash + * @param len the number of bytes to hash + * @return the 64-bit hash + * @throws IndexOutOfBoundsException if the range is not within {@code data} + */ + public static long hash(byte[] data, int off, int len) { + return hash(data, off, len, 0); + } + + private static long round(long acc, long input) { + acc += input * P2; + acc = Long.rotateLeft(acc, 31); + return acc * P1; + } + + private static long mergeRound(long acc, long val) { + acc ^= round(0, val); + return acc * P1 + P4; + } + + private static long avalanche(long h) { + h ^= h >>> 33; + h *= P2; + h ^= h >>> 29; + h *= P3; + h ^= h >>> 32; + return h; + } + + private static long r8(byte[] p, int i) { + return (long) LONG_LE.get(p, i); + } + + private static long r4(byte[] p, int i) { + return ((int) INT_LE.get(p, i)) & 0xFFFFFFFFL; + } + + /** + * Hashes {@code len} bytes of {@code data} starting at {@code off}, with the given seed. + * + * @param data the array holding the bytes + * @param off the index of the first byte to hash + * @param len the number of bytes to hash + * @param seed the seed + * @return the 64-bit hash, equal to the C reference {@code XXH64(data + off, len, seed)} + * @throws IndexOutOfBoundsException if the range is not within {@code data} + */ + public static long hash(byte[] data, int off, int len, long seed) { + Objects.checkFromIndexSize(off, len, data.length); + int p = off; + final int end = off + len; + long h; + if (len >= 32) { + long v1 = seed + P1 + P2; + long v2 = seed + P2; + long v3 = seed; + long v4 = seed - P1; + final int limit = end - 32; + do { + v1 = round(v1, r8(data, p)); + v2 = round(v2, r8(data, p + 8)); + v3 = round(v3, r8(data, p + 16)); + v4 = round(v4, r8(data, p + 24)); + p += 32; + } while (p <= limit); + h = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + Long.rotateLeft(v3, 12) + + Long.rotateLeft(v4, 18); + h = mergeRound(h, v1); + h = mergeRound(h, v2); + h = mergeRound(h, v3); + h = mergeRound(h, v4); + } else { + h = seed + P5; + } + h += len; + while (p + 8 <= end) { + h ^= round(0, r8(data, p)); + h = Long.rotateLeft(h, 27) * P1 + P4; + p += 8; + } + if (p + 4 <= end) { + h ^= r4(data, p) * P1; + h = Long.rotateLeft(h, 23) * P2 + P3; + p += 4; + } + while (p < end) { + h ^= (data[p] & 0xFFL) * P5; + h = Long.rotateLeft(h, 11) * P1; + p++; + } + return avalanche(h); + } + + /** + * Hashes the bytes of a buffer from its position to its limit, with seed 0. The buffer's + * position, limit and byte order are not changed. + * + * @param buf the buffer + * @return the 64-bit hash + */ + public static long hash(ByteBuffer buf) { + return hash(buf, buf.position(), buf.remaining(), 0); + } + + private static long r8(ByteBuffer p, int i, boolean swap) { + long v = p.getLong(i); + return swap ? Long.reverseBytes(v) : v; + } + + private static long r4(ByteBuffer p, int i, boolean swap) { + int v = p.getInt(i); + return (swap ? Integer.reverseBytes(v) : v) & 0xFFFFFFFFL; + } + + /** + * Hashes {@code len} bytes of a buffer starting at absolute index {@code off}, with the + * given seed. The buffer's position, limit and byte order are not changed. + * + * @param buf the buffer + * @param off the absolute index of the first byte to hash + * @param len the number of bytes to hash + * @param seed the seed + * @return the 64-bit hash, equal to {@link #hash(byte[], int, int, long)} on the same bytes + * @throws IndexOutOfBoundsException if the range is not within the buffer's limit + */ + public static long hash(ByteBuffer buf, int off, int len, long seed) { + if (buf.hasArray()) { + return hash(buf.array(), buf.arrayOffset() + Objects.checkFromIndexSize(off, len, + buf.limit()), len, seed); + } + Objects.checkFromIndexSize(off, len, buf.limit()); + final boolean swap = buf.order() != ByteOrder.LITTLE_ENDIAN; + int p = off; + final int end = off + len; + long h; + if (len >= 32) { + long v1 = seed + P1 + P2; + long v2 = seed + P2; + long v3 = seed; + long v4 = seed - P1; + final int limit = end - 32; + do { + v1 = round(v1, r8(buf, p, swap)); + v2 = round(v2, r8(buf, p + 8, swap)); + v3 = round(v3, r8(buf, p + 16, swap)); + v4 = round(v4, r8(buf, p + 24, swap)); + p += 32; + } while (p <= limit); + h = Long.rotateLeft(v1, 1) + Long.rotateLeft(v2, 7) + Long.rotateLeft(v3, 12) + + Long.rotateLeft(v4, 18); + h = mergeRound(h, v1); + h = mergeRound(h, v2); + h = mergeRound(h, v3); + h = mergeRound(h, v4); + } else { + h = seed + P5; + } + h += len; + while (p + 8 <= end) { + h ^= round(0, r8(buf, p, swap)); + h = Long.rotateLeft(h, 27) * P1 + P4; + p += 8; + } + if (p + 4 <= end) { + h ^= r4(buf, p, swap) * P1; + h = Long.rotateLeft(h, 23) * P2 + P3; + p += 4; + } + while (p < end) { + h ^= (buf.get(p) & 0xFFL) * P5; + h = Long.rotateLeft(h, 11) * P1; + p++; + } + return avalanche(h); + } +} diff --git a/src/main/java/io/github/fastfilter/jbinfuse/package-info.java b/src/main/java/io/github/fastfilter/jbinfuse/package-info.java index 3c66485..ef932ee 100644 --- a/src/main/java/io/github/fastfilter/jbinfuse/package-info.java +++ b/src/main/java/io/github/fastfilter/jbinfuse/package-info.java @@ -8,5 +8,12 @@ * {@link io.github.fastfilter.jbinfuse.BinaryFuseFilter#serialize(java.nio.ByteBuffer)}, and * query the serialized bytes in place with * {@link io.github.fastfilter.jbinfuse.BinaryFuseFilter#wrap(java.nio.ByteBuffer)}. + * + *

The same construction also gives compact immutable maps from keys to 64-bit values: + * {@link io.github.fastfilter.jbinfuse.ConstMap} (about 9 bytes per key, arbitrary answers for + * missing keys), and {@link io.github.fastfilter.jbinfuse.VerifiedConstMap} and + * {@link io.github.fastfilter.jbinfuse.PairedVerifiedConstMap} (about 18 bytes per key, missing + * keys detected). Their files are interchangeable with those of the Go, Rust and C/Python + * implementations (constmap, rsconstmap, fastconstmap). */ package io.github.fastfilter.jbinfuse; diff --git a/src/test/java/io/github/fastfilter/jbinfuse/ConstMapTest.java b/src/test/java/io/github/fastfilter/jbinfuse/ConstMapTest.java new file mode 100644 index 0000000..83bfda0 --- /dev/null +++ b/src/test/java/io/github/fastfilter/jbinfuse/ConstMapTest.java @@ -0,0 +1,430 @@ +package io.github.fastfilter.jbinfuse; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SplittableRandom; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class ConstMapTest { + + static final long NOT_FOUND = VerifiedConstMap.NOT_FOUND; + + static byte[] resource(String name) throws IOException { + try (InputStream in = ConstMapTest.class.getResourceAsStream("/constmap/" + name)) { + if (in == null) { + throw new IOException("missing resource " + name); + } + return in.readAllBytes(); + } + } + + static List keys(String prefix, int n) { + List keys = new ArrayList<>(n); + for (int i = 0; i < n; i++) { + keys.add(prefix + i); + } + return keys; + } + + static long[] values(int n) { + long[] v = new long[n]; + for (int i = 0; i < n; i++) { + v[i] = i * 0x9E3779B97F4A7C15L; + } + return v; + } + + // --------------------------------------------------------------------------------------- + // Interoperability with the Go, Rust and C implementations. + // + // src/test/resources/constmap/ holds the interop files of github.com/lemire/constmap + // (testdata/interop at commit 6f9f4a9): rust.* from rsconstmap, fastconstmap.* from + // fastconstmap 0.10+, and fastconstmap-0.9.* from the legacy XXH3 format. All were built + // from keys "key-i" mapped to 7 * i, for i in [0, 200). + + static final List INTEROP_KEYS = keys("key-", 200); + + static long[] interopValues() { + long[] v = new long[200]; + for (int i = 0; i < 200; i++) { + v[i] = 7L * i; + } + return v; + } + + interface Lookup { + long get(String key); + } + + static void checkInterop(String name, Lookup map, boolean verified) { + long[] v = interopValues(); + for (int i = 0; i < 200; i++) { + assertEquals(v[i], map.get(INTEROP_KEYS.get(i)), name + " key-" + i); + } + if (verified) { + for (int i = 0; i < 1000; i++) { + assertEquals(NOT_FOUND, map.get("absent-" + i), name + " absent-" + i); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"rust", "fastconstmap"}) + void readsOtherImplementations(String src) throws IOException { + checkInterop(src + ".cmap", ConstMap.load(resource(src + ".cmap"))::get, false); + checkInterop(src + ".vmap", VerifiedConstMap.load(resource(src + ".vmap"))::get, true); + checkInterop(src + ".pmap", PairedVerifiedConstMap.load(resource(src + ".pmap"))::get, + true); + } + + @Test + void writesTheSameBytesAsGoAndRust() throws IOException { + long[] v = interopValues(); + VerifiedConstMap vm = VerifiedConstMap.build(INTEROP_KEYS, v); + assertArrayEquals(resource("rust.cmap"), ConstMap.build(INTEROP_KEYS, v).toByteArray()); + assertArrayEquals(resource("rust.vmap"), vm.toByteArray()); + assertArrayEquals(resource("rust.pmap"), vm.paired().toByteArray()); + assertArrayEquals(resource("rust.pmap"), + PairedVerifiedConstMap.build(INTEROP_KEYS, v).toByteArray()); + } + + @Test + void writesWhatFastconstmapWritesApartFromTheKeyCount() throws IOException { + long[] v = interopValues(); + byte[] ours = ConstMap.build(INTEROP_KEYS, v).toByteArray(); + byte[] theirs = resource("fastconstmap.cmap"); + assertEquals("CMAP0003", new String(theirs, 0, 8, StandardCharsets.US_ASCII)); + assertArrayEquals(Arrays.copyOfRange(ours, 8, 28), Arrays.copyOfRange(theirs, 8, 28)); + assertEquals(200, ByteBuffer.wrap(theirs).order(ByteOrder.LITTLE_ENDIAN).getInt(28)); + assertArrayEquals(Arrays.copyOfRange(ours, 28, ours.length - 8), + Arrays.copyOfRange(theirs, 32, theirs.length - 8)); + VerifiedConstMap vm = VerifiedConstMap.build(INTEROP_KEYS, v); + for (String ext : new String[] {"vmap", "pmap"}) { + byte[] mine = ext.equals("vmap") ? vm.toByteArray() : vm.paired().toByteArray(); + byte[] c = resource("fastconstmap." + ext); + assertArrayEquals(Arrays.copyOfRange(mine, 0, 28), Arrays.copyOfRange(c, 0, 28), ext); + assertArrayEquals(Arrays.copyOfRange(mine, 32, mine.length - 8), + Arrays.copyOfRange(c, 32, c.length - 8), ext); + } + } + + @Test + void legacyFastconstmapFilesAreNamed() throws IOException { + for (String name : new String[] {"fastconstmap-0.9.cmap", "fastconstmap-0.9.vmap"}) { + byte[] b = resource(name); + for (Runnable load : List.of(() -> ConstMap.load(b), + () -> VerifiedConstMap.load(b), () -> PairedVerifiedConstMap.load(b))) { + InvalidFilterException e = assertThrows(InvalidFilterException.class, load::run); + assertTrue(e.getMessage().contains("fastconstmap 0.9"), e.getMessage()); + } + } + } + + @Test + void wrongKindIsNamed() throws IOException { + InvalidFilterException e = assertThrows(InvalidFilterException.class, + () -> ConstMap.load(resource("rust.vmap"))); + assertTrue(e.getMessage().contains("VerifiedConstMap"), e.getMessage()); + e = assertThrows(InvalidFilterException.class, + () -> VerifiedConstMap.load(resource("rust.pmap"))); + assertTrue(e.getMessage().contains("PairedVerifiedConstMap"), e.getMessage()); + e = assertThrows(InvalidFilterException.class, + () -> PairedVerifiedConstMap.load(resource("rust.cmap"))); + assertTrue(e.getMessage().contains("is a ConstMap"), e.getMessage()); + } + + /** + * Checks the bytes written for many sizes and two key sets (one non-ASCII) against + * github.com/lemire/constmap itself (tools/constmap_vectors): same size and same trailing + * FNV-1a checksum, which covers every other byte. This exercises the small-size + * parameters, the retries with halved and doubled segments below a million keys, and the + * path without them above. + */ + @Test + void matchesGoReferenceForManySizes() throws IOException { + int checked = 0; + try (BufferedReader r = new BufferedReader(new InputStreamReader( + ConstMapTest.class.getResourceAsStream("/constmap_vectors.txt"), + StandardCharsets.UTF_8))) { + String last = null; + ConstMap cm = null; + VerifiedConstMap vm = null; + for (String line; (line = r.readLine()) != null; ) { + String[] f = line.split(" "); + String prefix = f[1]; + int n = Integer.parseInt(f[2]); + String id = prefix + n; + if (!id.equals(last)) { + List k = keys(prefix, n); + long[] v = values(n); + cm = ConstMap.build(k, v); + vm = VerifiedConstMap.build(k, v); + last = id; + if (n <= 100000) { + for (int i = 0; i < n; i++) { + assertEquals(v[i], cm.get(k.get(i))); + assertEquals(v[i], vm.get(k.get(i))); + } + } + } + byte[] bytes = switch (f[0]) { + case "C" -> cm.toByteArray(); + case "V" -> vm.toByteArray(); + default -> vm.paired().toByteArray(); + }; + assertEquals(Integer.parseInt(f[3]), bytes.length, line); + long sum = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + .getLong(bytes.length - 8); + assertEquals(Long.parseUnsignedLong(f[4], 16), sum, line); + checked++; + } + } + assertTrue(checked > 150); + } + + // --------------------------------------------------------------------------------------- + // Behavior. + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2, 3, 5, 17, 1000, 50000}) + void allKindsAgree(int n) { + List k = keys("k", n); + long[] v = values(n); + ConstMap cm = ConstMap.build(k, v); + VerifiedConstMap vm = VerifiedConstMap.build(k, v); + PairedVerifiedConstMap pm = PairedVerifiedConstMap.build(k, v); + for (int i = 0; i < n; i++) { + String key = k.get(i); + byte[] b = key.getBytes(StandardCharsets.UTF_8); + assertEquals(v[i], cm.get(key)); + assertEquals(v[i], cm.get(b)); + assertEquals(v[i], vm.get(key)); + assertEquals(v[i], vm.get(ByteBuffer.wrap(b))); + assertEquals(v[i], pm.get(key)); + assertEquals(v[i], pm.getByHash(XxHash64.hash(b))); + assertTrue(vm.containsKey(key)); + assertTrue(pm.containsKey(key)); + } + SplittableRandom rnd = new SplittableRandom(n); + for (int i = 0; i < 10000; i++) { + String absent = "absent-" + rnd.nextLong(); + assertEquals(NOT_FOUND, vm.get(absent)); + assertEquals(NOT_FOUND, pm.get(absent)); + assertFalse(vm.containsKey(absent)); + } + assertEquals(vm.arrayLength(), cm.arrayLength()); + assertEquals(vm.arrayLength(), pm.arrayLength()); + assertEquals(vm.serializedSize(), pm.serializedSize()); + } + + @Test + void byteKeysMatchStringKeys() { + List k = keys("é-", 300); + List b = new ArrayList<>(); + for (String s : k) { + b.add(s.getBytes(StandardCharsets.UTF_8)); + } + long[] v = values(300); + assertArrayEquals(ConstMap.build(k, v).toByteArray(), + ConstMap.buildFromBytes(b, v).toByteArray()); + assertArrayEquals(VerifiedConstMap.build(k, v).toByteArray(), + VerifiedConstMap.buildFromBytes(b, v).toByteArray()); + assertArrayEquals(PairedVerifiedConstMap.build(k, v).toByteArray(), + PairedVerifiedConstMap.buildFromBytes(b, v).toByteArray()); + ConstMap cm = ConstMap.buildFromBytes(b, v); + byte[] padded = new byte[64]; + byte[] key = b.get(42); + System.arraycopy(key, 0, padded, 9, key.length); + assertEquals(v[42], cm.get(padded, 9, key.length)); + ByteBuffer direct = ByteBuffer.allocateDirect(64); + direct.position(3); + direct.put(key); + direct.flip().position(3); + assertEquals(v[42], cm.get(direct)); + assertEquals(3, direct.position()); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 7, 8, 9, 100, 1001}) + void getManyMatchesGet(int n) { + List k = keys("m", n); + long[] v = values(n); + List queries = new ArrayList<>(k); + for (int i = 0; i < n / 2 + 3; i++) { + queries.add("missing-" + i); + } + ConstMap cm = ConstMap.build(k, v); + VerifiedConstMap vm = VerifiedConstMap.build(k, v); + PairedVerifiedConstMap pm = vm.paired(); + long[] c = cm.getMany(queries); + long[] w = vm.getMany(queries); + long[] p = new long[queries.size() + 2]; + p[queries.size()] = 12345; + pm.getMany(queries, p); + for (int i = 0; i < queries.size(); i++) { + String q = queries.get(i); + assertEquals(cm.get(q), c[i]); + assertEquals(vm.get(q), w[i]); + assertEquals(pm.get(q), p[i]); + } + assertEquals(12345, p[queries.size()]); + assertThrows(IndexOutOfBoundsException.class, + () -> vm.getMany(queries, new long[queries.size() - 1])); + } + + @Test + void emptyMaps() { + ConstMap cm = ConstMap.build(List.of(), new long[0]); + VerifiedConstMap vm = VerifiedConstMap.build(List.of(), new long[0]); + PairedVerifiedConstMap pm = vm.paired(); + assertEquals(NOT_FOUND, vm.get("x")); + assertEquals(NOT_FOUND, pm.get("x")); + cm.get("x"); // arbitrary value, but no exception + assertEquals(0, ConstMap.load(cm.toByteArray()).arrayLength()); + assertEquals(NOT_FOUND, VerifiedConstMap.load(vm.toByteArray()).get("x")); + assertEquals(NOT_FOUND, PairedVerifiedConstMap.load(pm.toByteArray()).get("x")); + } + + @Test + void roundTripsAtAnOffsetWithoutTouchingTheBuffer() { + List k = keys("r", 5000); + long[] v = values(5000); + VerifiedConstMap vm = VerifiedConstMap.build(k, v); + ByteBuffer buf = ByteBuffer.allocateDirect(vm.serializedSize() + 20) + .order(ByteOrder.BIG_ENDIAN); + buf.position(13); + vm.serialize(buf); + assertEquals(13 + vm.serializedSize(), buf.position()); + assertEquals(ByteOrder.BIG_ENDIAN, buf.order()); + buf.position(13); + VerifiedConstMap back = VerifiedConstMap.load(buf); + assertEquals(13, buf.position()); + assertArrayEquals(vm.toByteArray(), back.toByteArray()); + for (int i = 0; i < 5000; i += 7) { + assertEquals(v[i], back.get(k.get(i))); + } + + ConstMap cm = ConstMap.build(k, v); + assertArrayEquals(cm.toByteArray(), ConstMap.load(cm.toByteArray()).toByteArray()); + PairedVerifiedConstMap pm = vm.paired(); + assertArrayEquals(pm.toByteArray(), + PairedVerifiedConstMap.load(pm.toByteArray()).toByteArray()); + + assertThrows(java.nio.BufferOverflowException.class, + () -> cm.serialize(ByteBuffer.allocate(cm.serializedSize() - 1))); + assertThrows(java.nio.ReadOnlyBufferException.class, + () -> cm.serialize(ByteBuffer.allocate(cm.serializedSize()).asReadOnlyBuffer())); + } + + @Test + void detectsCorruptionAndTruncation() throws IOException { + for (String name : new String[] {"rust.cmap", "fastconstmap.cmap", "rust.vmap", + "rust.pmap"}) { + byte[] good = resource(name); + for (int i = 0; i < good.length; i++) { + byte[] bad = good.clone(); + bad[i] ^= 0x10; + assertThrows(InvalidFilterException.class, () -> load(name, bad), name + " byte " + i); + } + for (int len = 0; len < good.length; len += 13) { + byte[] cut = Arrays.copyOf(good, len); + assertThrows(InvalidFilterException.class, () -> load(name, cut), name + " len " + len); + } + } + } + + private static void load(String name, byte[] b) { + if (name.endsWith("cmap")) { + ConstMap.load(b); + } else if (name.endsWith("vmap")) { + VerifiedConstMap.load(b); + } else { + PairedVerifiedConstMap.load(b); + } + } + + /** Rewrites the FNV-1a checksum of a serialized map after editing it. */ + private static void reseal(byte[] b) { + long h = ConstMaps.FNV_OFFSET; + for (int i = 0; i < b.length - 8; i++) { + h ^= b[i] & 0xFF; + h *= ConstMaps.FNV_PRIME; + } + ByteBuffer.wrap(b).order(ByteOrder.LITTLE_ENDIAN).putLong(b.length - 8, h); + } + + @Test + void rejectsInconsistentParametersEvenWithAValidChecksum() throws IOException { + byte[] good = resource("rust.vmap"); + ByteBuffer g = ByteBuffer.wrap(good).order(ByteOrder.LITTLE_ENDIAN); + int segmentLength = g.getInt(16); + int segmentCount = g.getInt(20); + int[][] edits = { + {16, segmentLength + 1}, + {16, 0}, + {16, segmentLength * 2}, + {20, 0}, + {20, segmentCount + 1}, + {20, -1}, + }; + for (int[] e : edits) { + byte[] bad = good.clone(); + ByteBuffer.wrap(bad).order(ByteOrder.LITTLE_ENDIAN).putInt(e[0], e[1]); + reseal(bad); + assertThrows(InvalidFilterException.class, () -> VerifiedConstMap.load(bad), + "offset " + e[0] + " value " + e[1]); + } + // The pad word (fastconstmap's key count) is ignored. + byte[] counted = good.clone(); + ByteBuffer.wrap(counted).order(ByteOrder.LITTLE_ENDIAN).putInt(28, 12345); + reseal(counted); + checkInterop("counted", VerifiedConstMap.load(counted)::get, true); + } + + @Test + void rejectsBadInput() { + assertThrows(IllegalArgumentException.class, + () -> ConstMap.build(List.of("a", "b"), new long[1])); + assertThrows(IllegalArgumentException.class, + () -> VerifiedConstMap.build(List.of("a", "b", "a"), new long[3])); + assertThrows(IllegalArgumentException.class, + () -> ConstMap.buildFromHashes(new long[] {5, 5}, new long[2])); + List many = keys("d", 1000); + many.add("d500"); + assertThrows(IllegalArgumentException.class, + () -> ConstMap.build(many, new long[many.size()])); + assertThrows(InvalidFilterException.class, () -> ConstMap.load(new byte[40])); + } + + @Test + void valuesUseAll64Bits() { + List k = List.of("min", "max", "minus-one", "zero"); + long[] v = {Long.MIN_VALUE, Long.MAX_VALUE, -1L, 0}; + ConstMap cm = ConstMap.build(k, v); + PairedVerifiedConstMap pm = PairedVerifiedConstMap.build(k, v); + for (int i = 0; i < 4; i++) { + assertEquals(v[i], cm.get(k.get(i))); + assertEquals(v[i], pm.get(k.get(i))); + } + // A value equal to NOT_FOUND cannot be told from a missing key. + assertFalse(pm.containsKey("minus-one")); + assertNotEquals(NOT_FOUND, pm.get("zero")); + } +} diff --git a/src/test/java/io/github/fastfilter/jbinfuse/XxHash64Test.java b/src/test/java/io/github/fastfilter/jbinfuse/XxHash64Test.java new file mode 100644 index 0000000..15d7ca2 --- /dev/null +++ b/src/test/java/io/github/fastfilter/jbinfuse/XxHash64Test.java @@ -0,0 +1,115 @@ +package io.github.fastfilter.jbinfuse; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; + +/** + * Checks XxHash64 against vectors produced by the reference C implementation + * (tools/xxh64_vectors.c). + */ +class XxHash64Test { + + private static long hex(String s) { + return Long.parseUnsignedLong(s, 16); + } + + @Test + void matchesReferenceVectors() throws IOException { + byte[] data = WyHashTest.referenceData(); + int bytesChecked = 0; + int stringsChecked = 0; + try (BufferedReader r = new BufferedReader(new InputStreamReader( + XxHash64Test.class.getResourceAsStream("/xxh64_vectors.txt"), StandardCharsets.UTF_8))) { + for (String line; (line = r.readLine()) != null; ) { + if (line.charAt(0) == 'B') { + String[] f = line.split(" "); + int len = Integer.parseInt(f[1]); + long seed = hex(f[2]); + long want = hex(f[3]); + assertEquals(want, XxHash64.hash(data, 0, len, seed), line); + // Unaligned offset inside a larger array. + byte[] shifted = new byte[len + 3]; + System.arraycopy(data, 0, shifted, 3, len); + assertEquals(want, XxHash64.hash(shifted, 3, len, seed), line); + // Direct buffers, in both byte orders. + ByteBuffer direct = ByteBuffer.allocateDirect(len + 5).order(ByteOrder.BIG_ENDIAN); + direct.position(5); + direct.put(data, 0, len); + assertEquals(want, XxHash64.hash(direct, 5, len, seed), line); + direct.order(ByteOrder.LITTLE_ENDIAN); + assertEquals(want, XxHash64.hash(direct, 5, len, seed), line); + if (seed == 0) { + direct.position(5); + assertEquals(want, XxHash64.hash(direct), line); + assertEquals(5, direct.position()); + } + bytesChecked++; + } else { + int sp = line.lastIndexOf(' '); + String s = line.substring(2, sp); + long want = hex(line.substring(sp + 1)); + assertEquals(want, XxHash64.hash(s), line); + assertEquals(want, XxHash64.hash(s.getBytes(StandardCharsets.UTF_8)), line); + stringsChecked++; + } + } + } + assertTrue(bytesChecked > 600); + assertEquals(8, stringsChecked); + } + + /** + * hash(String) hashes ASCII strings without encoding them; it must agree with hashing the + * UTF-8 encoding for every string, whatever the position of the first non-ASCII char. + */ + @Test + void stringHashEqualsHashOfUtf8() { + char[] specials = {0x7F, 0x80, 0xA0, 0xFF, 0x100, 0x180, 0x7FF, 0x800, 0xFFFF, 0xD83D, + 0xDE00}; + java.util.SplittableRandom r = new java.util.SplittableRandom(42); + for (int len = 0; len <= 80; len++) { + char[] c = new char[len]; + for (int i = 0; i < len; i++) { + c[i] = (char) (0x20 + r.nextInt(0x5F)); + } + String ascii = new String(c); + assertEquals(XxHash64.hash(ascii.getBytes(StandardCharsets.UTF_8)), XxHash64.hash(ascii)); + for (int pos = 0; pos < len; pos++) { + for (char sp : specials) { + char[] d = c.clone(); + d[pos] = sp; + String t = new String(d); + assertEquals(XxHash64.hash(t.getBytes(StandardCharsets.UTF_8)), XxHash64.hash(t), + "len " + len + " pos " + pos + " char " + (int) sp); + } + } + } + String pair = "abc\uD83D\uDE00defghijklmnopqrstuvwxyz0123456789"; + assertEquals(XxHash64.hash(pair.getBytes(StandardCharsets.UTF_8)), XxHash64.hash(pair)); + } + + @Test + void heapBufferWithOffset() { + byte[] data = WyHashTest.referenceData(); + ByteBuffer slice = ByteBuffer.wrap(data, 7, 100).slice(); + assertEquals(XxHash64.hash(data, 7, 100), XxHash64.hash(slice)); + } + + @Test + void rejectsBadRanges() { + byte[] b = new byte[10]; + assertThrows(IndexOutOfBoundsException.class, () -> XxHash64.hash(b, 5, 6)); + assertThrows(IndexOutOfBoundsException.class, () -> XxHash64.hash(b, -1, 2)); + ByteBuffer d = ByteBuffer.allocateDirect(10); + assertThrows(IndexOutOfBoundsException.class, () -> XxHash64.hash(d, 5, 6, 0)); + } +} diff --git a/src/test/resources/constmap/fastconstmap-0.9.cmap b/src/test/resources/constmap/fastconstmap-0.9.cmap new file mode 100644 index 0000000000000000000000000000000000000000..4b6959dd2e624ab61cb28b634a8ac00487adbe47 GIT binary patch literal 2600 zcmb7_%}bPF6vf~CYzm1Ii%h~yNf?FE2!c>%5HT^$1{sP5#f2iGt`q{pl^8RNLei*( z^A|{~OmX2?pw*&9;3jG_ir8S(A|Z6s2*0zK%M33wAZO0K=bn3?`@Zwo)E?;c`F!>3 z;he}p?U`x6<2bp=KmRGm`Itoe8BE5r`5c^>rTKUYQiIDmsoahAxafLo_hs+ocxLk;05F_(6(_*3)=;tlM(0sfwb=QGSbjBWrQ%_ofM9Q3=0&lTpMr|uazWzHLeU&AMaaV2~P z-(%tf%=4T+6X<%0-$8#FJ_mMD{}ybb?k{t#h@=0-o^<~Ni9cZ8W$F&W4d&^97on$} z(*{2V*XVZ|K7z0mpAdP~cW|yop=ThQTV+r=R|&%+dXz_h{18Rlpa)hs?DG53y(YcG9N~ zT>yTBt_2>1YyVF;SAzFt7(B$sZ}vGuJdRHNYtZT5Qe5bvPF$$_TwK3Xap4EkPtPe# z-#zxy&UsWPyutY+`o)265Z;HEP|ph-ZTg^JGU2()0S||K4Hf|B zgZwP$s|fyEA&(FV?^QYI{slfSQGf`#5kL$-$6o*?p$~?i5%d^GlmUG#ybr;Idepf< z+!N*#1H4AyBG9iF&LakUI08HZ^g|HWgMI4(FN20Ne>1l)=)AxO=R?fFb3i`{@R5f3 zi=ZwBkVQCe4scoUaR6fl;3?qy8RA|rPXhFb1f46ySAbp$cs8IN)Sm}5g1S+ds|Gpf zU&Eg8{;`4hT9~&A>NtS)Fpm{*VbDW6k_GV3fVI$10Jt9r=YWp`I7<2}}$3bTX+y*%AKNQY2 z1n-M4AP0Q(3HwZe_%qO9e^t=oy@hc^L8wEHi1#^i{5wUC*g)uq&xrwjpTl0(zG4Ej#tyOHq<;#he9R9>zE}T( zn0W5O(iBtWNMO9mL$=YiSM4kEsqn<|%PjdF(JsH!-ArrQ=6~eD|tGRnxK#Q}<<@M2zF^`x&D&MN`#udM&V0aQI+B0^C z%2A9lApYJ^^s2jpC2}IRv%_lI4%(Cu?--=7?QC@SAbeZt$~fAwQO3u*2g zTkD4Pv=#Ka>l%B)%<}yCH5bIIzZ(#jh%w*gh6#-JVVJyvT9r7uF4!mzD;}~ zjnbhq`PX|v`;xrj_E+kdC*ofR!|H1Jcj<7QBAq!Ql2$^cocJbifA<-p?l8-jLRZ`S zAYD9&6G2Y4aQ2K=ER{5I8n)bXOITWWVF0~~7Pyq=^WI6SvpzS)*}SyLR$zL8xoZ8z za?&{0u=VIjis(Ac{#D{$2g?gr$lhkFIVUE%-sHUfuqZ+4n(~lGTukp5rJii+S3b6Z z!@IAcMbTO(do+o2Fm~kfO=!hT=TbYXC3EEITw-1vaU(l|kEEVr zcZAEHct18HvrmuHex+)V{4rxopN=de@;D<&d!Njn_`xGm=%ev)EJb=HB)}w$y8#XCHNOsz-sbh=zpLVJ9oW$~mm-nsM_skIH_nYEhpv?Uj$K7auhFqzTU5P%kb(I@luyDp5A2BW%b%3B&Ml#L8X&q@Z>#OsC&%wjaiue zhfwY?kHEPs!|WQpYpS+)BIr!1H1g#<>v>f1hKMhI{%5^aDV2k|b{Z|p`vZhxJ`Jlf zlpobl-p8wm1-cfgke_CD6#7}TterjnY{3!X&B~NaWq(_hQ#e=m$V~mivme}CSxNSj zNLAm3@hXWLgXW>&w4))an?JL71u%{n(%aYE6tepmazqWye7+pxJU!&e^*ghQ-@kZ- z9Fc!jhTI#AP%3gw^2vfGo*mb`omHwF+_Nj>Xu-<`FP*K|WvPkZ*Bkxmdx5T&(%p?; z7|PfG88EKgYVKrTDR_P9u7IC(yio%yi{>8uX!Xy;K&hwfdVOjp?Sg;!C+tf^OiM;KjrvB+sik9UF6u>BoVY*>H{0)BkV`%B@mv(f5p`20 zT36fNoV;IED!zQQ=nAuPSVT!&{aEiV>JWP@Uv{)jyMbRfF-)pn93*LyAeB!RTHdhN z#v$@*9hELvC9vDEZiBv9e#mW~40_(7=z-kEJqZ=}S{^g8aK}s?&**}DGvuYrP^xcH iup)BA5bN$*azH8#dCx>lCZ+!z2RS9b&Cw}_x9vZ-qilZw literal 0 HcmV?d00001 diff --git a/src/test/resources/constmap/fastconstmap.cmap b/src/test/resources/constmap/fastconstmap.cmap new file mode 100644 index 0000000000000000000000000000000000000000..a4eae7ee8742c428a1342e7b95dbca8601feebd1 GIT binary patch literal 2600 zcmbW2Ur5t&6vcmYRvKE9W`q_>e}sl!{PAe zzF5|URK@bkQP1=I>7S_2^9IuRI6M7o-sjn}-$CuXT&6z<(*6JQdALa5p{&1CtzVj% z*U$W!tL|*fl=948_L=%eQ&+id5&sn;{v(!~y;Ut)SfIu5#V#=UA7+nl9rLgK^=KCO z#Ep)q4;=Y8_Z-3Rk@5l(o6GUI%*Kzu3`vWud zY2QBLDEtk~cL3kl#6|cNqVxjnp}&>7z8`+x%N{U}UNd|R6u?*SvB1>D3zENtP7=R4 z;?Kn5cf-F({UC)JaG1GX(bq^`@4J|MCp^8!kPIA{PkxGXy&-Pqer}^XiLPW~%$Y|$ z0KbP=yd>wkL4E?fh2Bx_D+I3}e2U(8_}_>V)ZZJQ>n4`J?n&_x=2m>4xPkNQeoKjS z@Y_Z_%iODprQ^bB=E`RNE;HvGI_uC4vX65P!Hx7^HU91NsjY}}9mZFA>t90s3%a8a zPk?1=H24lb{r}TDInG=u_;sc)!CcSD>z+?>{%-ICbM=~Y-lMMp|7)f$yY=YA&0c04 weDw+S|4;FL^iDF@Ebn4B=V>)QBkZ#S-dXykCv4aL(J29|cWj;dbf^CGPgUh^0{{R3 literal 0 HcmV?d00001 diff --git a/src/test/resources/constmap/fastconstmap.pmap b/src/test/resources/constmap/fastconstmap.pmap new file mode 100644 index 0000000000000000000000000000000000000000..6fe3f43a5f5e3d09138e4911254ee713851fcf50 GIT binary patch literal 5160 zcmcIn2{crF8z}zSrc(YdC%6yU{W9(z+wd;$LC`x54Ns^r?ghH0)wIxY> zqC_&1v4rHcv|!5md5>r6oZIQB^S$4kGxy%#{XNg`S^mqt*U&)6P)SKixz=h8c}#I# z%pN8tCYI&@9%d${#^rPA+U3LYuUw!x1Iw#-as_=m*0HZ#s|&VXQc21!H2{i#-{cE9 z#f1&uEPw8*aXIeazjs~O;H+Ju$}8yqcPxf@f9~LrE4<)_{nbe3YgRnGjyO}ACFV>Iq!L}WBl*=ZHyG4K z%A3a8*q5$m|G)BEVZJE~_u@CwyFMZH)L2*U7|agV%nyIeMDnZQ0ML(^?8~Mpm=&yM z&kOc-3vIVtsq6ws`$d?KM?E`aZHtG(k$eyIKf4DdOVG?LLh?u~^7rPl{3B566`sxe zx~-s}KSo17RFl@j9g~AHt9j;uy{mF`vrvH9y4CEXVLjilLgSn%MOqqAUtK_t>zf@K zM@u084=N&5mD`E@NIhogXHvc94E62Mcl+fi^T&66(AD8RxAnXdsQ$=Ye{k`e=;JrQs}*{6?6sIqCI{;$$gw0OWtZ z>A6GAaKHzoeJu2c-_XXU@S$`7l1Jif(3e-umfy?LOJVk%XL&@y|Jj{6DO1}?c@NN64T!QaBzc-_2Rtp1F`1W0KG6nxFPaR|UB`z` zT?hYnTb@ZsP&Ux0V1Jr?$pKTQf#HXE(to9*39(NY+v(S`82fXYokP>~6d~(hz!c7j z$wpx@n6HLExafd*FK&c}e!6QMfix3<`Lc$!O1WVF$aXi#zgXbbUcCKDW+iYax@;yn z);`q233@TS2WfXzW2H9%#>y#l)^c0dBK1RHJutb)P{w(;<*_i<@9CQ11~53OLKFGzP{80z zsOQAa&mUzMDdt~5-*fFm$6noG&v@Y3go$+;;;dX^;IF(bHlf~VHjS~L(h_bAz1->k zv<2cl5lVT`t<=Y=3-+I0uuyQgT~FJfzMOz|J|FE=(FCw(HS0{|HWo+!2KGm^MyJ^6 zUo17Cf17tyxFzVx+;RqNZFv20zZ<4j5#~>1qaVLn{eSnSt-_ADDA^O8 zJfI5dwP)AqSn@rAOi6=$cKYf4wdR_q!hp1382Debe?TehDUxP9KeS5eBt8yB zI~LeqqKgiutHnoA4gxNgzM!tIDbpVf`H`<$N9m+CRUZTW-w7UaseRG?$VTx%y{F@L zo#Kzc)-v`-*B8^O5%Jv^UhuCo%<<9|CawCufdc!`?oNR_M}pCpAs;Kcn{G3iUA_V8 z>y+W1SFtNdTmoKF<`W0q2;Um)F`}~6dbtS{5BkM^6UhmvM#+h=j zRloNb>q*mMvhk&d6dtK(4gJ2$EkHh+#C03V=K|m2Q1<%cxWBCcd7ya;DU|`|bw*KUx^IH+B|yI`lhqM%v5qgYz0e z;760ShE8qmyMoN;SMf0P%hFU&UL0$0w-)+OH0qsdt&G8?0DpvQ#)B42Vl3aUVU{4D zB6`EFcl&+XFpw`vS5D;*hi$Sbm><(Ru6n+UwYM&SXXSt<5q(cf;j7ROE@UCT96DuO zL^Ftz2=Rm)1n?o7@gTB8bo%QLwe(-S^)DvjKH1o#5@)0sWD$LKk z)Hk@;!M`jLK=0Z^CNurCkhpUYzc=?>aw(U`E~Gya$3cG2p4mS1)3ta%1Nq3$IOpPR a@>bq~-k^cfyNPFb~ literal 0 HcmV?d00001 diff --git a/src/test/resources/constmap/fastconstmap.vmap b/src/test/resources/constmap/fastconstmap.vmap new file mode 100644 index 0000000000000000000000000000000000000000..b5da57985e30a054b4f1b9e09e1aa659aae28d84 GIT binary patch literal 5160 zcmbVP2{_dI8YWpv$ugSAM6#ELOea}dPGzp7$Qb*+U0ZacB$Y~Kq9n;0Q3!=BInpLc zS1KhL$&ip7ON*tfr~BOR^PJ~@np5}A^YG65eed^v-|zST{pUZ!G+u9_p`oE!VLOA> zt3Eq=9S;u=-?zVYygWP)zhUvrZ~y-EsucV$G5%BRdv70ptN(vqg+HjXn(=e4P6FT8 z-G}+VpXDC4C8XvM_Ju*n_Y?A%KKXwHP5jN!Z60F7wO$ z=g|DkD~jNQ(i|@FD7el#(65N#&r6u|IP7;X_|4+7|Jaus+7Q+!O6WTe_BEK`A48Z2 z{WC%T8uB`jXdu+1elDz=PVh?veFe-T3wa;VF9&=Mx(kqJLtKKe4m{8Qz{Uh@fI19I z!oGNczXl{`fo=goe+aGvA0Q3t?*iULW58sXuNwGE=<^lyalqvukLUC;)EPkhIrNK% z_%9IGf%)74)nR`~;E(q!2kN108oHql_nQ~^I?yY@d`#$D2V5HZNP^QNz%r;0hdkaN zd~ja207ik|F3{sZqRr3h`FJH1KnP z>m>`i`+&E=ZvymVz+)i)p3o6EbVr~b{Saeu|G`HAuuQTxjVfli7j>)W*PYTz z%r2JLIPgM6eFAS=Oi$zcCTeKL8d=Thz)!yUZH16Z{xL5qH~Eh3?xnTz<3-%^7-!ih z^o8lY?MIF{E6i2DfOEJE`Oji6$*}f#_u^R8pFS9JyxTwR4RRx+0fnALJ4nc5_^*1K zbZpZ_j_9X+J53?MV>xmnRz|8?#5gz?6XQSPn$oTGc_U4qQD;2bac8{iQ#x`YMhvuN za?~vIi1Esa=a)U%g7~=N+3DM$>{!@&0(F@!BxQzfdI54oMroDW)A0Z=%M7SFdUbasB!l3{CINsvHYVwLVN`(5Ysxh(N9 z4cpZy>c%NK6&~0!=h7CEY>kz-mOAQx7B76E`RUI$Rz&?1f1Td36x#I1M^{!eXB{xl_6;hcvA@_1FJbHn&%1lp zTMl)Iuhdu$yK^TuP9=rA8AeV`-(M->BFPFu{%ohVQ{m2yhee-W9aFGy98g^ueA2;@#ub;}u7m@05SYer})3 zLDXM;kijyfQ8M;wab$kk8r-pG)t67olciR^Gy5Ah7V<+p515HVX!-mcd(C^+)fX5oq( zUh#%Xm+ee0Jb2?psfdhJy>s2>1SK>xz}w8LK6gs|liir&HT{N5LOn9^d{Q_UVy=9W z^~y1e_zySw#>2`PPnzGfk9m1H3%D6=7@XAb(6+bG&nrx(i^OH8z8bAsTkP8F_~y(F zCYhHvAVY-XA(mIAKRSE8jktY>xRcRpDTcy~bG~Zrit~><-!08=DNxia9SM(NA+HW?~CWEQ8cH36E{piY}aLaywnTCL0CymI4H%6a1uw6UL z-QP@%2y$+VFe7?3)UpKZCe~(@>mF^%Z!Q>j?tA25bi2-W!!6z~^Ck^K-r;@{(Yb5s zjUKisgB`Jeq3Rf!zhJ$(^?o}OC4cJI-6M>uhtvxvEX{J445b%_JWjh(QG)f)S|#FM z6&V_J4kxnfJ^d#wZ?pxY+27S1pjfJ7=Vyd$f)3kN4t~{EyU3l1s6KaoN?|BMFs@J^V zKt=1HF3XgM)_WQTOjcYjZ`Wm#uj{E1eO1WGeKNZHOQowN>4z5NPJin-96Lch z*Yl_l^&T6?XPY*7W}WcjTo=1(z{0rDompfYj68HFI?MHp6`OgYh5NeTMpy)dxBoi=_=cdoEtI?gH}gqhD~2zCi&Kguh63B zapZ9_pZ>xnqjIMQO3ziYI_nbKMpQcT^#|2q_iVa@N|mgD`+eccYC83tM3T{Umvw&t55RlD literal 0 HcmV?d00001 diff --git a/src/test/resources/constmap/rust.cmap b/src/test/resources/constmap/rust.cmap new file mode 100644 index 0000000000000000000000000000000000000000..fab75ae6e16f5ec9f04b1827fb5d87c9a7021355 GIT binary patch literal 2596 zcmbW2OGwmV6vhAJ*u&BqX^fC!42Fur_zMv+n-wC))r@-v9aBo2Tz^-ruPnU6`BK z&-}To?rg%83e8^joBC%{SNYsL{>w!CN8Dugwr#=O0>%r>&Pd$=NNwXjefw)d4_#w&?|84U%oD^G3N^z-{tJJ)##_pJkq}f|0VSpg;CQl zei7%cGTV?nsxrZRQ zh5lafk%Yl6=1UWg;WG{Y5^))Iz0*PZc99>!uY>$n@-gPC2g}({7=3+TU(-+9ADEy| z`wkGt;jd%9gZK^;m*P``(g4^)e;ajuKZ3lMyx;hJT*=Aqth?7;_ELS5IE=yNrA%JiW(=3>;WQ{vGFfPTa!%+(h>+x{^sSXCd_v z{2pTQ(wyrW`8VJV^lG`U2)usqF?!$OeNpW*}P9cQj7-o+ly(`J0e*=Gm5Q}jzu*slGf69p>{z!XA zj^#TvQsP*s?(1`3b3~$-aEYWOkrte&}N+l3xi^K|f-mH=ChsRj`shKiJnT zv|jh1a|$8t7hygg^ln#hC>{()^4-w?>~53-K_{~a$s@7&e>az}KSFuE^3!>L$};-( zV>IMLw`kelJ~1G_l4l;+d#FV>iBhfBtYjYz>-nl38s}DCq^k|})ls`WUT@VtSOWQf zP#34G-$)ch>ajvUlj?P*>2C&q*e}n){4Vn{3q91q=Q;}cG0M(|1$3{y#tr;lltiPo zr@9JK|L?}OP~Vl$IA1+s6FH^wy#oI(r)usK7P;Qq;1F1Xqkp!YmT8Ch zgd<@E_bz!#BK44X9gO!&Wg7W#{^Bk?=r1LQxf(?-{qX!s0{t`r-V|9bKYA_n$FcW( zm{Z3oJ_+!DdUH;e*t~()#L3u4Trqrf#(n-)@e(z^GJff+UD4Vdb8Oy=6=dZxH|Y^o9B~ei1%1D zG9cX@&Z7sBHp%dZ`i#V9##Vk;rQ!Ph_N0&28Io3<~lO&i3~-r?llWpRY>-AMNMlZs{=~ ze)~195%B~zQ3&_b(=f!#mrl0Pn6pe^&hPh?WAmX^geR=PclGtpQ7BFvJ3!ync<1Dz zs=KB<^p9kzS5TaS=OseD@ww?QXY03@wgB%k>d<$wl86?AdRSTgvv?Wq{LBYbLA~DW z8UtH_$B-#$K)}&hOO#Ln@nu(C_Ezqf6AlCab58fl3%iTtn9mR0(ixHfmx?1B>@U$p zd($=JqssRJE|xp1rKKa^7Y+H*F4;#Jq&D6;1p0pxycAP=qx+DJ;)8lm#OWUsjKJ11 z_eYQCQyLNRT^N4wuQSc@(Gw%B_`ZPx`_Qfqp_>PS(H9^eyBUgM6OB`;0qX0J=bcw~ zEJ$1eUPk@csfT$No2P(3{a2Hy%ln7S*PU-^=YxKUAL>j@#G=`G@b|*;4{seU##n)6 ze{~mbk&4CPnb@uePny-3dI9|kvl{mkZ}6xn%B^o73~<>?)dfV(yP)kIBY&TkLj9fZiR}&+8WrgWdrA z_DZ;w zdA`h4KjfQP9Mqg~qqYp0h#mCvS&v}+%Mv5wwctNEo#d+RV04xb^liJ39=aovo?Zd` zQi7Iy+*Y9pq+SDzS7L^r@yXfT^mMQfpG-U3v%yfO2QbTxW18zZWce~6KkB%d3x0Mx zo($`Ye!`kudMkh~0ea*23vzsl)>%FPtg1`4928eExC8mUA3Z#2%2Q5U0R1HlF!3D-YJ@!nDXC9h|xqkT?B#Emp z`35KgKOUlB!4+td$_4ddqa(Rf>OB3o1OIPbjOJ@c8~hCPJ9Jvk$K`|DDq-M9l641< zZR)*<%;$IU5cJE|(nv`P>*Tl^`cE|LnQWJ|IW|ylIE8VWzuJ(7v}e}O zRm1K2?n78Rl9vU4_uau8aifzH2PGSE->Lc+Rs4pa%`XSJ|tdaTsXCp=nY-+c*V5sXTE?7wTTk7&EEd-{TK`V5y)Hvp|=* zo`Xv&5s%xkPn;qysSg##;j=2 zYcP?psSsKtiL@UEebSM@S8InV_8|G+#mw(NqEVn$gj(ni>(>JEBckujcyn!R4S;^d zqvz3VdK+YBV857^yf9IIxsGNG<1^Tj)H|az@GuPO2_myhzc#UbfXt5$^D{5?H7<7G zolOGhUB1s^WtN>Zs*h9pTYi713ZC|kNo z(h((6Q!+|OwoBz!uDkQ>wdQ&Dzn|H4=6QJMUF%)%THp77|NJvd3lmFiZEc-Orx~nX zt=VyF`1tq)zW=S^|Nig$t90g{i2ti*{rS!U13d5%)bmL53>W{P&VB~Z(!V6!h5kah z6jn%7enFyiAxCkrYHE%2B8E!swD{VDZ)Os@|a)l ze~0EDUa$HH|sfPN(ee}2N8$6>#F!B2$8{$pQW=tx+fIHB)6*w;{ke>`Cx z^v?$U8^{|#qM1;S`nj-fI>9dm^p!A=JmdpFzYMSubQd7chPV`A9eAGqg^dZ=40RZm zhJEn^e{D#JfNlXne+aIF03Z$O?*ZOIW586HuLk%F=<^NqiNF;gkLR==>P#Wt2>p^E z{tLtnU_Ng^E!ZCt_~ZS`fqE$8LO0ame)9ug1A0}Mj|qJnfXhN3X>fWBSPu1(kjMK& z0M5%sz*z9x33?nz9QxvUlpy3$M}qh%@Tq}*)4+#;qdp(>Cn0YK2|C~lnCl7Dc|#n} zw*tgdK!@j19vkq$xe)J!b-e<<6RuAt__l*D+Bm_Sb09Ac`f}i?tATZ;L%aa! zddY+C0pM-$n*{wh@Oa37B=kumaO{uQ3Ar=OjXW2)C#)Z@uR3sH=(ig9D9k+{IQsCy zE|_a3%%4h_a~gao;7fx2IL59G z0R8a!hvy^^=IRB#BcU!H=BkA_Ugt1ae?DLj%yo~j&K#)og#PCUd2F{Bd}0av(#=DU zb%^-5^bVr~b{SXsy|G`HYuw1$>gDPRR7j^pc8&2w` z6*W9-mAno&@RM(TTQ00#c+8K=OTKN3cUhg{LLvC(9pxm=)8wq*5;8lOi zj?IS15x1+|$xu%AS%#d5RVOrT<6S*UiSZwC9oaVeywO*KsIwUBxI5AHB@;OjqX*t* zb2RM=i1Dh)#>>9!S@^i(+Y#8T=2qMpg}Urkk{ZJ>vj{mNqpVu<>BKHS3MlWMpvboJ$)?^0oH~6JS-kL-w%vHXg$nAQ1R3;>r_s0{A35pOj~26( z$wzCSZN1wS`WbanmaeXa_fyucT=i9Yo^t9x9(rG*Q;ea{;68|2*9=}A=eD}gbE_3^~ z9z^}shgmE$8YOG5E=TT{O`#ooR(}1WHdSUHIJ3Wbt!Y=FkSw#xV6mX6&eHg#YOAp{ zPSVBB7c2drJeEnl*P{JaaPYZ68I88;Nls{!WRUTBc|p6E$zs-v$ySbrk3lHDx^@Z$@$vbzcK>U&%cf3e%>5NShY_onp)}me0W84Vhg3p zGH08ytjpg%HzHbIN?+X)PIemgGp%FI(s-#VHlgcYkgTM;@9@LC)AdJwtCR8k^pN|r zvm}UgG%k9|Du5=@y}Y-W(tfLjHWf;xwL7)ZUB*@nN80rV$u;l#b;_J5U|*azHCB6@T! zx!J?EVz8qZFf`mw6fQ8)a@g-|sTxH6wtJLO{fK%Y%Fa4pX*jbutUlvP<#nuo)+Uws zy2Q-9b0o#fz|HKKW9#0Xw-=IyUy!q{Xs)fck*jT|oR-O_-rYl%qGikvwvoF76e_ge zNC@7D$X1=>Gi%c{DJpo7UoWyGMW(3*`h%^`5V2v}V=E z^+Am;rs}%?-nvw6*u>Xt*Hq=@igrUL`G%1u(N~?E+9zkYzf879nto_OKKEP4;e<)* z*`CM6sP|bnG25!yHz&%Eb3@{mDGTGmcSXE@{~Xzj+-H6G!rUzWr7DR&UIPnTFWOkU zTd$KFT)96p_}m6N557z4P2sORBL9PW#6z*6pF`}g*E-0uLYivA^fVkNVhp#ZRrye5 zbAt|VUu8_rj0=cs+AQ59!!)q^$bCM}-=7`Fsi&N=pJethG7F5tOKv78_T1e>lA;>Y z`_?Po*g&~zqqq6nyhuGOMS)8$w^VI6eBoF6ihYao8WJDv*KX|S@nY`mHZ8ICzk9G^ z<;oqHM--5oP ../../src/test/resources/constmap_vectors.txt +// +// For each size n and key set, the keys are fmt.Sprintf(prefix+"%d", i) for i +// in [0, n), and the value of key i is i * 0x9E3779B97F4A7C15 (mod 2^64). +// Output lines: +// +// where kind is C (ConstMap), V (VerifiedConstMap) or P (PairedVerifiedConstMap). +package main + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + + "github.com/lemire/constmap" +) + +func emit(kind, prefix string, n int, w io.WriterTo) { + var b bytes.Buffer + if _, err := w.WriteTo(&b); err != nil { + panic(err) + } + raw := b.Bytes() + sum := binary.LittleEndian.Uint64(raw[len(raw)-8:]) + fmt.Printf("%s %s %d %d %016x\n", kind, prefix, n, len(raw), sum) +} + +func main() { + sizes := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 31, 32, 33, 100, 200, 255, + 256, 1000, 1023, 4096, 10000, 65537, 100000, 999999, 1000000, 1234567} + for _, prefix := range []string{"key-", "clé-日本-"} { + for _, n := range sizes { + if prefix != "key-" && n > 100000 { + continue + } + keys := make([]string, n) + values := make([]uint64, n) + for i := range keys { + keys[i] = fmt.Sprintf(prefix+"%d", i) + values[i] = uint64(i) * 0x9E3779B97F4A7C15 + } + cm, err := constmap.New(keys, values) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + vm, err := constmap.NewVerified(keys, values) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + emit("C", prefix, n, cm) + emit("V", prefix, n, vm) + if n > 0 { + emit("P", prefix, n, vm.Paired()) + } + } + } +} diff --git a/tools/xxh64_vectors.c b/tools/xxh64_vectors.c new file mode 100644 index 0000000..d34ff5d --- /dev/null +++ b/tools/xxh64_vectors.c @@ -0,0 +1,45 @@ +// Generates XXH64 test vectors from the reference C implementation of xxHash +// (https://github.com/Cyan4973/xxHash, xxhash.h; any release since 0.8 gives +// the same XXH64 results), for the Java XxHash64 class. +// +// Build and run (from the repository root): +// cc -O2 -I path/to/xxhash tools/xxh64_vectors.c -o /tmp/xxh64_vectors +// /tmp/xxh64_vectors > src/test/resources/xxh64_vectors.txt +// +// The input bytes are produced by the same 64-bit LCG as tools/wyhash_vectors.c, +// which the Java test reproduces: +// state = state * 6364136223846793005 + 1442695040888963407; byte = state >> 56 +// Output lines: +// B XXH64(data[0..len), len, seed) +// S XXH64 of an ASCII string, seed 0 +#define XXH_INLINE_ALL +#include +#include +#include "xxhash.h" + +int main(void) { + static uint8_t data[1024]; + uint64_t state = 0x9E3779B97F4A7C15ULL; + for (int i = 0; i < 1024; i++) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + data[i] = (uint8_t)(state >> 56); + } + const uint64_t seeds[] = {0, 1, 0x9E3779B97F4A7C15ULL, 0xFFFFFFFFFFFFFFFFULL}; + for (size_t s = 0; s < sizeof(seeds) / sizeof(seeds[0]); s++) { + for (int len = 0; len <= 160; len++) { + printf("B %d %016llx %016llx\n", len, (unsigned long long)seeds[s], + (unsigned long long)XXH64(data, (size_t)len, seeds[s])); + } + for (int len = 161; len <= 1024; len += 37) { + printf("B %d %016llx %016llx\n", len, (unsigned long long)seeds[s], + (unsigned long long)XXH64(data, (size_t)len, seeds[s])); + } + } + const char *strings[] = {"", "a", "abc", "apple", "banana", "key-0", "key-199", + "The quick brown fox jumps over the lazy dog"}; + for (size_t i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + printf("S %s %016llx\n", strings[i], + (unsigned long long)XXH64(strings[i], strlen(strings[i]), 0)); + } + return 0; +}