diff --git a/benchmarks/DBPEDIA_MANIFEST.json b/benchmarks/DBPEDIA_MANIFEST.json new file mode 100644 index 00000000..d596abcf --- /dev/null +++ b/benchmarks/DBPEDIA_MANIFEST.json @@ -0,0 +1,21 @@ +{ + "dataset": "openai-1536-angular", + "source": "SuperKMeans benchmark data_openai.bin and data_openai_test.bin", + "metric": "inner_product", + "normalization": "unit-normalized source values preserved exactly", + "dimension": 1536, + "base_count": 999000, + "query_count": 1000, + "groundtruth_k": 100, + "files": { + "base.fvecs": { + "bytes": 6141852000 + }, + "queries.fvecs": { + "bytes": 6148000 + }, + "queries.ivecs": { + "bytes": 404000 + } + } +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..85d2cf07 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,24 @@ +# Benchmarks + +See the [benchmark overview](../docs/docs/benchmarks/bench.md) for datasets, +installation, measurement definitions, file locations, and reproduction options. +Results: [IVF](../docs/docs/benchmarks/ivf.md), +[graphs](../docs/docs/benchmarks/graphs.md), and +[k-means](../docs/docs/benchmarks/kmeans.md). + +From the repository root, after installing the documented system prerequisites +and activating a Python 3.13 environment: + +```bash +python benchmarks/install_baselines.py --jobs 12 +source benchmarks/env.sh +python benchmarks/reproduce.py +``` + +For a subset or a command preview: + +```bash +python benchmarks/reproduce.py --benchmarks ivf --datasets gist1m +python benchmarks/reproduce.py --datasets dbpedia --methods qgkmeans +python benchmarks/reproduce.py --dry-run +``` diff --git a/benchmarks/build_indexes.py b/benchmarks/build_indexes.py new file mode 100644 index 00000000..15c19c9d --- /dev/null +++ b/benchmarks/build_indexes.py @@ -0,0 +1,199 @@ +"""Build ANN indexes and verify search results before parameter sweeps.""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RUNNER = ROOT / "benchmarks" / "run.py" + +DATASETS = { + "gist1m": ( + "data/gist/gist_base.fvecs", + "data/gist/gist_query.fvecs", + "data/gist/gist_groundtruth.ivecs", + "l2", + ), + "dbpedia": ( + "data/dbpedia/base.fvecs", + "data/dbpedia/queries.fvecs", + "data/dbpedia/queries.ivecs", + "ip", + ), +} +DATASET_SHAPES = {"gist1m": (1000000, 960), "dbpedia": (999000, 1536)} + +CASES = ( + ("rabitq-ivf-raw", "rabitq-ivf", ("--bits", "32")), + ("rabitq-ivf-1p4", "rabitq-ivf", ("--bits", "5")), + ("rabitq-ivf-1p8", "rabitq-ivf", ("--bits", "9")), + ("faiss-rabitqfs-raw", "faiss-ivf-rabitq-fs", ("--bits", "1")), + ("faiss-rabitqfs-1p4", "faiss-ivf-rabitq-fs", ("--bits", "5")), + ("faiss-rabitqfs-1p8", "faiss-ivf-rabitq-fs", ("--bits", "9")), + ("faiss-pqfs", "faiss-ivf-pqfs", ()), + ("faiss-opq-pqfs", "faiss-ivf-pqfs", ("--opq",)), + ("rabitq-symqg-raw", "rabitq-symqg", ("--quantization-bits", "0")), + ("rabitq-symqg-4", "rabitq-symqg", ("--quantization-bits", "4")), + ("rabitq-symqg-8", "rabitq-symqg", ("--quantization-bits", "8")), + ("rabitq-hnsw-1p4", "rabitq-hnsw", ("--bits", "5")), + ("rabitq-hnsw-1p8", "rabitq-hnsw", ("--bits", "9")), + ("faiss-hnsw-flat", "faiss-hnsw-flat", ()), + ("faiss-hnsw-sq4", "faiss-hnsw-sq4", ()), + ("faiss-hnsw-sq8", "faiss-hnsw-sq8", ()), +) + +LABELS = ( + "RaBitQ IVF raw", + "RaBitQ IVF 1+4", + "RaBitQ IVF 1+8", + "Faiss IVF RaBitQfs raw + RFlat", + "Faiss IVF RaBitQfs 1+4", + "Faiss IVF RaBitQfs 1+8", + "Faiss IVF PQfs + RFlat", + "Faiss OPQ-IVF PQfs + RFlat", + "RaBitQ SymphonyQG raw", + "RaBitQ SymphonyQG 4-bit", + "RaBitQ SymphonyQG 8-bit", + "RaBitQ HNSW 1+4", + "RaBitQ HNSW 1+8", + "Faiss HNSW Flat", + "Faiss HNSW SQ4", + "Faiss HNSW SQ8", +) + + +def validate_record(record, dataset, name): + """Reject results that do not match the published ANN configuration.""" + method, extra = next((m, e) for n, m, e in CASES if n == name) + base, queries, truth, metric = DATASETS[dataset] + n, dim = DATASET_SHAPES[dataset] + expected = { + "method": method, + "metric": metric, + "base_count": n, + "query_count": 1000, + "base_limit": None, + "query_limit": None, + "threads": 48, + "socket": 0, + "k": 10, + } + if "ivf" in method: + expected.update(nlist=4096, opq="--opq" in extra) + if "hnsw" in method: + expected.update(m=16, ef_construction=200) + if method == "rabitq-hnsw": + expected["hnsw_clusters"] = 16 + if method == "rabitq-symqg": + expected.update(degree=32, ef_construction=200) + for option in ("--bits", "--quantization-bits"): + if option in extra: + expected[option[2:].replace("-", "_")] = int(extra[extra.index(option) + 1]) + cfg = record["config"] + if ( + any(cfg.get(key) != value for key, value in expected.items()) + or record["data"]["dimension"] != dim + or any( + Path(cfg[key]).resolve() != (ROOT / path).resolve() + for key, path in ( + ("base", base), + ("queries", queries), + ("groundtruth", truth), + ) + ) + ): + raise ValueError(f"unexpected ANN configuration: {dataset}/{name}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--datasets", nargs="+", choices=DATASETS, default=list(DATASETS) + ) + parser.add_argument( + "--output-dir", type=Path, default=Path("results/benchmarks/ann") + ) + parser.add_argument("--methods", nargs="+", choices=[case[0] for case in CASES]) + parser.add_argument("--threads", type=int, default=48) + args = parser.parse_args() + + failed = [] + for dataset in args.datasets: + base, queries, truth, metric = DATASETS[dataset] + for name, method, extra in CASES: + if args.methods is not None and name not in args.methods: + continue + output = args.output_dir / dataset / f"{name}.json" + index_dir = args.output_dir / "indexes" / dataset / name + log = args.output_dir / "logs" / dataset / f"{name}.log" + if output.exists(): + validate_record(json.loads(output.read_text()), dataset, name) + print(f"SKIP {dataset} {name}: result exists", flush=True) + continue + if index_dir.exists(): + failed.append(f"{dataset}/{name}: incomplete index directory exists") + print(f"SKIP {failed[-1]}", flush=True) + continue + output.parent.mkdir(parents=True, exist_ok=True) + log.parent.mkdir(parents=True, exist_ok=True) + command = [ + sys.executable, + str(RUNNER), + "--base", + base, + "--queries", + queries, + "--groundtruth", + truth, + "--metric", + metric, + "--method", + method, + "--nlist", + "4096", + "--nprobe", + "64", + "--m", + "16", + "--degree", + "32", + "--hnsw-clusters", + "16", + "--ef-construction", + "200", + "--ef-search", + "128", + "--k-factor", + "1", + "--threads", + str(args.threads), + "--socket", + "0", + "--k", + "10", + "--index-dir", + str(index_dir), + "--output", + str(output), + *extra, + ] + print(f"START {dataset} {name}", flush=True) + with log.open("w") as stream: + stream.write(" ".join(command) + "\n") + stream.flush() + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=stream) + if result.returncode: + failed.append(f"{dataset}/{name}: exit {result.returncode}") + print(f"FAIL {failed[-1]} (see {log})", flush=True) + else: + print(f"DONE {dataset} {name}", flush=True) + if failed: + report = args.output_dir / "build_failures.json" + report.write_text(json.dumps(failed, indent=2) + "\n") + raise SystemExit(f"{len(failed)} index builds failed; see {report}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/download_datasets.py b/benchmarks/download_datasets.py new file mode 100644 index 00000000..f8ddbfb3 --- /dev/null +++ b/benchmarks/download_datasets.py @@ -0,0 +1,244 @@ +"""Download and prepare GIST1M and OpenAI-1536.""" + +import argparse +import json +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path + +import h5py +import numpy as np + +if __package__: + from .run import load_vectors +else: + from run import load_vectors + +ROOT = Path(__file__).resolve().parents[1] +GIST_URL = "ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz" +OPENAI_ID = "1cwIY76n_HEbbZAANJVWiVkjBsIy3b9jB" +GIST_FILES = { + "gist_base.fvecs": { + "bytes": 3844000000, + }, + "gist_query.fvecs": { + "bytes": 3844000, + }, + "gist_groundtruth.ivecs": { + "bytes": 404000, + }, +} + + +def write_fvecs(stream, vectors): + rows = np.empty((len(vectors), vectors.shape[1] + 1), dtype="")) + continue + # Select only this build's wheel, never an older cached version or Python ABI. + with tempfile.TemporaryDirectory(dir=work / "wheels", prefix=name + "-") as tmp: + run([*command, "-w", tmp], log) + built = list(Path(tmp).glob("*.whl")) + if len(built) != 1: + raise RuntimeError(f"expected one wheel from {name}, got {len(built)}") + wheel = work / "wheels" / built[0].name + shutil.copyfile(built[0], wheel) + wheels.append(str(wheel)) + records.append( + { + "source": name, + **provenance, + "cmake": flags, + "compiler": subprocess.check_output([compiler, "--version"], text=True), + "wheel": str(wheel), + } + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-deps", + "--force-reinstall", + *wheels, + ] + ) + if args.dry_run: + print( + "Then verify versions, environment paths, MKL/GNU OpenMP links, and write builds.json and installed.json." + ) + return + (work / "builds.json").write_text(json.dumps(records, indent=2) + "\n") + expected = {"rabitqlib": version, "faiss-cpu": "1.15.0", "superkmeans": "0.2.0"} + verified = subprocess.check_output( + [sys.executable, "-c", VERIFY, json.dumps(expected)], + cwd=ROOT, + env=env, + text=True, + ) + (work / "installed.json").write_text(verified) + activation = ROOT / "benchmarks/env.sh" + print( + f"Installed and verified all three libraries in {sys.prefix}.\nBefore running benchmarks: source {shlex.quote(str(activation))}" + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/mkl/MKLConfig.cmake b/benchmarks/mkl/MKLConfig.cmake new file mode 100644 index 00000000..e65b0028 --- /dev/null +++ b/benchmarks/mkl/MKLConfig.cmake @@ -0,0 +1,12 @@ +# MKL runtime package for local Linux/GCC Python baseline builds. +# Set MKL_THREADING_LAYER=GNU at runtime to share the caller's OpenMP runtime. +find_path(MKL_INCLUDE_DIR mkl.h PATH_SUFFIXES mkl REQUIRED) +find_library(MKL_RUNTIME_LIBRARY NAMES mkl_rt REQUIRED) +if(NOT TARGET MKL::MKL) + add_library(MKL::MKL INTERFACE IMPORTED) + set_target_properties(MKL::MKL PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MKL_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${MKL_RUNTIME_LIBRARY}" + ) +endif() +set(MKL_FOUND TRUE) diff --git a/benchmarks/plot_results.py b/benchmarks/plot_results.py new file mode 100644 index 00000000..c0300bbc --- /dev/null +++ b/benchmarks/plot_results.py @@ -0,0 +1,219 @@ +"""Plot QPS-recall curves and export measurements from single-query-set sweeps.""" + +import argparse +import csv +import json +import math +from itertools import product +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.lines import Line2D + +if __package__: + from .build_indexes import CASES, DATASETS, LABELS, validate_record + from .query_index import settings +else: + from build_indexes import CASES, DATASETS, LABELS, validate_record + from query_index import settings + + +def frontier(points): + """Keep measured points not dominated in both recall and throughput.""" + result = [] + best_qps = -math.inf + for recall, qps in sorted(set(points), reverse=True): + if qps > best_qps: + result.append((recall, qps)) + best_qps = qps + return list(reversed(result)) + + +def observations(record): + if record.get("status") != "single_query_set": + raise ValueError("expected a single_query_set sweep record") + for point in record["query_points"]: + yield ( + point["setting"], + point["recall"], + record["query_count"] / point["one_query_service_s"], + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path, required=True) + parser.add_argument( + "--output-dir", type=Path, default=Path("docs/docs/benchmarks/figures") + ) + parser.add_argument( + "--datasets", nargs="+", choices=DATASETS, default=list(DATASETS) + ) + parser.add_argument( + "--methods", + nargs="+", + choices=[case[0] for case in CASES], + default=[case[0] for case in CASES], + ) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + measurements = {} + csv_path = args.output_dir / "qps-recall-points.csv" + if csv_path.exists(): + with csv_path.open(newline="") as previous: + for row in csv.DictReader(previous): + measurements.setdefault((row["dataset"], row["method"]), []).append( + ( + json.loads(row["setting"]), + float(row["recall_at_10"]), + float(row["qps"]), + ) + ) + for dataset, name in product(args.datasets, args.methods): + initial = json.loads((args.input_dir / dataset / f"{name}.json").read_text()) + validate_record(initial, dataset, name) + record = json.loads( + (args.input_dir / "sweeps" / dataset / f"{name}.json").read_text() + ) + if ( + record["dataset"] != dataset + or record["query_count"] != initial["config"]["query_count"] + or [p["setting"] for p in record["query_points"]] + != settings(initial["config"]) + or any( + record["thread_environment"].get(key) != "1" + for key in ( + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + ) + ) + or record["cpu_affinity"] + != initial["build"]["cpu_placement"]["allowed_cpus"] + ): + raise ValueError(f"sweep does not match its index build: {dataset}/{name}") + measurements[dataset, name] = list(observations(record)) + colors = list(plt.get_cmap("tab10").colors) + markers = ("o", "s", "^", "D", "v", "P") + rows = [] + plt.rcParams.update({"font.size": 10, "svg.fonttype": "none"}) + + for dataset, family in product(args.datasets, ("ivf", "graphs")): + if not any( + name in args.methods and ("ivf" if "ivf" in method else "graphs") == family + for name, method, _ in CASES + ): + continue + fig, ax = plt.subplots(figsize=(12, 9)) + fig.subplots_adjust(top=0.85, bottom=0.32) + title = {"gist1m": "GIST1M", "dbpedia": "DBpedia"}[dataset] + family_title = "IVF indexes" if family == "ivf" else "Graph indexes" + fig.suptitle( + f"{title} · {family_title} | Recall@10 vs QPS", fontsize=20, y=0.975 + ) + fig.text( + 0.5, + 0.92, + "One Intel Xeon Gold 6418H CPU · one query per call", + ha="center", + color="#444444", + ) + handles = [] + recalls = [] + for ordinal, ((name, method, _), label) in enumerate( + (case, label) + for case, label in zip(CASES, LABELS, strict=True) + if ("ivf" if "ivf" in case[1] else "graphs") == family + ): + if (dataset, name) not in measurements: + continue + points = [] + for setting, recall, qps in measurements[dataset, name]: + if not (0 <= recall <= 1 and math.isfinite(qps) and qps > 0): + raise ValueError(f"invalid measurement: {dataset}/{name}") + points.append((recall, qps)) + recalls.extend(recall for recall, _ in points) + color = colors[ordinal] + marker = markers[ordinal % len(markers)] + style = "-" if name.startswith("rabitq-") else "--" + handles.append( + Line2D([], [], color=color, marker=marker, linestyle=style, label=label) + ) + if points: + x, y = zip(*points) + ax.scatter(x, y, color=color, marker=marker, s=22, alpha=0.45) + x, y = zip(*frontier(points)) + ax.plot( + x, + y, + color=color, + marker=marker, + markersize=4, + linewidth=1.4, + linestyle=style, + ) + ax.set_title( + "Query set · full sweep · 1 search thread · one timing pass", fontsize=11 + ) + ax.set_xlabel("Recall@10 (measured)") + ax.set_ylabel("Queries / second (log scale)") + recall_min = max(0, math.floor((min(recalls) - 0.005) * 20) / 20) + ax.set_xlim(recall_min, 1.005) + ax.set_yscale("log") + ax.grid(True, which="major", alpha=0.2) + ax.spines[["top", "right"]].set_visible(False) + fig.legend( + handles=handles, + loc="lower center", + bbox_to_anchor=(0.5, 0.09), + ncol=2, + frameon=False, + fontsize=9, + ) + fig.text( + 0.5, + 0.046, + "Faint markers: all measured settings. Lines: per-method Pareto frontier; no fitted or extrapolated points.", + ha="center", + fontsize=10, + color="#444444", + ) + path = args.output_dir / f"{family}-{dataset}-qps-recall.svg" + fig.savefig(path) + path.write_text( + "\n".join(line.rstrip() for line in path.read_text().splitlines()) + "\n" + ) + plt.close(fig) + print(f"Wrote {path}") + for dataset in DATASETS: + for name, _, _ in CASES: + for setting, recall, qps in measurements.get((dataset, name), []): + rows.append( + ( + dataset, + name, + json.dumps(setting, sort_keys=True), + recall, + qps, + ) + ) + with csv_path.open("w", newline="") as stream: + writer = csv.writer(stream, lineterminator="\n") + writer.writerow( + ( + "dataset", + "method", + "setting", + "recall_at_10", + "qps", + ) + ) + writer.writerows(rows) + print(f"Exported {len(rows)} measurements") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/query_index.py b/benchmarks/query_index.py new file mode 100644 index 00000000..5b936f46 --- /dev/null +++ b/benchmarks/query_index.py @@ -0,0 +1,145 @@ +"""Sweep a retained index with one search thread on one fixed query set.""" + +import argparse +import json +import os +import time +from pathlib import Path + +import numpy as np + +if __package__: + from .build_indexes import DATASETS + from .run import ( + load_index, + load_vectors, + read_matrix, + recall_at_k, + search, + validate_results, + ) +else: + from build_indexes import DATASETS + from run import ( + load_index, + load_vectors, + read_matrix, + recall_at_k, + search, + validate_results, + ) + + +def query_set(dataset, k): + _, query_path, truth_path, _ = DATASETS[dataset] + queries, _ = load_vectors(query_path) + truth, _ = read_matrix(truth_path) + if len(queries) != len(truth) or truth.shape[1] < k: + raise ValueError("query and ground-truth shapes differ") + return queries, truth[:, :k] + + +def settings(cfg): + method = cfg["method"] + k = cfg["k"] + if method in ("rabitq-ivf", "faiss-ivf-rabitq-fs", "faiss-ivf-pqfs"): + probes = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512) + factors = ( + (1, 2, 5, 10, 20) + if cfg["bits"] == 1 or method == "faiss-ivf-pqfs" + else (1,) + ) + if method == "rabitq-ivf": + factors = (1,) + return [ + {"nprobe": probe, "k_factor": factor} + for probe in probes + for factor in factors + ] + return [{"ef_search": ef} for ef in (16, 32, 64, 128, 256, 512, 1024) if ef >= k] + + +def configure(index, cfg, setting): + selected = cfg | setting + method = cfg["method"] + if method.startswith("faiss-ivf-"): + import faiss + + faiss.ParameterSpace().set_index_parameter(index, "nprobe", selected["nprobe"]) + if hasattr(index, "k_factor"): + index.k_factor = selected["k_factor"] + elif method.startswith("faiss-hnsw-"): + index.hnsw.efSearch = selected["ef_search"] + return selected + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("initial_record", type=Path) + parser.add_argument("--dataset", choices=DATASETS, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error(f"output already exists: {args.output}") + initial = json.loads(args.initial_record.read_text()) + cfg = initial["config"] + if cfg["socket"] != 0 or os.sched_getaffinity(0) != set( + initial["search"]["cpu_placement"]["allowed_cpus"] + ): + parser.error("run the sweep with numactl --cpunodebind=0 --membind=0") + if Path(cfg["base"]).resolve() != Path(DATASETS[args.dataset][0]).resolve(): + parser.error( + "index uses a different dataset; rebuild for the current query protocol" + ) + k = cfg["k"] + queries, truth = query_set(args.dataset, k) + if len(queries) != cfg["query_count"]: + parser.error("query count differs from the index build record") + index = load_index(cfg, Path(cfg["index_dir"]) / "index.bin") + points = [] + for setting in settings(cfg): + selected = configure(index, cfg, setting) + for query in queries[:20]: + search(index, selected, query.reshape(1, -1), 1) + results = [] + started = time.perf_counter() + for query in queries: + results.append(search(index, selected, query.reshape(1, -1), 1)) + service_s = time.perf_counter() - started + ids = np.concatenate([result[0] for result in results]) + distances = np.concatenate([result[1] for result in results]) + validate_results(ids, distances, cfg["base_count"], k) + recall = recall_at_k(ids, truth) + points.append( + { + "setting": setting, + "recall": recall, + "one_query_service_s": service_s, + } + ) + print(f"SWEEP {setting} recall={recall:.4f}", flush=True) + + result = { + "status": "single_query_set", + "dataset": args.dataset, + "initial_record": str(args.initial_record), + "query_count": len(queries), + "query_points": points, + "recall_mode": "one_query_per_call", + "cpu_affinity": sorted(os.sched_getaffinity(0)), + "thread_environment": { + name: os.environ.get(name) + for name in ( + "OMP_NUM_THREADS", + "MKL_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_THREADING_LAYER", + ) + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/reproduce.py b/benchmarks/reproduce.py new file mode 100644 index 00000000..e97ca23c --- /dev/null +++ b/benchmarks/reproduce.py @@ -0,0 +1,192 @@ +"""Reproduce all benchmarks or selected datasets, families, methods, and stages.""" + +import argparse +import os +import shlex +import subprocess +import sys +from pathlib import Path + +if __package__: + from .build_indexes import CASES, DATASETS, ROOT + from .run_kmeans import METHODS as KMEANS_METHODS +else: + from build_indexes import CASES, DATASETS, ROOT + from run_kmeans import METHODS as KMEANS_METHODS + + +def main(): + families = { + name: "ivf" if "ivf" in method else "graphs" for name, method, _ in CASES + } + families.update(dict.fromkeys(KMEANS_METHODS, "kmeans")) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--datasets", + nargs="+", + choices=DATASETS, + default=list(DATASETS), + help="default: GIST1M and DBpedia for all families", + ) + parser.add_argument( + "--benchmarks", + nargs="+", + choices=("ivf", "graphs", "kmeans"), + default=["ivf", "graphs", "kmeans"], + ) + parser.add_argument( + "--methods", + nargs="+", + choices=families, + help="configuration names; omit to run all methods in the selected families", + ) + parser.add_argument( + "--stage", + choices=("all", "build", "query", "export"), + default="all", + help="build includes k-means fitting; query reuses saved ANN indexes", + ) + parser.add_argument("--output-dir", type=Path, default=ROOT / "results/benchmarks") + parser.add_argument( + "--dry-run", + action="store_true", + help="print commands without running or writing results", + ) + args = parser.parse_args() + selected = [ + name + for name, family in families.items() + if family in args.benchmarks and (args.methods is None or name in args.methods) + ] + if args.methods is not None and set(args.methods) - set(selected): + parser.error("--methods includes configurations outside --benchmarks") + ann = [name for name in selected if families[name] != "kmeans"] + kmeans = [name for name in selected if families[name] == "kmeans"] + if args.stage == "query" and not ann: + parser.error("query stage requires an IVF or graph method") + if args.stage == "query": + kmeans = [] + ann_datasets = args.datasets if ann else [] + kmeans_datasets = args.datasets if kmeans else [] + output = args.output_dir.resolve() + env = dict( + os.environ, + MKL_THREADING_LAYER="GNU", + OMP_NUM_THREADS="48", + MKL_NUM_THREADS="48", + OPENBLAS_NUM_THREADS="48", + ) + env["LD_LIBRARY_PATH"] = os.pathsep.join( + p for p in (str(Path(sys.prefix) / "lib"), env.get("LD_LIBRARY_PATH")) if p + ) + placement = ["numactl", "--cpunodebind=0", "--membind=0"] + commands = [] + if args.stage != "export": + commands.append( + [ + *placement, + sys.executable, + str(ROOT / "benchmarks/download_datasets.py"), + "--datasets", + *args.datasets, + ] + ) + ann_selection = ["--datasets", *ann_datasets, "--methods", *ann] + kmeans_selection = ["--datasets", *kmeans_datasets, "--methods", *kmeans] + if args.stage in ("all", "build"): + if ann: + commands.append( + [ + *placement, + sys.executable, + str(ROOT / "benchmarks/build_indexes.py"), + "--output-dir", + str(output / "ann"), + "--threads", + "48", + *ann_selection, + ] + ) + if kmeans: + commands.append( + [ + *placement, + sys.executable, + str(ROOT / "benchmarks/run_kmeans.py"), + "--output-dir", + str(output / "kmeans"), + "--no-publish", + *kmeans_selection, + ] + ) + if args.stage in ("all", "query") and ann: + commands.append( + [ + *placement, + sys.executable, + str(ROOT / "benchmarks/run_sweeps.py"), + "--output-dir", + str(output / "ann"), + *ann_selection, + ] + ) + if args.stage in ("all", "export"): + if ann: + commands.append( + [ + sys.executable, + str(ROOT / "benchmarks/update_indexing.py"), + "--indexing-dir", + str(output / "ann"), + *ann_selection, + ] + ) + commands.append( + [ + sys.executable, + str(ROOT / "benchmarks/plot_results.py"), + "--input-dir", + str(output / "ann"), + *ann_selection, + ] + ) + if kmeans: + commands.append( + [ + sys.executable, + str(ROOT / "benchmarks/run_kmeans.py"), + "--output-dir", + str(output / "kmeans"), + "--export-only", + *kmeans_selection, + ] + ) + commands.append( + [ + sys.executable, + "-m", + "mkdocs", + "build", + "--strict", + "--config-file", + str(ROOT / "docs/mkdocs.yml"), + ] + ) + if not args.dry_run and args.stage == "export": + required = [ + output / folder / dataset / f"{name}.json" + for name in ann + kmeans + for dataset in (kmeans_datasets if name in kmeans else ann_datasets) + for folder in (("kmeans",) if name in kmeans else ("ann", "ann/sweeps")) + ] + missing = [str(path) for path in required if not path.is_file()] + if missing: + parser.error("selected results are missing: " + ", ".join(missing)) + for command in commands: + print(shlex.join(command), flush=True) + if not args.dry_run: + subprocess.run(command, cwd=ROOT, env=env, check=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 00000000..de98f467 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,539 @@ +"""Build and check one CPU ANN configuration in isolated build/search processes.""" + +import argparse +import importlib +import importlib.metadata +import json +import os +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import numpy as np + +METHODS = ( + "rabitq-ivf", + "rabitq-hnsw", + "rabitq-symqg", + "faiss-ivf-rabitq-fs", + "faiss-ivf-pqfs", + "faiss-hnsw-flat", + "faiss-hnsw-sq4", + "faiss-hnsw-sq8", +) + + +def import_search_library(method): + if method.startswith("faiss-"): + importlib.import_module("faiss") + elif method.startswith("rabitq-"): + importlib.import_module("rabitqlib") + else: + raise ValueError(f"unsupported benchmark method: {method}") + + +def read_matrix(path, limit=None): + """Read a row-major fvecs/ivecs matrix, checking every selected row header.""" + path = Path(path) + first = np.fromfile(path, dtype=" 100: + batches = [ + index.search(queries[start : start + 100], k) + for start in range(0, len(queries), 100) + ] + distances = np.concatenate([batch[0] for batch in batches]) + ids = np.concatenate([batch[1] for batch in batches]) + else: + distances, ids = index.search(queries, k) + return ids, distances + if method == "rabitq-ivf": + return index.search(queries, k=k, nprobe=cfg["nprobe"], num_threads=threads) + if method in ("rabitq-hnsw", "rabitq-symqg"): + return index.search(queries, k=k, ef=cfg["ef_search"], num_threads=threads) + raise ValueError(f"unsupported benchmark method: {method}") + + +def validate_results(ids, distances, count, k): + if ids.ndim != 2 or ids.shape[1] != k or distances.shape != ids.shape: + raise ValueError("index returned an incorrect result shape") + if np.any(ids < 0) or np.any(ids >= count): + raise ValueError("index returned an invalid point ID") + if any(len(set(row)) != k for row in ids.tolist()): + raise ValueError("index returned duplicate point IDs") + if not np.all(np.isfinite(distances)): + raise ValueError("index returned a nonfinite distance") + + +def recall_at_k(ids, groundtruth): + if ( + ids.ndim != 2 + or groundtruth.ndim != 2 + or len(ids) != len(groundtruth) + or groundtruth.shape[1] < ids.shape[1] + ): + raise ValueError("result and ground-truth shapes differ") + k = ids.shape[1] + return float( + np.mean( + [len(set(row) & set(truth[:k])) / k for row, truth in zip(ids, groundtruth)] + ) + ) + + +def cpu_placement(socket): + if socket is None: + return None + allowed = sorted(os.sched_getaffinity(0)) + sockets = sorted( + { + int( + Path( + f"/sys/devices/system/cpu/cpu{cpu}/topology/physical_package_id" + ).read_text() + ) + for cpu in allowed + } + ) + if sockets != [socket]: + raise RuntimeError( + f"expected CPU socket {socket}, but process affinity spans sockets {sockets}" + ) + return {"socket": socket, "allowed_cpus": allowed} + + +def worker(stage, directory): + directory = Path(directory) + cfg = json.loads((directory / "config.json").read_text()) + placement = cpu_placement(cfg["socket"]) + if placement is not None and cfg["threads"] > len(placement["allowed_cpus"]): + raise RuntimeError("thread count exceeds CPUs available on the selected socket") + index_directory = ( + Path(cfg["index_dir"]) if cfg["index_dir"] else directory / "index" + ) + path = index_directory / "index.bin" + if stage == "_build": + base = np.load(directory / "base.npy", mmap_mode="r") + import_search_library(cfg["method"]) + details = build_index(cfg, base, path) + details["cpu_placement"] = placement + (directory / "build.json").write_text(json.dumps(details)) + return + queries = np.load(directory / "queries.npy") + groundtruth = np.load(directory / "groundtruth.npy") + import_search_library(cfg["method"]) + load_start = time.perf_counter() + index = load_index(cfg, path) + load_s = time.perf_counter() - load_start + search(index, cfg, queries[: min(len(queries), 20)], 1) + ids, distances = search(index, cfg, queries, 1) + validate_results(ids, distances, cfg["base_count"], cfg["k"]) + recall = recall_at_k(ids, groundtruth) + (directory / "search.json").write_text( + json.dumps( + { + "recall_at_k": recall, + "load_s": load_s, + "cpu_placement": placement, + } + ) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, help="base vectors in fvecs format") + parser.add_argument( + "--queries", required=True, help="query vectors in fvecs format" + ) + parser.add_argument( + "--groundtruth", help="ivecs file; only valid with the full base" + ) + parser.add_argument("--method", required=True, choices=METHODS) + parser.add_argument("--metric", choices=("l2", "ip"), default="l2") + parser.add_argument("--base-limit", type=int) + parser.add_argument("--query-limit", type=int) + parser.add_argument("--nlist", type=int, default=4096) + parser.add_argument("--nprobe", type=int, default=64) + parser.add_argument("--bits", type=int, default=5) + parser.add_argument("--quantization-bits", type=int, default=0) + parser.add_argument("--hnsw-clusters", type=int, default=16) + parser.add_argument("--degree", type=int, default=32) + parser.add_argument("--k-factor", type=int, default=1) + parser.add_argument("--opq", action="store_true") + parser.add_argument("--m", type=int, default=16) + parser.add_argument("--ef-construction", type=int, default=200) + parser.add_argument("--ef-search", type=int, default=128) + parser.add_argument("--threads", type=int, default=48) + parser.add_argument( + "--socket", + type=int, + help="bind build and search CPUs and memory to this Linux NUMA socket", + ) + parser.add_argument("--k", type=int, default=10) + parser.add_argument( + "--index-dir", + type=Path, + help="persist the built index here for later search-parameter sweeps", + ) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + for name in ( + "nlist", + "nprobe", + "bits", + "hnsw_clusters", + "degree", + "k_factor", + "m", + "ef_construction", + "ef_search", + "threads", + "k", + ): + if getattr(args, name) <= 0: + parser.error(f"--{name.replace('_', '-')} must be positive") + if args.socket is not None: + if args.socket < 0: + parser.error("--socket must be nonnegative") + if shutil.which("numactl") is None: + parser.error("--socket requires numactl") + for name in ("base_limit", "query_limit"): + if getattr(args, name) is not None and getattr(args, name) <= 0: + parser.error(f"--{name.replace('_', '-')} must be positive") + if ( + args.method + in ( + "rabitq-hnsw", + "rabitq-symqg", + "faiss-hnsw-flat", + "faiss-hnsw-sq4", + "faiss-hnsw-sq8", + ) + and args.ef_search < args.k + ): + parser.error("--ef-search must be at least k for graph indexes") + if args.nprobe > args.nlist: + parser.error("--nprobe must be at most --nlist") + if args.method == "rabitq-ivf" and args.bits not in (*range(1, 10), 32): + parser.error("RaBitQ-Library IVF bits must be 1 through 9 or 32") + if args.method == "rabitq-hnsw" and args.bits not in range(1, 10): + parser.error("HNSW-RaBitQ bits must be 1 through 9") + if args.method == "rabitq-symqg" and args.quantization_bits not in (0, 4, 8): + parser.error("SymphonyQG quantization bits must be 0, 4, or 8") + if args.method == "faiss-ivf-rabitq-fs" and args.bits not in range(1, 10): + parser.error("Faiss RaBitQ FastScan bits must be 1 through 9") + if args.output.exists(): + parser.error(f"output already exists: {args.output}") + if args.index_dir is not None and args.index_dir.exists(): + parser.error(f"index directory already exists: {args.index_dir}") + base, total_base = load_vectors(args.base, args.base_limit) + queries, _ = load_vectors(args.queries, args.query_limit) + if base.shape[1] != queries.shape[1]: + parser.error("base/query dimensions differ") + if args.k > len(base): + parser.error("k exceeds the base vector count") + if args.nlist >= len(base): + parser.error("nlist must be smaller than the base vector count") + if args.method == "rabitq-ivf" and args.metric == "ip": + norms = np.linalg.norm(base, axis=1) + if not np.allclose(norms, 1, atol=1e-4): + parser.error( + "RaBitQKMeans spherical training requires normalized base vectors" + ) + if args.groundtruth: + if len(base) != total_base: + parser.error("supplied ground truth is invalid for a truncated base") + gt, _ = read_matrix(args.groundtruth, len(queries)) + if ( + gt.shape[0] != len(queries) + or gt.shape[1] < args.k + or np.any(gt[:, : args.k] < 0) + or np.any(gt[:, : args.k] >= len(base)) + ): + parser.error("ground truth shape or IDs are incompatible") + gt = gt[:, : args.k].copy() + gt_source = str(args.groundtruth) + else: + gt = exact_neighbors(base, queries, args.k, args.metric, args.threads) + gt_source = "Faiss IndexFlat on selected base and queries" + cfg = vars(args).copy() + cfg["output"] = str(args.output) + cfg["index_dir"] = str(args.index_dir.resolve()) if args.index_dir else None + cfg["base_count"] = len(base) + cfg["query_count"] = len(queries) + with tempfile.TemporaryDirectory(prefix="rabitq-benchmark-") as tmp: + directory = Path(tmp) + index_directory = ( + Path(cfg["index_dir"]) if cfg["index_dir"] else directory / "index" + ) + index_directory.mkdir(parents=True) + np.save(directory / "base.npy", base) + np.save(directory / "queries.npy", queries) + np.save(directory / "groundtruth.npy", gt) + (directory / "config.json").write_text(json.dumps(cfg)) + for stage in ("_build", "_search"): + command = [ + sys.executable, + str(Path(__file__).resolve()), + stage, + str(directory), + ] + if args.socket is not None: + command = [ + "numactl", + f"--cpunodebind={args.socket}", + f"--membind={args.socket}", + *command, + ] + subprocess.run( + command, + check=True, + ) + build = json.loads((directory / "build.json").read_text()) + measured = json.loads((directory / "search.json").read_text()) + artifact_bytes = sum( + p.stat().st_size for p in index_directory.rglob("*") if p.is_file() + ) + packages = { + name: importlib.metadata.version(name) for name in ("rabitqlib", "faiss-cpu") + } + cpuinfo = Path("/proc/cpuinfo") + model_lines = ( + [ + line + for line in cpuinfo.read_text().splitlines() + if line.startswith("model name") + ] + if cpuinfo.exists() + else [] + ) + cpu_model = ( + model_lines[0].split(":", 1)[1].strip() if model_lines else platform.processor() + ) + record = { + "schema_version": 3, + "status": "built", + "packages": packages, + "hardware": { + "machine": platform.machine(), + "cpu": cpu_model, + "os": platform.platform(), + }, + "config": cfg, + "data": { + "groundtruth_source": gt_source, + "dimension": base.shape[1], + "total_base_rows": total_base, + }, + "build": build, + "search": measured, + "serialized_index_bytes": artifact_bytes, + "reproduction_command": shlex.join( + [sys.executable, str(Path(__file__).resolve()), *sys.argv[1:]] + ), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as out: + json.dump(record, out, indent=2) + out.write("\n") + print( + f"{args.method}: recall@{args.k}={measured['recall_at_k']:.4f}; {args.output}" + ) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] in ("_build", "_search"): + worker(sys.argv[1], sys.argv[2]) + else: + main() diff --git a/benchmarks/run_kmeans.py b/benchmarks/run_kmeans.py new file mode 100644 index 00000000..f956f7f6 --- /dev/null +++ b/benchmarks/run_kmeans.py @@ -0,0 +1,564 @@ +"""Compare flat float32 k-means with a 25-iteration limit and normal early stopping.""" + +import argparse +import csv +import importlib.metadata +import io +import json +import math +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np + +if __package__: + from .build_indexes import DATASET_SHAPES + from .build_indexes import DATASETS as ANN_DATASETS + from .run import ( + cpu_placement, + exact_neighbors, + load_vectors, + read_matrix, + ) +else: + from build_indexes import DATASET_SHAPES + from build_indexes import DATASETS as ANN_DATASETS + from run import ( + cpu_placement, + exact_neighbors, + load_vectors, + read_matrix, + ) + +ROOT = Path(__file__).resolve().parents[1] +METHODS = { + "qgkmeans": "QGKMeans", + "faiss": "Faiss K-means", + "superkmeans": "SuperKMeans", +} +DATASETS = { + # Base path, query path, base count, dimensions, clusters. + name: (*paths[:2], *DATASET_SHAPES[name], 4096) + for name, paths in ANN_DATASETS.items() +} + + +def write_json(path, value): + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + temporary.replace(path) + + +def load_base(cfg): + return np.memmap( + cfg["base_file"], mode="r", dtype="= k): + raise ValueError("invalid final assignments") + if not np.isfinite(centroids).all(): + raise ValueError("nonfinite centroids") + if ( + truth.shape != (len(queries), 10) + or np.any(truth < 0) + or np.any(truth >= len(base)) + ): + raise ValueError("ground truth must contain ten valid neighbors per query") + wcss = 0.0 + for start in range(0, len(base), 8192): + stop = start + 8192 + residual = base[start:stop].astype(np.float64) - centroids[ + labels[start:stop] + ].astype(np.float64) + wcss += float(np.einsum("ij,ij->", residual, residual)) + # Independent direct-float64 audit, outside timing, including evenly spaced vectors. + max_gap = 0.0 + c64 = centroids.astype(np.float64) + for i in np.linspace(0, len(base) - 1, min(128, len(base)), dtype=int): + residual = c64 - base[i].astype(np.float64) + distances = np.einsum("ij,ij->i", residual, residual) + best = float(distances.min()) + gap = float(distances[labels[i]]) - best + max_gap = max(max_gap, gap) + if gap > 1e-5 * max(1.0, best): + raise ValueError(f"final assignment audit failed at vector {i}: gap={gap}") + faiss.omp_set_num_threads(threads) + index = faiss.IndexFlatL2(d) + index.add(centroids) + hits = 0 + for start in range(0, len(queries), 256): + distances, ids = index.search(queries[start : start + 256], k) + # Sort all centroid distances to make equal-distance cutoff ties deterministic. + order = np.lexsort((ids, distances), axis=1)[:, :nprobe] + probes = np.take_along_axis(ids, order, axis=1) + neighbor_labels = labels[truth[start : start + len(probes)]] + hits += int( + (neighbor_labels[:, :, None] == probes[:, None, :]).any(axis=2).sum() + ) + return { + "wcss": wcss, + "recall_at_10": hits / (10 * len(queries)), + "nprobe": nprobe, + "assignment_audit_max_l2_gap": max_gap, + } + + +def worker(stage, config_path): + cfg = json.loads(config_path.read_text()) + cpu_placement(cfg["socket"]) + x = load_base(cfg) + folder = Path(cfg["work_dir"]) + if stage == "_prepare": + queries = np.load(cfg["query_file"]) + truth = exact_neighbors(x, queries, 10, "l2", cfg["threads"]) + np.save(cfg["truth_file"], truth) + elif stage == "_train": + centroids, labels, timing = train_python( + cfg["method"], x, cfg["k"], cfg["threads"], cfg["seed"] + ) + centroids.astype("", f"" + if text.count(start) != 1 or text.count(end) != 1: + raise ValueError(f"missing or repeated result marker: {name}") + before, rest = text.split(start) + _, after = rest.split(end) + return before + start + "\n" + body + "\n" + end + after + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--methods", nargs="+", choices=METHODS, default=list(METHODS)) + parser.add_argument( + "--export-only", + action="store_true", + help="export saved results without running experiments", + ) + parser.add_argument( + "--no-publish", action="store_true", help="save results without updating docs" + ) + parser.add_argument( + "--datasets", nargs="+", choices=DATASETS, default=list(DATASETS) + ) + args = parser.parse_args() + output = ( + args.output_dir + or Path( + "results/benchmarks-smoke/kmeans" + if args.smoke + else "results/benchmarks/kmeans" + ) + ).resolve() + if args.export_only: + if args.smoke: + parser.error("smoke results cannot be published") + publish(output, datasets=args.datasets, methods=args.methods) + return + output.mkdir(parents=True, exist_ok=True) + placement = cpu_placement(None if args.smoke else 0) + if not args.smoke: + cores = { + Path(f"/sys/devices/system/cpu/cpu{c}/topology/core_id").read_text().strip() + for c in placement["allowed_cpus"] + } + if len(cores) != 24 or len(placement["allowed_cpus"]) != 48: + raise RuntimeError( + "benchmark requires 24 physical cores and 48 logical CPUs" + ) + threads = 2 if args.smoke else 48 + env = { + **os.environ, + "OMP_NUM_THREADS": str(threads), + "OPENBLAS_NUM_THREADS": str(threads), + "MKL_NUM_THREADS": str(threads), + "MKL_THREADING_LAYER": "GNU", + # Let MKL resolve the same GNU OpenMP library as the Python environment. + "LD_LIBRARY_PATH": os.pathsep.join( + p + for p in (str(Path(sys.prefix) / "lib"), os.environ.get("LD_LIBRARY_PATH")) + if p + ), + } + metadata = { + "rabitqlib_version": importlib.metadata.version("rabitqlib"), + "faiss_version": importlib.metadata.version("faiss-cpu"), + "superkmeans_version": importlib.metadata.version("superkmeans"), + "cpu_placement": placement, + "platform": platform.platform(), + } + failures = [] + for dataset in ("smoke",) if args.smoke else args.datasets: + pending = [ + m for m in args.methods if not (output / dataset / f"{m}.json").exists() + ] + if not pending: + continue + print(f"PREPARE {dataset}", flush=True) + with tempfile.TemporaryDirectory(prefix="rabitq-kmeans-") as tmp: + staging = Path(tmp) + if args.smoke: + rng = np.random.default_rng(42) + base = rng.standard_normal((2048, 192)).astype("" + end = f"" + if text.count(start) != 1 or text.count(end) != 1: + raise ValueError(f"expected one generated block: {name}") + before, rest = text.split(start) + _, after = rest.split(end) + return before + start + "\n" + body + "\n" + end + after + + +def update(indexing_dir, docs_dir, datasets=None, methods=None): + datasets = list(DATASETS) if datasets is None else datasets + cases = [case for case in CASES if methods is None or case[0] in methods] + rows = {} + csv_path = docs_dir / "figures/indexing.csv" + if csv_path.exists(): + with csv_path.open(newline="") as previous: + rows = { + (row["dataset"], row["method"]): tuple(row[key] for key in FIELDS) + for row in csv.DictReader(previous) + } + for dataset in datasets: + for name, method, _ in cases: + path = indexing_dir / dataset / f"{name}.json" + record = json.loads(path.read_text()) + validate_record(record, dataset, name) + build = record["build"] + values = [build[key] for key in FIELDS[3:7]] + values.append(record["serialized_index_bytes"]) + if not all(math.isfinite(value) and value >= 0 for value in values): + raise ValueError(f"invalid indexing measurements: {path}") + family = "ivf" if "ivf" in method else "graphs" + rows[dataset, name] = (dataset, family, name, *values, 48) + + pages = {} + for family in ("ivf", "graphs"): + if not any( + ("ivf" if "ivf" in method else "graphs") == family for _, method, _ in cases + ): + continue + path = docs_dir / f"{family}.md" + text = path.read_text() + for dataset in datasets: + table = [ + "| Method | Train s | Construct s | Total s | Save s | Index GiB |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + for (name, method, _), label in zip(CASES, LABELS, strict=True): + if ("ivf" if "ivf" in method else "graphs") != family: + continue + row = rows.get((dataset, name)) + if row is None: + cells = ["Pending", *["—"] * 4] + else: + values = [ + *(float(value) for value in row[3:7]), + float(row[7]) / 2**30, + ] + cells = [f"{value:.2f}" for value in values] + table.append("| " + label + " | " + " | ".join(cells) + " |") + text = replace_block(text, f"indexing:{dataset}", "\n".join(table)) + pages[path] = text + + stream = io.StringIO(newline="") + writer = csv.writer(stream, lineterminator="\n") + writer.writerow(FIELDS) + writer.writerows( + rows[key] for key in ((d, c[0]) for d in DATASETS for c in CASES) if key in rows + ) + pages[csv_path] = stream.getvalue() + for path, text in pages.items(): + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(text) + temporary.replace(path) + print(f"Updated {len(cases) * len(datasets)} indexing results.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--indexing-dir", type=Path, required=True) + parser.add_argument("--docs-dir", type=Path, default=Path("docs/docs/benchmarks")) + parser.add_argument("--datasets", nargs="+", choices=DATASETS) + parser.add_argument("--methods", nargs="+", choices=[case[0] for case in CASES]) + args = parser.parse_args() + update(args.indexing_dir, args.docs_dir, args.datasets, args.methods) + + +if __name__ == "__main__": + main() diff --git a/docs/docs/benchmarks/bench.md b/docs/docs/benchmarks/bench.md new file mode 100644 index 00000000..d04adfd7 --- /dev/null +++ b/docs/docs/benchmarks/bench.md @@ -0,0 +1,136 @@ +# Benchmark overview + +Compare RaBitQ-Library, Faiss, and SuperKMeans on one Intel Xeon Gold 6418H: +[IVF indexes](ivf.md), [graph indexes](graphs.md), and [k-means](kmeans.md). + +## Datasets and ground truth + +| Dataset | Base vectors | Dimensions | ANN metric | Queries | +| --- | ---: | ---: | --- | ---: | +| GIST1M | 1,000,000 | 960 | L2 | 1,000 | +| DBpedia / OpenAI-1536 | 999,000 | 1,536 | Inner product | 1,000 | + +GIST1M uses the standard TexMex split and supplied L2 ground truth. OpenAI-1536 +uses [SuperKMeans' dataset](https://github.com/cwida/SuperKMeans/blob/main/BENCHMARKING.md), +preserving its split, row order, and normalized float32 values. ANN ground truth +for OpenAI is computed by exhaustive inner-product search. + +All methods share one fixed query set per dataset. Queries are used only for +evaluation. [K-means](kmeans.md) uses L2 for both datasets and computes separate +L2 ground truth for OpenAI. + +## Hardware and software + +| Setting | Value | +| --- | --- | +| CPU | One Intel Xeon Gold 6418H, 24 cores, 48 hardware threads | +| Construction / clustering | 48 threads | +| Query throughput | 1 process, 1 search thread, 1 query per call | +| Platform | Ubuntu 22.04.5 LTS, Linux 6.8.0-84-generic, x86-64 | +| Python / NumPy | 3.13.15 / 2.5.2 | +| Construction and clustering compiler / CMake | GCC 13.1.0 / 3.28.3, Release | +| RaBitQ-Library | 0.5.2 | +| Baselines | Faiss 1.15.0, SuperKMeans 0.2.0; Intel MKL 2020.4, GNU OpenMP | + +## Package installation + +Use a Python 3.13 environment on Ubuntu x86-64. Ubuntu 22.04 needs a package +source providing `g++-13` and the `multiverse` component for MKL. + +```bash +sudo apt-get update +sudo apt-get install build-essential g++-13 git numactl curl \ + libmkl-dev libmkl-rt libmkl-gnu-thread + +python benchmarks/install_baselines.py --jobs 12 +source benchmarks/env.sh +``` + +The installer builds RaBitQ from the checkout, Faiss 1.15.0, and SuperKMeans 0.2.0 +in the active environment. Both baselines link to MKL with GNU OpenMP. Build logs +and verification records are saved in `build/benchmark-baselines/`. +Use `--dry-run` to preview installation commands. + +## Construction and search settings + +IVF uses 4,096 clusters. HNSW uses `M=16` and `efConstruction=200`; +SymphonyQG uses degree 32. Family pages list all method settings. + +| Family | Search grid | +| --- | --- | +| IVF | `nprobe=1,2,4,8,16,32,64,128,256,512` | +| Faiss IVF with RFlat | Above × `k_factor=1,2,5,10,20` | +| Graph | `ef=16,32,64,128,256,512,1024` | + +Each setting searches all 1,000 queries at `k=10`, after warm-up. QPS is queries +divided by elapsed search time, including Python call overhead and excluding +index loading. Each setting has one timing pass. + +## Reading the figures + +Recall@10 is the mean fraction of exact top-10 neighbors returned. Faint markers +show all measured settings; lines connect each method's Pareto frontier. QPS uses +a logarithmic axis. The CSV contains every measured point. + +## Reading the indexing tables + +| Column | Meaning | +| --- | --- | +| Train | Training or centroid clustering time | +| Construct | Encoding and index construction after training | +| Total | Construction including setup and training, excluding saving | +| Save | Serialization time | +| Index GiB | Serialized files, including sidecars and retained raw vectors | + +Times are seconds. Data preparation, ground-truth generation, and correctness +checks are outside construction timing. + +## Scope and limitations + +Each method has one build or fit per dataset; results do not measure run-to-run +variation. + +## Data and reproduction + +Run from the repository root after installation: + +```bash +# Prepare data, run all benchmarks, and export results +python benchmarks/reproduce.py + +# Prepare data only +python benchmarks/download_datasets.py + +# Selected results +python benchmarks/reproduce.py --benchmarks ivf --datasets gist1m +python benchmarks/reproduce.py --datasets dbpedia --methods qgkmeans +``` + +Downloads preserve the published splits and check file sizes and vector shapes. GIST and +OpenAI need about 26 GB for downloads and prepared data; indexes and temporary +workspace require additional space. + +| Content | Default location | +| --- | --- | +| Download cache | `data/benchmark-downloads/` | +| GIST vectors and ground truth | `data/gist/` | +| OpenAI vectors and ANN ground truth | `data/dbpedia/` | +| ANN build records / query records | `results/benchmarks/ann//` / `results/benchmarks/ann/sweeps//` | +| Saved indexes | `results/benchmarks/ann/indexes/` | +| K-means records, centroids, assignments | `results/benchmarks/kmeans//` | +| Logs | `results/benchmarks/ann/logs/`, `results/benchmarks/kmeans/logs/` | +| Tables / CSVs and SVGs / website | `docs/docs/benchmarks/` / `docs/docs/benchmarks/figures/` / `docs/site/` | + +Existing results are skipped. Use a fresh `--output-dir` to repeat measurements; +this changes raw result and log paths, while documentation exports retain the +paths above. Partial exports preserve unselected results. + +Use `--stage build`, `--stage query`, or `--stage export` for one stage. Querying +reuses saved indexes; export runs no experiments. `--dry-run` previews commands; +`--help` lists methods. Both datasets are selected by default. + +`data/` and `results/` are ignored by Git. The downloader accepts `--datasets`, +`--data-dir`, and `--cache-dir`; runners expect the default data paths. + +[Query CSV](figures/qps-recall-points.csv) · [Indexing CSV](figures/indexing.csv) · +[K-means CSV](figures/kmeans.csv) diff --git a/docs/docs/benchmarks/figures/graphs-dbpedia-qps-recall.svg b/docs/docs/benchmarks/figures/graphs-dbpedia-qps-recall.svg new file mode 100644 index 00000000..eb25c644 --- /dev/null +++ b/docs/docs/benchmarks/figures/graphs-dbpedia-qps-recall.svg @@ -0,0 +1,969 @@ + + + + + + + + 2026-10-05T12:13:35.734224 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.65 + + + + + + + + + + + + + 0.70 + + + + + + + + + + + + + 0.75 + + + + + + + + + + + + + 0.80 + + + + + + + + + + + + + 0.85 + + + + + + + + + + + + + 0.90 + + + + + + + + + + + + + 0.95 + + + + + + + + + + + + + 1.00 + + + + Recall@10 (measured) + + + + + + + + + + + + + + + + + + + + 1 + 0 + 3 + + + + + + + + + + + + + + + + + + 1 + 0 + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Queries / second (log scale) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Query set · full sweep · 1 search thread · one timing pass + + + + DBpedia · Graph indexes | Recall@10 vs QPS + + + One Intel Xeon Gold 6418H CPU · one query per call + + + Faint markers: all measured settings. Lines: per-method Pareto frontier; no fitted or extrapolated points. + + + + + + + + + + + + + RaBitQ SymphonyQG raw + + + + + + + + + + + + RaBitQ SymphonyQG 4-bit + + + + + + + + + + + + RaBitQ SymphonyQG 8-bit + + + + + + + + + + + + RaBitQ HNSW 1+4 + + + + + + + + + + + + RaBitQ HNSW 1+8 + + + + + + + + + + + + Faiss HNSW Flat + + + + + + + + + + + + Faiss HNSW SQ4 + + + + + + + + + + + + Faiss HNSW SQ8 + + + + + + + + + diff --git a/docs/docs/benchmarks/figures/graphs-gist1m-qps-recall.svg b/docs/docs/benchmarks/figures/graphs-gist1m-qps-recall.svg new file mode 100644 index 00000000..41a330be --- /dev/null +++ b/docs/docs/benchmarks/figures/graphs-gist1m-qps-recall.svg @@ -0,0 +1,976 @@ + + + + + + + + 2026-10-05T12:13:35.578028 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.3 + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + + 0.5 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.7 + + + + + + + + + + + + + 0.8 + + + + + + + + + + + + + 0.9 + + + + + + + + + + + + + 1.0 + + + + Recall@10 (measured) + + + + + + + + + + + + + + + + + + + + 1 + 0 + 3 + + + + + + + + + + + + + + + + + + 1 + 0 + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Queries / second (log scale) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Query set · full sweep · 1 search thread · one timing pass + + + + GIST1M · Graph indexes | Recall@10 vs QPS + + + One Intel Xeon Gold 6418H CPU · one query per call + + + Faint markers: all measured settings. Lines: per-method Pareto frontier; no fitted or extrapolated points. + + + + + + + + + + + + + RaBitQ SymphonyQG raw + + + + + + + + + + + + RaBitQ SymphonyQG 4-bit + + + + + + + + + + + + RaBitQ SymphonyQG 8-bit + + + + + + + + + + + + RaBitQ HNSW 1+4 + + + + + + + + + + + + RaBitQ HNSW 1+8 + + + + + + + + + + + + Faiss HNSW Flat + + + + + + + + + + + + Faiss HNSW SQ4 + + + + + + + + + + + + Faiss HNSW SQ8 + + + + + + + + + diff --git a/docs/docs/benchmarks/figures/indexing.csv b/docs/docs/benchmarks/figures/indexing.csv new file mode 100644 index 00000000..cd0899e2 --- /dev/null +++ b/docs/docs/benchmarks/figures/indexing.csv @@ -0,0 +1,33 @@ +dataset,family,method,training_s,indexing_s,build_s,save_s,serialized_index_bytes,construction_threads +gist1m,ivf,rabitq-ivf-raw,14.703828662168235,0.6624991721473634,15.366377223748714,1.55181243410334,4000476046,48 +gist1m,ivf,rabitq-ivf-1p4,14.841738895047456,1.7135350201278925,16.55532868532464,0.2537568719126284,648476034,48 +gist1m,ivf,rabitq-ivf-1p8,14.541446471121162,7.025055922102183,21.566568977665156,0.45164870796725154,1128476034,48 +gist1m,ivf,faiss-rabitqfs-raw,41.60687803989276,6.758632214739919,48.367109845392406,1.5819987310096622,4009934430,48 +gist1m,ivf,faiss-rabitqfs-1p4,41.239345733076334,13.162328890990466,54.40425459900871,0.28943439619615674,728962696,48 +gist1m,ivf,faiss-rabitqfs-1p8,41.37886307295412,39.81617270922288,81.19657412171364,0.51593446591869,1275188872,48 +gist1m,ivf,faiss-pqfs,34.52272639609873,6.5879326411522925,41.11183687672019,1.5885270270518959,4119607590,48 +gist1m,ivf,faiss-opq-pqfs,232.45455483580008,7.514239463023841,239.97109729098156,1.629965650383383,4123370861,48 +gist1m,graphs,rabitq-symqg-raw,0.0,38.73717326577753,38.73719131015241,3.268616213928908,8064000518,48 +gist1m,graphs,rabitq-symqg-4,0.0,38.40828195307404,38.40830117510632,1.8603721661493182,4712004378,48 +gist1m,graphs,rabitq-symqg-8,0.0,38.416935969144106,38.41695372806862,2.042469825129956,5192004378,48 +gist1m,graphs,rabitq-hnsw-1p4,1.2380108321085572,70.56828525522724,71.80634994897991,0.35435156989842653,768589041,48 +gist1m,graphs,rabitq-hnsw-1p8,1.3915513390675187,76.69357230188325,78.0851734620519,0.5540446750819683,1248589041,48 +gist1m,graphs,faiss-hnsw-flat,3.4594908356666565e-05,78.93601581081748,78.93702552421018,1.6070536905899644,3984258362,48 +gist1m,graphs,faiss-hnsw-sq4,1.0616188147105277,55.10552563937381,56.16821300517768,0.24274761974811554,624266078,48 +gist1m,graphs,faiss-hnsw-sq8,1.0816914499737322,47.12117446027696,48.20351363904774,0.41724564833566546,1104266078,48 +dbpedia,ivf,rabitq-ivf-raw,24.29245251091197,1.8862061300314963,26.178707384038717,2.582001658156514,6383774478,48 +dbpedia,ivf,rabitq-ivf-1p4,24.332880606874824,2.885876853018999,27.218810306861997,0.3972435803152621,1021142466,48 +dbpedia,ivf,rabitq-ivf-1p8,24.355551805812865,12.417853215243667,36.77345839003101,0.6979606719687581,1788374466,48 +dbpedia,ivf,faiss-rabitqfs-raw,85.28691623080522,12.971096782013774,98.26071831583977,2.706343464087695,6460068638,48 +dbpedia,ivf,faiss-rabitqfs-1p4,86.94140290375799,28.046236151829362,114.99026683531702,0.5991422357037663,1423356232,48 +dbpedia,ivf,faiss-rabitqfs-1p8,119.86079298425466,91.1069279271178,210.96929090702906,0.9999383529648185,2511745352,48 +dbpedia,ivf,faiss-pqfs,99.81868983665481,15.401424441952258,115.22205780772492,2.5630933842621744,6579360742,48 +dbpedia,ivf,faiss-opq-pqfs,1414.2767510288395,13.646546002943069,1427.9251843532547,2.72974043013528,6588736557,48 +dbpedia,graphs,rabitq-symqg-raw,0.0,59.398011999670416,59.398029705975205,5.063656691927463,12659328806,48 +dbpedia,graphs,rabitq-symqg-4,0.0,57.92335021216422,57.923367423936725,2.850183868780732,7296702970,48 +dbpedia,graphs,rabitq-symqg-8,0.0,57.16500317910686,57.165021975990385,3.3480491428636014,8063934970,48 +dbpedia,graphs,rabitq-hnsw-1p4,1.5930713429115713,111.02135393023491,112.61447793105617,0.5676363878883421,1127498521,48 +dbpedia,graphs,rabitq-hnsw-1p8,1.5328591028228402,122.56575142173097,124.09866199130192,0.8250842890702188,1894730521,48 +dbpedia,graphs,faiss-hnsw-flat,3.532087430357933e-05,147.53378360485658,147.53478899691254,2.6907204217277467,6281970202,48 +dbpedia,graphs,faiss-hnsw-sq4,1.7431773450225592,101.82403805106878,103.56827840721235,0.3557668989524245,911358526,48 +dbpedia,graphs,faiss-hnsw-sq8,1.7862217500805855,80.66227205796167,82.44955762801692,0.6384039986878633,1678590526,48 diff --git a/docs/docs/benchmarks/figures/ivf-dbpedia-qps-recall.svg b/docs/docs/benchmarks/figures/ivf-dbpedia-qps-recall.svg new file mode 100644 index 00000000..170b8553 --- /dev/null +++ b/docs/docs/benchmarks/figures/ivf-dbpedia-qps-recall.svg @@ -0,0 +1,1141 @@ + + + + + + + + 2026-10-05T12:13:35.657946 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.5 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.7 + + + + + + + + + + + + + 0.8 + + + + + + + + + + + + + 0.9 + + + + + + + + + + + + + 1.0 + + + + Recall@10 (measured) + + + + + + + + + + + + + + + + + + + + 1 + 0 + 2 + + + + + + + + + + + + + + + + + + 1 + 0 + 3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Queries / second (log scale) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Query set · full sweep · 1 search thread · one timing pass + + + + DBpedia · IVF indexes | Recall@10 vs QPS + + + One Intel Xeon Gold 6418H CPU · one query per call + + + Faint markers: all measured settings. Lines: per-method Pareto frontier; no fitted or extrapolated points. + + + + + + + + + + + + + RaBitQ IVF raw + + + + + + + + + + + + RaBitQ IVF 1+4 + + + + + + + + + + + + RaBitQ IVF 1+8 + + + + + + + + + + + + Faiss IVF RaBitQfs raw + RFlat + + + + + + + + + + + + Faiss IVF RaBitQfs 1+4 + + + + + + + + + + + + Faiss IVF RaBitQfs 1+8 + + + + + + + + + + + + Faiss IVF PQfs + RFlat + + + + + + + + + + + + Faiss OPQ-IVF PQfs + RFlat + + + + + + + + + diff --git a/docs/docs/benchmarks/figures/ivf-gist1m-qps-recall.svg b/docs/docs/benchmarks/figures/ivf-gist1m-qps-recall.svg new file mode 100644 index 00000000..cb772745 --- /dev/null +++ b/docs/docs/benchmarks/figures/ivf-gist1m-qps-recall.svg @@ -0,0 +1,1136 @@ + + + + + + + + 2026-10-05T12:13:35.413670 + image/svg+xml + + + Matplotlib v3.11.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.2 + + + + + + + + + + + + + 0.4 + + + + + + + + + + + + + 0.6 + + + + + + + + + + + + + 0.8 + + + + + + + + + + + + + 1.0 + + + + Recall@10 (measured) + + + + + + + + + + + + + + + + + + + + 1 + 0 + 2 + + + + + + + + + + + + + + + + + + 1 + 0 + 3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Queries / second (log scale) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Query set · full sweep · 1 search thread · one timing pass + + + + GIST1M · IVF indexes | Recall@10 vs QPS + + + One Intel Xeon Gold 6418H CPU · one query per call + + + Faint markers: all measured settings. Lines: per-method Pareto frontier; no fitted or extrapolated points. + + + + + + + + + + + + + RaBitQ IVF raw + + + + + + + + + + + + RaBitQ IVF 1+4 + + + + + + + + + + + + RaBitQ IVF 1+8 + + + + + + + + + + + + Faiss IVF RaBitQfs raw + RFlat + + + + + + + + + + + + Faiss IVF RaBitQfs 1+4 + + + + + + + + + + + + Faiss IVF RaBitQfs 1+8 + + + + + + + + + + + + Faiss IVF PQfs + RFlat + + + + + + + + + + + + Faiss OPQ-IVF PQfs + RFlat + + + + + + + + + diff --git a/docs/docs/benchmarks/figures/kmeans.csv b/docs/docs/benchmarks/figures/kmeans.csv new file mode 100644 index 00000000..f883c611 --- /dev/null +++ b/docs/docs/benchmarks/figures/kmeans.csv @@ -0,0 +1,7 @@ +dataset,method,clusters,threads,iteration_limit,actual_iterations,total_s,training_s,assignment_s,wcss,recall_at_10,nprobe +gist1m,qgkmeans,4096,48,25,25,16.000009641051292,,,1034787.8701221772,0.8196,40 +gist1m,faiss,4096,48,25,25,84.17002899572253,81.0265609798953,3.1434680158272386,1034624.8831152479,0.8157,40 +gist1m,superkmeans,4096,48,25,25,28.14010765682906,24.449575346894562,3.690532309934497,1034648.386635695,0.8207,40 +dbpedia,qgkmeans,4096,48,25,25,24.079052528832108,,,620378.1316164403,0.8939,40 +dbpedia,faiss,4096,48,25,25,171.03296095784754,164.48468135716394,6.5482796006836,620737.5338938761,0.8947,40 +dbpedia,superkmeans,4096,48,25,24,35.231373528018594,28.58055611886084,6.650817409157753,620268.6283306194,0.8968,40 diff --git a/docs/docs/benchmarks/figures/qps-recall-points.csv b/docs/docs/benchmarks/figures/qps-recall-points.csv new file mode 100644 index 00000000..bc7e182d --- /dev/null +++ b/docs/docs/benchmarks/figures/qps-recall-points.csv @@ -0,0 +1,513 @@ +dataset,method,setting,recall_at_10,qps +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 1}",0.1475,2005.5746227980385 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 2}",0.23810000000000003,1906.8524901153387 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 4}",0.35020000000000007,1779.1404505805135 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 8}",0.48510000000000003,1774.4217638060154 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 16}",0.6271,1651.0425174693853 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 32}",0.766,1476.567344819367 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 64}",0.8728,1192.7934358876 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 128}",0.9492,932.5364963464742 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 256}",0.9821,662.6287968573585 +gist1m,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 512}",0.9953,407.7514834188819 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 1}",0.1474,2294.9258917582733 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 2}",0.2378,2187.2140967943665 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 4}",0.3495,2234.462407651983 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 8}",0.48360000000000003,1990.4888666202082 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 16}",0.6241000000000001,1911.0539099126759 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 32}",0.7588,1613.7567165441699 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 64}",0.8593000000000001,1383.108674821 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 128}",0.9264000000000001,934.567005118496 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 256}",0.9537,643.7376483715516 +gist1m,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 512}",0.9634000000000001,383.0911513839726 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 1}",0.1475,1994.9712648142875 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 2}",0.23820000000000002,1989.035275960661 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 4}",0.3502,1975.6591202286056 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 8}",0.48510000000000003,1818.6157801264026 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 16}",0.6269,1731.9996225494447 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 32}",0.7657999999999999,1514.4611830237568 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 64}",0.8723,1270.5339074039125 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 128}",0.9478,918.0106485865324 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 256}",0.9803000000000001,619.537917897483 +gist1m,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 512}",0.9930999999999999,367.29196973273844 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 1}",0.16110000000000002,2014.0927239452988 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 1}",0.1637,1997.9254110236252 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 1}",0.16390000000000002,1918.577262395092 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 1}",0.16390000000000002,1728.2122414469577 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 1}",0.16390000000000002,1623.3449678326283 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 2}",0.24520000000000003,1982.6481064807965 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 2}",0.2534,1955.1694877773923 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 2}",0.2539,1785.8741169920397 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 2}",0.2539,1666.4905214807104 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 2}",0.2539,1488.2101025879165 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 4}",0.3422,1879.6173478863616 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 4}",0.36169999999999997,1721.1598559405081 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 4}",0.36310000000000003,1734.9449097960292 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 4}",0.36319999999999997,1559.2713780062663 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 4}",0.36319999999999997,1415.9496188801315 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 8}",0.44570000000000004,1693.7471941799326 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 8}",0.49010000000000004,1743.757572578668 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 8}",0.49720000000000003,1550.3316411678136 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 8}",0.4973,1486.7822071791318 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 8}",0.4973,1294.9142110662222 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 16}",0.5398000000000001,1493.5445660838593 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 16}",0.6217,1465.727031861363 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 16}",0.6379000000000001,1374.653297476162 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 16}",0.6386,1239.6214525725215 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 16}",0.6386,1120.8485626975164 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 32}",0.61,1294.3514915225362 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 32}",0.7322000000000001,1255.3903799922696 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 32}",0.7671,1166.8116808652376 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 32}",0.7691,1136.2692968177803 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 32}",0.7693000000000001,935.6597894326466 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 64}",0.6579,943.098879589341 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 64}",0.8146000000000001,901.6137426901545 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 64}",0.8736000000000002,890.9907884671277 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 64}",0.8785,806.469413126132 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 64}",0.879,729.7286761370431 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 128}",0.6807,616.4404780876743 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 128}",0.8627,600.7859352611199 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 128}",0.9431000000000002,577.4858472220362 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 128}",0.9517,557.4700368054645 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 128}",0.9526000000000001,517.993713585731 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 256}",0.6902,353.06352073256744 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 256}",0.8806000000000002,354.2028342092967 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 256}",0.9741000000000001,334.15929223192876 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 256}",0.9851,317.7777737113751 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 256}",0.9864,316.74291882398336 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 512}",0.692,193.8137230021644 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 512}",0.8851000000000001,192.8411054978358 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 512}",0.983,188.14739466321043 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 512}",0.9954000000000001,185.46629153308214 +gist1m,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 512}",0.997,178.57588989620405 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 1}",0.16369999999999998,1592.032676264678 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 2}",0.2537,1500.9674093432843 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 4}",0.36269999999999997,1359.3810190949885 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 8}",0.49560000000000004,1190.2382362413925 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 16}",0.6345,992.6690772655062 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 32}",0.7622000000000001,775.7118107776843 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 64}",0.8656,556.0360719656089 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 128}",0.9311000000000001,360.7544748091443 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 256}",0.9578000000000001,220.3956629987671 +gist1m,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 512}",0.9658000000000001,121.115565416893 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 1}",0.16390000000000002,1543.401622462444 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 2}",0.2539,1461.8017783546595 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 4}",0.36319999999999997,1348.5977355145262 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 8}",0.49710000000000004,1156.3799051268225 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 16}",0.6379000000000001,958.277046678444 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 32}",0.7677999999999999,746.508123092053 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 64}",0.8767,536.1587668921776 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 128}",0.9494000000000001,356.60418284621016 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 256}",0.9829000000000001,209.40029602322767 +gist1m,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 512}",0.9934,114.47128637315288 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 1}",0.1526,2028.605677633766 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 1}",0.16010000000000002,2095.7239728662466 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 1}",0.16310000000000002,2016.8953344994682 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 1}",0.16440000000000002,1961.349900767315 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 1}",0.16469999999999999,1844.1920677175103 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 2}",0.2266,2101.220895249185 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 2}",0.244,2055.688747916972 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 2}",0.2495,2004.7142987037864 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 2}",0.2519,1873.0682695966493 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 2}",0.2525,1800.8618714648921 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 4}",0.3119,2006.9042761545759 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 4}",0.34270000000000006,2028.6071857696131 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 4}",0.35579999999999995,1902.4321166168388 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 4}",0.3592,1857.58568323878 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 4}",0.36069999999999997,1723.5035748248365 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 8}",0.4051,1909.3937852631207 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 8}",0.4636,1925.3308030627045 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 8}",0.4885,1880.3941743892394 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 8}",0.4929,1792.6170974723138 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 8}",0.4955,1643.2750325469647 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 16}",0.4904,1730.3759506185766 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 16}",0.5854000000000001,1625.7567470770084 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 16}",0.6278,1667.2822663865024 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 16}",0.6347,1592.8240755298643 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 16}",0.6381,1475.1356127223619 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 32}",0.5509,1464.5933358616192 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 32}",0.6864000000000001,1480.0441787488417 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 32}",0.7527,1383.295575861576 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 32}",0.762,1360.9845192222233 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 32}",0.766,1263.614207674181 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 64}",0.5962000000000001,1126.6829642104362 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 64}",0.7684000000000001,1134.1650158568912 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 64}",0.8582000000000001,1058.1950373252105 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 64}",0.8712000000000001,1056.6600296676477 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 64}",0.8753,1015.4670345066979 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 128}",0.6179,777.6889858423657 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 128}",0.8159000000000001,775.3911795331736 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 128}",0.9255,759.3758320477604 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 128}",0.943,744.8929681592651 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 128}",0.9478000000000001,712.837261614546 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 256}",0.6256,474.89216874781187 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 256}",0.8359000000000001,470.6893005827663 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 256}",0.9577,469.5202108017892 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 256}",0.9778000000000001,467.1444682918804 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 256}",0.9829000000000001,452.19644630794085 +gist1m,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 512}",0.6277999999999999,268.8052681068619 +gist1m,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 512}",0.8407,271.03520789355 +gist1m,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 512}",0.9677,263.67616723233624 +gist1m,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 512}",0.9881,261.6563159383658 +gist1m,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 512}",0.9933,257.08451350728467 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 1}",0.1559,1784.5656018012419 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 1}",0.1615,1747.6089459574653 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 1}",0.1632,1670.3552134518961 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 1}",0.1635,1638.6820197860545 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 1}",0.1635,1487.6372643917318 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 2}",0.2384,1706.4431079014078 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 2}",0.25010000000000004,1723.8385836280152 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 2}",0.2533,1602.1742428780115 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 2}",0.2542,1576.378233543038 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 2}",0.2544,1531.8189308215767 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 4}",0.33120000000000005,1675.9650655326373 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 4}",0.3546,1608.9755707749812 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 4}",0.36179999999999995,1572.0997940764921 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 4}",0.36329999999999996,1555.2117189817004 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 4}",0.36369999999999997,1464.2734122498275 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 8}",0.43410000000000004,1600.535334589287 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 8}",0.4806,1611.6876736575734 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 8}",0.49339999999999995,1618.0509087462776 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 8}",0.49539999999999995,1521.8816006204013 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 8}",0.4965,1403.0743504167926 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 16}",0.5307999999999999,1501.2403513499332 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 16}",0.6125,1489.3607652025612 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 16}",0.6347999999999999,1446.4064906493961 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 16}",0.6384000000000001,1387.1015503098088 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 16}",0.6399,1246.6591387967349 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 32}",0.6036,1305.9999047886893 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 32}",0.7263000000000001,1289.8005079703714 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 32}",0.763,1236.3791536276754 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 32}",0.7673000000000001,1165.5193168251756 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 32}",0.7689000000000001,1137.3662712092678 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 64}",0.6566,1005.7368015230768 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 64}",0.8166000000000001,984.8700316146329 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 64}",0.8704,987.2523788517957 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 64}",0.8756000000000002,938.875334083024 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 64}",0.8772000000000001,906.1464612449558 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 128}",0.6884,689.8760278294645 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 128}",0.8736000000000002,707.8029011410888 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 128}",0.9434000000000001,689.8866328114932 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 128}",0.9499000000000001,679.3009026373027 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 128}",0.9516000000000001,658.6409236423225 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 256}",0.7012,453.52078414434004 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 256}",0.8972000000000001,451.71709261321024 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 256}",0.9761,442.2456822544139 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 256}",0.9837,440.5149620758961 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 256}",0.9854,421.70265787816714 +gist1m,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 512}",0.7028999999999999,258.6469441946739 +gist1m,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 512}",0.9036000000000002,261.18576029889203 +gist1m,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 512}",0.9863,254.4628404248078 +gist1m,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 512}",0.9944,253.25533247918946 +gist1m,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 512}",0.996,250.83951174852962 +gist1m,rabitq-symqg-raw,"{""ef_search"": 16}",0.5277000000000001,43049.345108730246 +gist1m,rabitq-symqg-raw,"{""ef_search"": 32}",0.6781,28444.967397249373 +gist1m,rabitq-symqg-raw,"{""ef_search"": 64}",0.8024000000000001,17233.629283747097 +gist1m,rabitq-symqg-raw,"{""ef_search"": 128}",0.8978,9687.31560212227 +gist1m,rabitq-symqg-raw,"{""ef_search"": 256}",0.9528,5224.452300620841 +gist1m,rabitq-symqg-raw,"{""ef_search"": 512}",0.9825000000000002,2766.5813449059688 +gist1m,rabitq-symqg-raw,"{""ef_search"": 1024}",0.9942000000000001,1434.8375549171385 +gist1m,rabitq-symqg-4,"{""ef_search"": 16}",0.46,42474.21657960787 +gist1m,rabitq-symqg-4,"{""ef_search"": 32}",0.6099000000000001,28386.14149336272 +gist1m,rabitq-symqg-4,"{""ef_search"": 64}",0.7346,17445.149760988257 +gist1m,rabitq-symqg-4,"{""ef_search"": 128}",0.8256000000000001,9685.49389174208 +gist1m,rabitq-symqg-4,"{""ef_search"": 256}",0.8785,5352.848416029368 +gist1m,rabitq-symqg-4,"{""ef_search"": 512}",0.9071000000000001,2795.8813332582777 +gist1m,rabitq-symqg-4,"{""ef_search"": 1024}",0.9180000000000001,1423.7580116393246 +gist1m,rabitq-symqg-8,"{""ef_search"": 16}",0.5310999999999999,41533.079304319974 +gist1m,rabitq-symqg-8,"{""ef_search"": 32}",0.6903,31805.500220722224 +gist1m,rabitq-symqg-8,"{""ef_search"": 64}",0.8132,19773.006623137546 +gist1m,rabitq-symqg-8,"{""ef_search"": 128}",0.8988,11038.674484585328 +gist1m,rabitq-symqg-8,"{""ef_search"": 256}",0.9487000000000001,5527.2223112275515 +gist1m,rabitq-symqg-8,"{""ef_search"": 512}",0.9769000000000001,3069.082987321068 +gist1m,rabitq-symqg-8,"{""ef_search"": 1024}",0.987,1585.6912675570236 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 16}",0.48269999999999996,16623.81288332163 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 32}",0.6199,13256.81845651098 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 64}",0.7386000000000001,9565.57921067687 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 128}",0.8246000000000001,5913.002459902097 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 256}",0.8876000000000002,3286.413632427297 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 512}",0.9227000000000002,1664.012086521201 +gist1m,rabitq-hnsw-1p4,"{""ef_search"": 1024}",0.9413,863.4080965711864 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 16}",0.4811,17188.089078216053 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 32}",0.6146,12660.226395341044 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 64}",0.7385,8378.505355339506 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 128}",0.8385,5973.355754343055 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 256}",0.9103000000000001,3222.2040553716165 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 512}",0.9482,1738.7735148166698 +gist1m,rabitq-hnsw-1p8,"{""ef_search"": 1024}",0.9714,895.9809987931593 +gist1m,faiss-hnsw-flat,"{""ef_search"": 16}",0.49739999999999995,6294.544088577019 +gist1m,faiss-hnsw-flat,"{""ef_search"": 32}",0.6306999999999999,3912.743611640825 +gist1m,faiss-hnsw-flat,"{""ef_search"": 64}",0.7554,2388.2965969556044 +gist1m,faiss-hnsw-flat,"{""ef_search"": 128}",0.8548000000000001,1331.380958697115 +gist1m,faiss-hnsw-flat,"{""ef_search"": 256}",0.9228,751.0712246713382 +gist1m,faiss-hnsw-flat,"{""ef_search"": 512}",0.9574000000000001,407.8336037636356 +gist1m,faiss-hnsw-flat,"{""ef_search"": 1024}",0.9771,222.5859880216539 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 16}",0.34470000000000006,6197.970217555145 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 32}",0.4183,3636.312820419594 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 64}",0.47479999999999994,2352.13635964156 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 128}",0.5145,1293.9673255270873 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 256}",0.5379,718.0683898430536 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 512}",0.5458,394.58646313798147 +gist1m,faiss-hnsw-sq4,"{""ef_search"": 1024}",0.549,212.53348975574198 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 16}",0.4975,6094.257849760436 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 32}",0.6339,3791.4577553857866 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 64}",0.7509000000000001,2225.8304053714965 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 128}",0.8434,1275.5174278302363 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 256}",0.9039,698.224703435659 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 512}",0.9374000000000001,385.09922727219015 +gist1m,faiss-hnsw-sq8,"{""ef_search"": 1024}",0.9548000000000001,210.04956181553456 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 1}",0.4968,1374.4212189774769 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 2}",0.6109,1370.4518594659282 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 4}",0.6996,1347.4148879925317 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 8}",0.7773,1320.816066814605 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 16}",0.8334000000000001,1186.589405565198 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 32}",0.8849000000000001,1098.5848988193557 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 64}",0.9256,910.514271530349 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 128}",0.9569000000000001,695.9454324537319 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 256}",0.9774,436.47399494701176 +dbpedia,rabitq-ivf-raw,"{""k_factor"": 1, ""nprobe"": 512}",0.9906,281.38156180266844 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 1}",0.49570000000000003,1391.1172346906208 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 2}",0.6088,1392.3712271631307 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 4}",0.6961,1370.8888049917703 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 8}",0.7725,1322.5239117561923 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 16}",0.8275,1232.0487448574072 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 32}",0.8771,1104.393741637969 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 64}",0.9152,876.0008199462742 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 128}",0.9441000000000002,656.2514529510163 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 256}",0.9633,433.53707927739987 +dbpedia,rabitq-ivf-1p4,"{""k_factor"": 1, ""nprobe"": 512}",0.9751000000000001,254.98675174629898 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 1}",0.4968,1271.513630759813 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 2}",0.6107,1234.3159645906096 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 4}",0.6992,1229.7119781374638 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 8}",0.7769000000000001,1179.5804838484507 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 16}",0.8329000000000001,1098.0677908415962 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 32}",0.8842000000000001,986.4392441072248 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 64}",0.9248000000000002,820.9255438807701 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 128}",0.9557,611.6030848186699 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 256}",0.9761000000000002,407.84238320781185 +dbpedia,rabitq-ivf-1p8,"{""k_factor"": 1, ""nprobe"": 512}",0.9892000000000001,244.97560100124187 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 1}",0.4696,968.0227209640274 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 1}",0.4868,964.4443231982366 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 1}",0.48760000000000003,923.9652572942317 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 1}",0.4877,912.6461488194564 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 1}",0.4877,872.0379317355168 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 2}",0.5753,939.9173573635761 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 2}",0.6055,917.9221395303979 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 2}",0.6072000000000001,911.2899651766315 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 2}",0.6073,892.0257837615146 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 2}",0.6073,843.016559458561 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 4}",0.6524000000000001,904.5856228241572 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 4}",0.6984,904.2420363318543 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 4}",0.7008000000000001,873.2209043282227 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 4}",0.7009000000000001,831.8374546976711 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 4}",0.7009000000000001,806.4086176992099 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 8}",0.7143,852.4079588648309 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 8}",0.775,843.0938172938759 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 8}",0.7789,827.8275576433284 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 8}",0.779,796.2944762690637 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 8}",0.779,752.1383224169732 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 16}",0.7577,768.9078966246028 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 16}",0.8328000000000002,759.0074396963016 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 16}",0.8379,742.6401397452793 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 16}",0.838,734.0170078706251 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 16}",0.838,683.6893265657947 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 32}",0.7901,631.7635288569667 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 32}",0.8777,628.251282336309 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 32}",0.8842000000000001,604.1106522307605 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 32}",0.8843,591.6300892217697 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 32}",0.8843,566.7554004416071 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 64}",0.8160000000000001,461.5795570890865 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 64}",0.9163000000000001,465.14511198475395 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 64}",0.9253000000000001,464.299218067614 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 64}",0.9255,463.8142727836377 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 64}",0.9255,435.69214220010394 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 128}",0.8347,314.9745859958468 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 128}",0.9451000000000002,311.54180700626364 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 128}",0.9567,308.54082670237926 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 128}",0.957,298.1115146139451 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 128}",0.957,294.1281228048454 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 256}",0.8457,181.37061191132904 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 256}",0.9654,183.00220126319184 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 256}",0.9779000000000001,181.9718697932391 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 256}",0.9782000000000001,180.31477024788836 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 256}",0.9782000000000001,178.07071264081796 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 1, ""nprobe"": 512}",0.8502000000000001,98.70005756713317 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 2, ""nprobe"": 512}",0.9766000000000001,96.99534741166593 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 5, ""nprobe"": 512}",0.9897999999999999,99.08805652452423 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 10, ""nprobe"": 512}",0.9902000000000001,97.82257797000989 +dbpedia,faiss-rabitqfs-raw,"{""k_factor"": 20, ""nprobe"": 512}",0.9902000000000001,94.4008829040908 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 1}",0.48669999999999997,825.0482400638849 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 2}",0.6056,775.65225957504 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 4}",0.6978000000000001,724.745866787313 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 8}",0.7748,638.217158167753 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 16}",0.8328000000000001,553.6877684149682 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 32}",0.8778,435.31099997503213 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 64}",0.9169,333.26588528787596 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 128}",0.9458000000000001,223.703495138427 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 256}",0.9655,134.68955966992752 +dbpedia,faiss-rabitqfs-1p4,"{""k_factor"": 1, ""nprobe"": 512}",0.9763000000000001,75.56264372587856 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 1}",0.48760000000000003,847.3705705918125 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 2}",0.607,789.7986329445511 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 4}",0.7004000000000001,727.4150281955918 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 8}",0.7785,646.7648584028299 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 16}",0.8374000000000001,561.243542843872 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 32}",0.8837,447.43567160397447 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 64}",0.9247000000000001,332.5258855329815 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 128}",0.9559000000000001,226.94462431412106 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 256}",0.9767,139.69709301950812 +dbpedia,faiss-rabitqfs-1p8,"{""k_factor"": 1, ""nprobe"": 512}",0.9885999999999999,75.64067479048292 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 1}",0.4701,1295.6882144217889 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 1}",0.48760000000000003,1277.6125812296723 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 1}",0.4877,1323.6018703778934 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 1}",0.4877,1265.9186715306453 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 1}",0.4877,1186.028558274091 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 2}",0.5791000000000001,1318.195103552386 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 2}",0.6071,1255.4120512423547 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 2}",0.6073999999999999,1258.736108636392 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 2}",0.6073999999999999,1251.8981495689254 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 2}",0.6073999999999999,1158.5294431200525 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 4}",0.6611999999999999,1277.4898869025874 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 4}",0.7004000000000001,1223.7098422204895 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 4}",0.7009000000000001,1237.20019115721 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 4}",0.7009000000000001,1213.1183526553668 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 4}",0.7009000000000001,1167.1537107107254 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 8}",0.7254,1217.9099021618445 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 8}",0.778,1212.2349126248885 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 8}",0.7791,1221.4943358680118 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 8}",0.7791,1171.4891067002634 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 8}",0.7791,1119.4126167761694 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 16}",0.7736000000000001,1113.356827371854 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 16}",0.8365,1152.804265182153 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 16}",0.8382000000000001,1109.55037773676 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 16}",0.8382000000000001,1059.0094510865404 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 16}",0.8382000000000001,1008.412883424824 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 32}",0.8084000000000001,990.3375202865902 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 32}",0.8823000000000001,980.6390055894976 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 32}",0.8843000000000001,934.3381468925984 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 32}",0.8843000000000001,889.3081631385256 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 32}",0.8843000000000001,848.2109758289333 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 64}",0.8373000000000002,762.3896122216739 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 64}",0.9228,769.1450456924327 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 64}",0.9255,759.0237056886601 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 64}",0.9255,751.0103895316598 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 64}",0.9255,723.3633774250434 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 128}",0.8594,563.1297619039736 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 128}",0.9537,561.6980305275633 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 128}",0.9569000000000001,546.2500894000085 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 128}",0.9569000000000001,525.6698312113972 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 128}",0.9569000000000001,513.0381049439138 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 256}",0.8726000000000002,350.58065282357325 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 256}",0.9747,343.9182135358094 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 256}",0.9782000000000001,343.5952007234832 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 256}",0.9782000000000001,337.3037258005554 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 256}",0.9782000000000001,334.8449388982761 +dbpedia,faiss-pqfs,"{""k_factor"": 1, ""nprobe"": 512}",0.8801000000000001,202.62996696348702 +dbpedia,faiss-pqfs,"{""k_factor"": 2, ""nprobe"": 512}",0.9864,200.84169988397014 +dbpedia,faiss-pqfs,"{""k_factor"": 5, ""nprobe"": 512}",0.9902000000000001,201.63053215772186 +dbpedia,faiss-pqfs,"{""k_factor"": 10, ""nprobe"": 512}",0.9902000000000001,201.8645930097436 +dbpedia,faiss-pqfs,"{""k_factor"": 20, ""nprobe"": 512}",0.9902000000000001,198.87698940503373 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 1}",0.4713,1008.9617269050393 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 1}",0.48769999999999997,1003.5576743274091 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 1}",0.48779999999999996,939.7464312542461 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 1}",0.48779999999999996,973.0575759570302 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 1}",0.48779999999999996,949.3471273735867 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 2}",0.5821,1028.4823982699827 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 2}",0.607,1021.7683570386928 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 2}",0.6073999999999999,987.8112264986651 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 2}",0.6073999999999999,970.0031546414048 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 2}",0.6073999999999999,937.0643086321577 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 4}",0.6652,949.6728925319356 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 4}",0.7004000000000001,997.8604793028255 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 4}",0.701,979.5610032121201 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 4}",0.701,957.0874976471736 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 4}",0.701,918.5603264908976 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 8}",0.7315000000000002,979.4959044485424 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 8}",0.7783000000000001,957.4495864802377 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 8}",0.779,951.1933656973697 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 8}",0.779,871.8322095758888 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 8}",0.779,888.7023386511922 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 16}",0.7796000000000001,899.7706610205627 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 16}",0.8372,904.6396394792969 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 16}",0.8379000000000001,895.0253809625042 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 16}",0.8379000000000001,873.6963683696433 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 16}",0.8379000000000001,836.131233909442 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 32}",0.8153000000000001,798.6155813341438 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 32}",0.8828,804.1283312525117 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 32}",0.8842000000000001,789.7190475337088 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 32}",0.8842000000000001,764.5939936757694 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 32}",0.8842000000000001,729.6210068825907 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 64}",0.8466000000000001,659.218034316247 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 64}",0.9242999999999999,642.4006857781694 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 64}",0.9264000000000001,632.8695988449366 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 64}",0.9264000000000001,632.5800970821572 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 64}",0.9264000000000001,614.4044615948985 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 128}",0.8678000000000001,482.62501651343 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 128}",0.9547,483.0607231122143 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 128}",0.9572,480.5873371519198 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 128}",0.9572,461.9904566510287 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 128}",0.9572,448.57333456590874 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 256}",0.8824000000000002,316.3086163853317 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 256}",0.9757,314.92861306695534 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 256}",0.9784,303.7093100948087 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 256}",0.9784,308.5776657202282 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 256}",0.9784,306.6846440561585 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 1, ""nprobe"": 512}",0.8897000000000003,190.96561085955966 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 2, ""nprobe"": 512}",0.9873000000000001,193.85763250385511 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 5, ""nprobe"": 512}",0.9903,190.42143321970212 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 10, ""nprobe"": 512}",0.9903,191.85913955260074 +dbpedia,faiss-opq-pqfs,"{""k_factor"": 20, ""nprobe"": 512}",0.9903,186.93282549390355 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 16}",0.7963,26698.333696752536 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 32}",0.9013000000000001,20376.29731345844 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 64}",0.9466,13301.415605960248 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 128}",0.9715,7848.863487608723 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 256}",0.9847,4387.329254954049 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 512}",0.9915999999999999,2501.5673769832497 +dbpedia,rabitq-symqg-raw,"{""ef_search"": 1024}",0.9952000000000001,1325.8432221354449 +dbpedia,rabitq-symqg-4,"{""ef_search"": 16}",0.6593,29952.68190614935 +dbpedia,rabitq-symqg-4,"{""ef_search"": 32}",0.809,20960.976465080435 +dbpedia,rabitq-symqg-4,"{""ef_search"": 64}",0.8912,13828.292001944128 +dbpedia,rabitq-symqg-4,"{""ef_search"": 128}",0.9349000000000001,7963.251127655006 +dbpedia,rabitq-symqg-4,"{""ef_search"": 256}",0.9496000000000001,4516.993546047341 +dbpedia,rabitq-symqg-4,"{""ef_search"": 512}",0.9588000000000001,2508.567032497219 +dbpedia,rabitq-symqg-4,"{""ef_search"": 1024}",0.9627,1301.3927377817472 +dbpedia,rabitq-symqg-8,"{""ef_search"": 16}",0.8014000000000001,32035.82717277324 +dbpedia,rabitq-symqg-8,"{""ef_search"": 32}",0.9055,23357.675584791188 +dbpedia,rabitq-symqg-8,"{""ef_search"": 64}",0.9498000000000001,15520.60669115036 +dbpedia,rabitq-symqg-8,"{""ef_search"": 128}",0.9708,9394.204203881305 +dbpedia,rabitq-symqg-8,"{""ef_search"": 256}",0.9816,5342.00032034833 +dbpedia,rabitq-symqg-8,"{""ef_search"": 512}",0.9899000000000001,2875.7455026921493 +dbpedia,rabitq-symqg-8,"{""ef_search"": 1024}",0.9939000000000001,1560.3160661903744 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 16}",0.8231000000000002,10915.582597467517 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 32}",0.8911,9327.080434161853 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 64}",0.9357000000000001,7299.961293423891 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 128}",0.9571000000000002,5698.5600994254655 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 256}",0.9676000000000001,3336.300026531672 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 512}",0.9742000000000001,1790.4287110935554 +dbpedia,rabitq-hnsw-1p4,"{""ef_search"": 1024}",0.9772000000000001,900.8496320505669 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 16}",0.8337,13272.244704899544 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 32}",0.9037000000000001,10268.81297442337 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 64}",0.9485,7346.406058279471 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 128}",0.9718,5455.384694823672 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 256}",0.985,3169.3801664859816 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 512}",0.9921,1685.5009030052531 +dbpedia,rabitq-hnsw-1p8,"{""ef_search"": 1024}",0.9946999999999999,872.3892420250611 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 16}",0.848,4172.367906120071 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 32}",0.9243000000000001,2742.54484463088 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 64}",0.9645,1649.1643632445584 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 128}",0.9824,928.6216566213442 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 256}",0.9907,532.9354364134833 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 512}",0.9964,286.02655488384795 +dbpedia,faiss-hnsw-flat,"{""ef_search"": 1024}",0.9984,153.97998040359198 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 16}",0.8098000000000001,4590.418383835556 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 32}",0.8811000000000001,2924.588471485705 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 64}",0.9127000000000001,1708.2005676533308 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 128}",0.9244000000000001,981.9610767098859 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 256}",0.9315000000000001,537.2605936900457 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 512}",0.9361000000000002,299.70730382770734 +dbpedia,faiss-hnsw-sq4,"{""ef_search"": 1024}",0.9388000000000002,154.95071054288263 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 16}",0.8485,4621.21956533345 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 32}",0.9218000000000001,2778.3851396221876 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 64}",0.9610000000000001,1727.917067022132 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 128}",0.9778000000000001,992.8289386887817 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 256}",0.9862000000000001,547.8647705358404 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 512}",0.9919,294.52508051061756 +dbpedia,faiss-hnsw-sq8,"{""ef_search"": 1024}",0.9937,157.9149542784322 diff --git a/docs/docs/benchmarks/graphs.md b/docs/docs/benchmarks/graphs.md new file mode 100644 index 00000000..ec1ecc2a --- /dev/null +++ b/docs/docs/benchmarks/graphs.md @@ -0,0 +1,78 @@ +# Graph benchmark + +Settings and results for eight configurations. See the [overview](bench.md) for +hardware, datasets, and measurement definitions. + +## Index configurations + +SymphonyQG uses maximum degree 32, `ef_construction=200`, `init="pipnn"`, and one +refinement iteration. Both HNSW implementations use `M=16` and +`efConstruction=200`, giving a base-layer degree bound of 32. + +| Method | Stored representation | Additional construction settings | +| --- | --- | --- | +| RaBitQ SymphonyQG raw | `quantization_bits=0`, raw vectors | Degree 32, PIPNN initialization | +| RaBitQ SymphonyQG 4-bit | `quantization_bits=4` | Degree 32, PIPNN initialization | +| RaBitQ SymphonyQG 8-bit | `quantization_bits=8` | Degree 32, PIPNN initialization | +| RaBitQ HNSW 1+4 | `nbits=5` | `M=16`, seed 42, 16 residual centroids | +| RaBitQ HNSW 1+8 | `nbits=9` | `M=16`, seed 42, 16 residual centroids | +| Faiss HNSW Flat | `HNSW16` | Float32 vectors | +| Faiss HNSW SQ4 | `HNSW16_SQ4` | Four-bit scalar quantization | +| Faiss HNSW SQ8 | `HNSW16_SQ8` | Eight-bit scalar quantization | + +RaBitQ HNSW trains 16 centroids with RaBitQKMeans (up to 10 iterations, seed 42, +exact final assignment, spherical training for DBpedia's inner-product metric). +Training is included in construction time. SymphonyQG 4/8-bit and HNSW +1+4/1+8 use different encodings; serialized sizes include their full storage costs. + +All graph methods sweep `ef=16,32,64,128,256,512,1024` (Faiss calls this +`efSearch`). + +## Reading the results + +Curves use one search thread and one query per call over all 1,000 queries. +Indexes are built with 48 threads and reused for the query sweeps. +See [figure definitions](bench.md#reading-the-figures) and +[indexing metrics](bench.md#reading-the-indexing-tables). + +## GIST1M + +![GIST1M graphs QPS–recall trade-off](figures/graphs-gist1m-qps-recall.svg) + +### Indexing time and size + + +| Method | Train s | Construct s | Total s | Save s | Index GiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| RaBitQ SymphonyQG raw | 0.00 | 38.74 | 38.74 | 3.27 | 7.51 | +| RaBitQ SymphonyQG 4-bit | 0.00 | 38.41 | 38.41 | 1.86 | 4.39 | +| RaBitQ SymphonyQG 8-bit | 0.00 | 38.42 | 38.42 | 2.04 | 4.84 | +| RaBitQ HNSW 1+4 | 1.24 | 70.57 | 71.81 | 0.35 | 0.72 | +| RaBitQ HNSW 1+8 | 1.39 | 76.69 | 78.09 | 0.55 | 1.16 | +| Faiss HNSW Flat | 0.00 | 78.94 | 78.94 | 1.61 | 3.71 | +| Faiss HNSW SQ4 | 1.06 | 55.11 | 56.17 | 0.24 | 0.58 | +| Faiss HNSW SQ8 | 1.08 | 47.12 | 48.20 | 0.42 | 1.03 | + + +## DBpedia + +![DBpedia graphs QPS–recall trade-off](figures/graphs-dbpedia-qps-recall.svg) + +### Indexing time and size + + +| Method | Train s | Construct s | Total s | Save s | Index GiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| RaBitQ SymphonyQG raw | 0.00 | 59.40 | 59.40 | 5.06 | 11.79 | +| RaBitQ SymphonyQG 4-bit | 0.00 | 57.92 | 57.92 | 2.85 | 6.80 | +| RaBitQ SymphonyQG 8-bit | 0.00 | 57.17 | 57.17 | 3.35 | 7.51 | +| RaBitQ HNSW 1+4 | 1.59 | 111.02 | 112.61 | 0.57 | 1.05 | +| RaBitQ HNSW 1+8 | 1.53 | 122.57 | 124.10 | 0.83 | 1.76 | +| Faiss HNSW Flat | 0.00 | 147.53 | 147.53 | 2.69 | 5.85 | +| Faiss HNSW SQ4 | 1.74 | 101.82 | 103.57 | 0.36 | 0.85 | +| Faiss HNSW SQ8 | 1.79 | 80.66 | 82.45 | 0.64 | 1.56 | + + +## Download measurements + +[QPS–recall CSV](figures/qps-recall-points.csv) · [Indexing CSV](figures/indexing.csv) diff --git a/docs/docs/benchmarks/ivf.md b/docs/docs/benchmarks/ivf.md new file mode 100644 index 00000000..9df7b91f --- /dev/null +++ b/docs/docs/benchmarks/ivf.md @@ -0,0 +1,80 @@ +# IVF benchmark + +Settings and results for eight configurations. See the [overview](bench.md) for +hardware, datasets, and measurement definitions. + +## Index configurations + +Every IVF index uses **4,096 clusters**. `d` is the dataset dimension and +`m=d/2` for PQ: 480 on GIST1M and 768 on DBpedia. PQ subquantizers +use four bits. RaBitQ clustering uses up to 10 iterations, seed 42, exact final +assignment, and spherical training for DBpedia's inner-product metric. Faiss uses its +factory training defaults. Training is included in construction time. + +| Method | Construction specification | Search parameters | +| --- | --- | --- | +| RaBitQ IVF raw | `IvfIndex(nbits=32)`; one-bit filter with original float32 vectors | `nprobe` | +| RaBitQ IVF 1+4 | `IvfIndex(nbits=5)` | `nprobe` | +| RaBitQ IVF 1+8 | `IvfIndex(nbits=9)` | `nprobe` | +| Faiss IVF RaBitQfs raw + RFlat | `HRd,IVF4096,RaBitQfs1,RFlat` | `nprobe`, `k_factor` | +| Faiss IVF RaBitQfs 1+4 | `HRd,IVF4096,RaBitQfs5` | `nprobe` | +| Faiss IVF RaBitQfs 1+8 | `HRd,IVF4096,RaBitQfs9` | `nprobe` | +| Faiss IVF PQfs + RFlat | `IVF4096,PQ{m}x4fs,RFlat` | `nprobe`, `k_factor` | +| Faiss OPQ-IVF PQfs + RFlat | `OPQ{m},IVF4096,PQ{m}x4fs,RFlat` | `nprobe`, `k_factor` | + +`1+4` and `1+8` denote one sign bit plus four or eight extra bits. +`nbits=32` retains raw float32 vectors in RaBitQ-Library; `RFlat` retains them +for Faiss reranking. Index size and QPS include retained vectors and reranking. + +The search grid is `nprobe=1,2,4,8,16,32,64,128,256,512`, crossed with +`k_factor=1,2,5,10,20` for the `RFlat` methods. RaBitQ quantized IVF uses its +default HACC search mode for total bit widths 5 and 9. + +## Reading the results + +Curves use one search thread and one query per call over all 1,000 queries. +Indexes are built with 48 threads and reused for the query sweeps. +See [figure definitions](bench.md#reading-the-figures) and +[indexing metrics](bench.md#reading-the-indexing-tables). + +## GIST1M + +![GIST1M ivf QPS–recall trade-off](figures/ivf-gist1m-qps-recall.svg) + +### Indexing time and size + + +| Method | Train s | Construct s | Total s | Save s | Index GiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| RaBitQ IVF raw | 14.70 | 0.66 | 15.37 | 1.55 | 3.73 | +| RaBitQ IVF 1+4 | 14.84 | 1.71 | 16.56 | 0.25 | 0.60 | +| RaBitQ IVF 1+8 | 14.54 | 7.03 | 21.57 | 0.45 | 1.05 | +| Faiss IVF RaBitQfs raw + RFlat | 41.61 | 6.76 | 48.37 | 1.58 | 3.73 | +| Faiss IVF RaBitQfs 1+4 | 41.24 | 13.16 | 54.40 | 0.29 | 0.68 | +| Faiss IVF RaBitQfs 1+8 | 41.38 | 39.82 | 81.20 | 0.52 | 1.19 | +| Faiss IVF PQfs + RFlat | 34.52 | 6.59 | 41.11 | 1.59 | 3.84 | +| Faiss OPQ-IVF PQfs + RFlat | 232.45 | 7.51 | 239.97 | 1.63 | 3.84 | + + +## DBpedia + +![DBpedia ivf QPS–recall trade-off](figures/ivf-dbpedia-qps-recall.svg) + +### Indexing time and size + + +| Method | Train s | Construct s | Total s | Save s | Index GiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| RaBitQ IVF raw | 24.29 | 1.89 | 26.18 | 2.58 | 5.95 | +| RaBitQ IVF 1+4 | 24.33 | 2.89 | 27.22 | 0.40 | 0.95 | +| RaBitQ IVF 1+8 | 24.36 | 12.42 | 36.77 | 0.70 | 1.67 | +| Faiss IVF RaBitQfs raw + RFlat | 85.29 | 12.97 | 98.26 | 2.71 | 6.02 | +| Faiss IVF RaBitQfs 1+4 | 86.94 | 28.05 | 114.99 | 0.60 | 1.33 | +| Faiss IVF RaBitQfs 1+8 | 119.86 | 91.11 | 210.97 | 1.00 | 2.34 | +| Faiss IVF PQfs + RFlat | 99.82 | 15.40 | 115.22 | 2.56 | 6.13 | +| Faiss OPQ-IVF PQfs + RFlat | 1414.28 | 13.65 | 1427.93 | 2.73 | 6.14 | + + +## Download measurements + +[QPS–recall CSV](figures/qps-recall-points.csv) · [Indexing CSV](figures/indexing.csv) diff --git a/docs/docs/benchmarks/kmeans.md b/docs/docs/benchmarks/kmeans.md new file mode 100644 index 00000000..a62923c3 --- /dev/null +++ b/docs/docs/benchmarks/kmeans.md @@ -0,0 +1,84 @@ +# K-means benchmark + +Compare **QGKMeans**, **Faiss K-means**, and **SuperKMeans** using their Python APIs. +Metrics are total time, WCSS, and recall@10 with 1% of clusters probed. + +## Settings + +| Setting | Value | +| --- | --- | +| Threads | 48; methods run sequentially in separate processes | +| Clustering | Flat L2, 4,096 clusters | +| Training | All raw float32 base vectors; no sampling, quantization, or hierarchy | +| Iterations | At most 25; stopping thresholds below | +| Final assignment | Exact nearest centroid after the last update | +| Seed / fits | 42 / one fit per method and dataset | +| Queries | One fixed set of 1,000 per dataset; never used for training or stopping | + +Both [datasets](bench.md#datasets-and-ground-truth) use the full base set and +1,000 queries. GIST uses supplied L2 ground truth; OpenAI uses exhaustive L2 +search. Centroids are not normalized. + +| Method | Additional settings | +| --- | --- | +| `rabitqlib.QGKMeans` | `quantization_bits=0`, degree 32, `ef_build=240`, `ef_search=16`, one graph refinement, `early_stop_threshold=0`, `FinalAssignmentMode.Exact` | +| `faiss.Kmeans` | CPU, `nredo=1`, `max_points_per_centroid=N`, `spherical=False`, `early_stop_threshold=0`; exact final `index.search(x, 1)` | +| `superkmeans.SuperKMeans` | v0.2.0, `quantizer="f32"`, `hierarchical=False`, `sampling_fraction=1`, `angular=False`, `use_blas_only=False`, `early_termination=True`, `tol=1e-4`; exact final `assign(x, centroids)` | + +`N` is the full base count. Methods retain their own training assignment algorithms +and initializers; the same seed does not guarantee identical starting centroids. + +## Metrics + +- **Time:** training plus exact final assignment, including initialization and + internal transformations. QGKMeans includes assignment inside `train()`; + Faiss and SuperKMeans report separate components in the CSV. Loading, evaluation, + and saving are excluded. +- **WCSS:** sum of squared L2 distances to final assigned centroids, recomputed + with float64 accumulation. Lower is better. +- **Recall@10:** fraction of exact top-10 neighbors in the nearest + `floor(clusters × 1%)` clusters, averaged over queries: 40 probes for both + datasets, following SuperKMeans. + This is the recall available from exact scanning of those lists. + +Exact assignments are checked on 128 base vectors using float64 distances, +outside timing. + +## Results + +### GIST1M + + +| Method | Total time (s) | WCSS | Recall@10 (40 probes) | Iterations | +| --- | ---: | ---: | ---: | ---: | +| QGKMeans | 16.00 | 1034787.87 | 0.819600 | 25 | +| Faiss K-means | 84.17 | 1034624.88 | 0.815700 | 25 | +| SuperKMeans | 28.14 | 1034648.39 | 0.820700 | 25 | + + +### DBpedia + + +| Method | Total time (s) | WCSS | Recall@10 (40 probes) | Iterations | +| --- | ---: | ---: | ---: | ---: | +| QGKMeans | 24.08 | 620378.132 | 0.893900 | 25 | +| Faiss K-means | 171.03 | 620737.534 | 0.894700 | 25 | +| SuperKMeans | 35.23 | 620268.628 | 0.896800 | 24 | + + +[Download measurements (CSV)](figures/kmeans.csv) + +## Reproduction + +Follow the [installation instructions](bench.md#package-installation), then run: + +```bash +python benchmarks/reproduce.py --benchmarks kmeans +``` + +Results, centroids, and assignments are saved in `results/benchmarks/kmeans/`. +Use `--datasets gist1m` or `--methods qgkmeans` for a subset, and a fresh +`--output-dir` to repeat measurements. See the [reproduction options](bench.md#data-and-reproduction). + +Protocol references: SuperKMeans' [benchmark guide](https://github.com/cwida/SuperKMeans/blob/main/BENCHMARKING.md) +and [recall evaluator](https://github.com/cwida/SuperKMeans/blob/main/benchmarks/bench_utils.h). diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 894d3ac6..830be558 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -25,6 +25,11 @@ nav: - Integrations: - Zvec: integrations/zvec.md - Clustering: clustering.md + - Benchmarks: + - Overview: benchmarks/bench.md + - IVF Indexes: benchmarks/ivf.md + - Graph Indexes: benchmarks/graphs.md + - K-means: benchmarks/kmeans.md theme: name: material diff --git a/tests/python/test_benchmark_protocol.py b/tests/python/test_benchmark_protocol.py new file mode 100644 index 00000000..18792270 --- /dev/null +++ b/tests/python/test_benchmark_protocol.py @@ -0,0 +1,188 @@ +"""Benchmark protocol checks without external datasets or baseline packages.""" + +import importlib +import json +import sys +from pathlib import Path + +import numpy as np +import pytest + +# Wheel tests may invoke pytest outside the checkout. +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT)) +build_indexes = importlib.import_module("benchmarks.build_indexes") +query_index = importlib.import_module("benchmarks.query_index") +reproduce = importlib.import_module("benchmarks.reproduce") + + +@pytest.fixture +def sweep(tmp_path, monkeypatch): + queries = np.arange(12, dtype="float32").reshape(3, 4) + truth = np.array([[0], [1], [2]], dtype="int32") + record = { + "config": { + "base": build_indexes.DATASETS["gist1m"][0], + "socket": 0, + "method": "rabitq-ivf", + "bits": 5, + "k": 1, + "base_count": 8, + "query_count": 3, + "index_dir": str(tmp_path), + }, + "search": {"cpu_placement": {"allowed_cpus": [0]}}, + } + initial = tmp_path / "initial.json" + initial.write_text(json.dumps(record)) + output = tmp_path / "sweep.json" + monkeypatch.setattr( + query_index.os, "sched_getaffinity", lambda _: {0}, raising=False + ) + monkeypatch.setattr(query_index, "query_set", lambda *_: (queries, truth)) + monkeypatch.setattr(query_index, "load_index", lambda *_: object()) + monkeypatch.setattr( + sys, + "argv", + [ + "query_index.py", + str(initial), + "--dataset", + "gist1m", + "--output", + str(output), + ], + ) + return initial, output + + +def test_recall_uses_timed_single_query_results(sweep, monkeypatch): + _, output = sweep + calls = [] + + def search(index, config, queries, threads): + # Batch and single-query paths deliberately return different neighbors. + calls.append((len(queries), threads)) + ids = ( + (queries[:, :1] / 4).astype("int32") + if len(queries) == 1 + else np.full((len(queries), 1), 7) + ) + return ids, np.zeros_like(ids, dtype="float32") + + monkeypatch.setattr(query_index, "search", search) + query_index.main() + record = json.loads(output.read_text()) + assert set(calls) == {(1, 1)} + assert record["recall_mode"] == "one_query_per_call" + assert len(record["query_points"]) == 10 + assert all(point["recall"] == 1 for point in record["query_points"]) + assert all(point["one_query_service_s"] > 0 for point in record["query_points"]) + + +def test_sweep_rejects_wrong_query_count_before_search(sweep, monkeypatch): + initial, output = sweep + record = json.loads(initial.read_text()) + record["config"]["query_count"] = 4 + initial.write_text(json.dumps(record)) + monkeypatch.setattr( + query_index, "load_index", lambda *_: pytest.fail("loaded index") + ) + with pytest.raises(SystemExit) as error: + query_index.main() + assert error.value.code == 2 + assert not output.exists() + + +@pytest.mark.parametrize( + "key,value", + [("bits", 9), ("base_count", 980000), ("threads", 16), ("metric", "ip")], +) +def test_ann_record_rejects_wrong_settings(key, value): + base, queries, truth, metric = build_indexes.DATASETS["gist1m"] + cfg = dict( + method="rabitq-ivf", + metric=metric, + base=str(ROOT / base), + queries=str(ROOT / queries), + groundtruth=str(ROOT / truth), + base_count=1000000, + query_count=1000, + base_limit=None, + query_limit=None, + threads=48, + socket=0, + k=10, + nlist=4096, + opq=False, + bits=5, + ) + record = {"config": cfg, "data": {"dimension": 960}} + build_indexes.validate_record(record, "gist1m", "rabitq-ivf-1p4") + cfg[key] = value + with pytest.raises(ValueError, match="unexpected ANN configuration"): + build_indexes.validate_record(record, "gist1m", "rabitq-ivf-1p4") + + +@pytest.mark.parametrize("stage", ["all", "build", "query", "export"]) +def test_partial_reproduction_selects_only_requested_work(monkeypatch, capsys, stage): + monkeypatch.setattr( + sys, + "argv", + [ + "reproduce.py", + "--datasets", + "dbpedia", + "--methods", + "faiss-pqfs", + "--stage", + stage, + "--dry-run", + ], + ) + reproduce.main() + commands = capsys.readouterr().out + assert "gist1m" not in commands + assert "run_kmeans.py" not in commands + assert ("download_datasets.py" in commands) == (stage != "export") + assert ("build_indexes.py" in commands) == (stage in ("all", "build")) + assert ("run_sweeps.py" in commands) == (stage in ("all", "query")) + assert ("plot_results.py" in commands) == (stage in ("all", "export")) + + +def test_recall_rejects_missing_query_results(): + from benchmarks.run import recall_at_k + + with pytest.raises(ValueError, match="shapes differ"): + recall_at_k(np.array([[0]]), np.array([[0], [1]])) + + +def test_kmeans_export_rejects_wrong_metric_before_writing(tmp_path): + from benchmarks.run_kmeans import publish + + docs = tmp_path / "docs" + docs.mkdir() + page = docs / "kmeans.md" + page.write_text("original content") + results = tmp_path / "results/gist1m" + results.mkdir(parents=True) + record = { + "config": { + "dataset": "gist1m", + "method": "faiss", + "metric": "ip", + "seed": 42, + "socket": 0, + "k": 4096, + "threads": 48, + "niter": 25, + "nq": 1000, + "n": 1000000, + "d": 960, + }, + "quality": {"nprobe": 40}, + } + (results / "faiss.json").write_text(json.dumps(record)) + with pytest.raises(ValueError, match="unexpected benchmark configuration"): + publish(tmp_path / "results", docs, datasets=["gist1m"], methods=["faiss"]) + assert page.read_text() == "original content"