diff --git a/CMakeLists.txt b/CMakeLists.txt index ed68dfcb..ac9eac53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -project(RaBitQLib VERSION 0.7.2 LANGUAGES CXX) +project(RaBitQLib VERSION 0.8.0 LANGUAGES CXX) include(CMakePackageConfigHelpers) include(GNUInstallDirs) diff --git a/README.md b/README.md index ebb47d6d..d247f358 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ approximate nearest-neighbor search. See the [research papers](https://vectordb- ## News +- **v0.8.0:** Faiss-style `train`, `add` and `search` for all Python indexes, + with 0.7 calls kept as deprecated forms until 0.9; optional k-means sampling + and assignment; stricter input validation and atomic index saves. C++ query-factor + accessors gain a `g_` prefix. See the [release notes](https://vectordb-ntu.github.io/RaBitQ-Library/releases/0.8.0/). - **v0.7.2:** Python type hints, expanded API documentation, path-like inputs, clearer input errors, and concise index representations. Internal quantization refactoring and streamlined regression checks preserve existing APIs and index formats. @@ -48,7 +52,7 @@ settings, and limitations. ### Python ```bash -python -m pip install --upgrade "rabitqlib>=0.7.2" +python -m pip install --upgrade "rabitqlib>=0.8.0" ``` Wheels support CPython 3.11–3.14 on Linux x86-64/ARM64, Windows x86-64, diff --git a/benchmarks/run.py b/benchmarks/run.py index 9a8e2034..49a2c949 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -135,25 +135,24 @@ def build_index(cfg, base, path): training_s = time.perf_counter() - stage_start stage_start = time.perf_counter() if method == "rabitq-ivf": - index = IvfIndex(dim, n, cluster_count, cfg["bits"], cfg["metric"]) + index = IvfIndex( + dim, cluster_count, nbits=cfg["bits"], metric=cfg["metric"] + ) else: from rabitqlib import HnswIndex index = HnswIndex( dim, - n, - cfg["m"], - cfg["ef_construction"], - cfg["bits"], - cfg["metric"], - random_seed=42, + nbits=cfg["bits"], + num_clusters=cluster_count, + M=cfg["m"], + ef_construction=cfg["ef_construction"], + metric=cfg["metric"], + seed=42, ) - index.build( - base, - kmeans.centroids, - kmeans.assignments, - num_threads=cfg["threads"], - ) + index.resize(n) + index.set_centroids(kmeans.centroids) + index.add(base, cluster_ids=kmeans.assignments, num_threads=cfg["threads"]) indexing_s = time.perf_counter() - stage_start details = { "clustering": "RaBitQKMeans", @@ -164,14 +163,17 @@ def build_index(cfg, base, path): elif method == "rabitq-symqg": from rabitqlib import SymqgIndex - index = SymqgIndex(dim, cfg["degree"], cfg["metric"], cfg["quantization_bits"]) - stage_start = time.perf_counter() - index.build( - base, + # 0.7 quantization_bits=0 (raw vectors) is nbits=32 in 0.8. + index = SymqgIndex( + dim, + max_degree=cfg["degree"], + metric=cfg["metric"], + nbits=cfg["quantization_bits"] or 32, ef_construction=cfg["ef_construction"], - num_threads=cfg["threads"], init="pipnn", ) + stage_start = time.perf_counter() + index.add(base, num_threads=cfg["threads"]) indexing_s = time.perf_counter() - stage_start details = {"initialization": "pipnn", "refinement_iterations": 1} else: diff --git a/docs/docs/cpp_integration.md b/docs/docs/cpp_integration.md index 03d70b5a..b99764e9 100644 --- a/docs/docs/cpp_integration.md +++ b/docs/docs/cpp_integration.md @@ -58,12 +58,12 @@ The core example is built as the `core_example` CMake target. For the GIST bench The [core quantization v1 API](rabitq/core_api.md) has a C++ source-compatibility contract. Other C++ APIs may evolve, and no cross-version C++ binary ABI is promised. -Pin the `v0.7.2` release and include RaBitQ-Library as a Git submodule: +Pin the `v0.8.0` release and include RaBitQ-Library as a Git submodule: ```bash git submodule add https://github.com/VectorDB-NTU/RaBitQ-Library.git third_party/rabitqlib git submodule update --init --recursive -git -C third_party/rabitqlib checkout v0.7.2 +git -C third_party/rabitqlib checkout v0.8.0 git add third_party/rabitqlib ``` diff --git a/docs/docs/index.md b/docs/docs/index.md index 194eeb6a..4191f622 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -26,7 +26,7 @@ estimation into your own C++ search engine.

```bash -python -m pip install "rabitqlib>=0.7.2" +python -m pip install "rabitqlib>=0.8.0" ``` [Installation and supported platforms](installation.md) · @@ -112,5 +112,5 @@ on your own dataset when choosing an index. ## Explore further - [Algorithm overview](rabitq/rabitq.md) for the rotation, encoding, and scoring workflow. -- [RaBitQ Library 0.7.2 release notes](releases/0.7.2.md) for the latest changes and upgrade guidance. +- [RaBitQ Library 0.8.0 release notes](releases/0.8.0.md) for the latest changes and upgrade guidance. - [Papers and citations](papers.md) for the original RaBitQ method, its multi-bit extension, and SymphonyQG. diff --git a/docs/docs/installation.md b/docs/docs/installation.md index 61c81bed..16d41570 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -1,6 +1,6 @@ # Installation -Platform requirements and source installation for RaBitQ Library **0.7.2**. +Platform requirements and source installation for RaBitQ Library **0.8.0**. For the first Python example, see [Quick Start](quick_start.md). ## Requirements @@ -40,7 +40,7 @@ build can use scalar kernels without the FastScan CPU baseline. ## Python wheels ```bash -python -m pip install "rabitqlib>=0.7.2" +python -m pip install "rabitqlib>=0.8.0" ``` For changes from 0.6.0, see the [0.7.0 upgrade notes](releases/0.7.0.md). @@ -53,7 +53,7 @@ Release wheels disable native CPU tuning and select supported kernels at runtime ### Python editor support -Version 0.7.2 includes type declarations (`__init__.pyi`) and a `py.typed` marker +Packages include type declarations (`__init__.pyi`, since 0.7.2) and a `py.typed` marker for editor completion and static type checking. Select the Python environment where `rabitqlib` is installed. Index and clustering docstrings describe parameters, defaults and returned arrays; use `help(IvfIndex)` or `help(RaBitQKMeans)` after @@ -61,11 +61,11 @@ importing those classes. Array shapes and parameter ranges are still checked at ## Get the source -For Python source builds or C++ use, obtain the 0.7.2 checkout once. With Git +For Python source builds or C++ use, obtain the 0.8.0 checkout once. With Git installed, run: ```bash -git clone --branch v0.7.2 --depth 1 https://github.com/VectorDB-NTU/RaBitQ-Library.git +git clone --branch v0.8.0 --depth 1 https://github.com/VectorDB-NTU/RaBitQ-Library.git cd RaBitQ-Library ``` diff --git a/docs/docs/quick_start.md b/docs/docs/quick_start.md index 939a4a61..bbaad793 100644 --- a/docs/docs/quick_start.md +++ b/docs/docs/quick_start.md @@ -1,14 +1,14 @@ # Quick start Build, search, save and reload a small IVF-RaBitQ index with Python. This guide describes -**0.7.2** and uses synthetic data, so no dataset download is required. Coming from 0.7? +**0.8.0** and uses synthetic data, so no dataset download is required. Coming from 0.7? See the [migration guide](python_migration.md). For C++, go directly to [C++ integration](cpp_integration.md). ## Install ```bash -python -m pip install "rabitqlib>=0.7.2" +python -m pip install "rabitqlib>=0.8.0" ``` Wheels support CPython 3.11–3.14 on Linux x86-64/ARM64, Windows x86-64 and diff --git a/docs/docs/releases/0.8.0.md b/docs/docs/releases/0.8.0.md new file mode 100644 index 00000000..1d72f7e7 --- /dev/null +++ b/docs/docs/releases/0.8.0.md @@ -0,0 +1,126 @@ +# RaBitQ Library 0.8.0 + +This release gives the Python indexes a Faiss-style `train` → `add` → `search` +workflow, adds optional k-means sampling, tightens input and state validation, +and renames three C++ query-factor accessors. + +## Python changes + +- `IvfIndex`, `HnswIndex` and `SymqgIndex` follow `train` → `add` → `search`. + IVF-RaBitQ and HNSW-RaBitQ learn their own centroids in `train`, configured by + `index.cp`, or accept external centroids through `set_centroids`. Storage grows + on `add`, so constructors no longer take `max_elements`. SymphonyQG builds its + graph in `add`, takes `nbits` (4, 8, or 32 for raw vectors) instead of + `quantization_bits`, and its `search` replaces `search_batch`. +- 0.7 constructors, `build`, `is_built`, positional `add` arguments and the + SymphonyQG `build`, `search_batch` and `quantization_bits` still work, but raise + a `DeprecationWarning` and will be removed in **0.9**. The + [migration guide](../python_migration.md) maps each old call to its 0.8 form. +- Behavior changes that apply without a warning, detailed in the migration + guide: + - `train` and `add` default to every available CPU, including 0.7-style + `add(x)` calls; `search` and the deprecated `build` keep one thread. + - Multi-threaded HNSW-RaBitQ `add` is not reproducible; pass + `num_threads=1` for the same graph on every run. + - HNSW-RaBitQ capacity grows on `add`, and `search` limits `k` to the + stored point count. + - SymphonyQG `search` accepts an omitted, zero or small `ef`, as + HNSW-RaBitQ does, and one seed now drives both rotation and graph + construction. +- `RaBitQKMeans` and `QGKMeans` accept `max_points_per_centroid` to train on a + seeded sample, and gain `assign(x)` for new vectors. Sampling is off by + default for these classes; index `train` samples at most 256 vectors per + centroid. See [training-set sampling](../clustering.md#training-set-sampling). +- Indexes reject NaN, infinite and oversized coordinates in data, centroids and + queries with `ValueError`. See [input requirements](../index/usage.md#input-requirements). +- `HnswIndex.search` clamps `ef` to the stored point count instead of failing on + a very large value, with unchanged results. + +## C++ changes + +- **Breaking:** query-factor accessors take a `g_` prefix, without aliases: + + | Old | New | Classes | + | --- | --- | --- | + | `k1xsumq()` | `g_k1xsumq()` | `SplitSingleQuery`, `BatchQuery`, `SplitBatchQuery`, `symqg::QuantizedQuery` | + | `kbxsumq()` | `g_kbxsumq()` | `SplitSingleQuery`, `SplitBatchQuery` | + | `kbase_sumq()` | `g_kbase_sumq()` | `SplitSingleQuery` | + + The optional `k1xsumq` pointer parameter of `BatchQuery` and the SymphonyQG + search entry points is now named `g_k1xsumq`; its position and type are + unchanged. +- **Breaking:** saving an index that stores no points throws `std::logic_error`. + Earlier releases wrote such an HNSW index as an empty file; loading old files + is unchanged. +- **Breaking:** the installed header `rabitqlib/utils/array.hpp` and its + `rabitqlib::Array` class are removed; use standard containers instead. +- IVF indexes can be created without a point count, trained or given centroids, + then filled with `add`. Pass the rotation seed to this constructor: four + arguments such as `IVF(dim, num_clusters, 4, METRIC_IP)` select the + point-count constructor instead. HNSW indexes keep their capacity argument and + can be trained or given centroids before the first `add`. Existing + constructors and `construct` are unchanged. See + [IVF-RaBitQ](../index/ivf.md#training-centroids-in-c) and + [HNSW-RaBitQ](../index/hnsw.md#training-centroids-in-c). +- On MSVC, libc++ and libstdc++ before GCC 11, a seed now gives a different + FHT/Kac rotation, and therefore different `RaBitQKMeans` results, because the + signs no longer depend on the standard library. `faster_config` reads a frozen + scale table, so its `t_const` can shift by a few percent on those toolchains. + GCC 11+ Linux results are unchanged. A seeded `MatrixRotator` is now + reproducible on the same toolchain. Saved indexes store their rotation and are + unaffected. +- Index entry points reject non-finite and oversized coordinates with + `std::invalid_argument`. HNSW `cluster_id_of` validates its argument, IVF loading + rejects trailing bytes, SymphonyQG rejects `ef_construction` of zero, and the + advanced encoders check their preconditions. + +## Fixes + +- Code compiled for AVX2 or AVX-512 no longer reaches generic callers through + shared inline functions and templates, which could fail on CPUs without those + instructions. CI now checks ISA-specific builds for such functions. +- All three indexes save a regular file by writing a temporary file beside it and + renaming it into place, so a failed save leaves the existing file unchanged. + The replaced file keeps its permission bits, but not its owner or other hard + links. Symbolic links and other non-regular destinations are still written in + place. +- HNSW-RaBitQ rejects a zero capacity with `ValueError` (C++: + `std::invalid_argument`) and reports allocation failure as `MemoryError` + (C++: `std::bad_alloc`) in construction, `add`, `resize` and `load` alike. +- IVF initializers clamp `nprobe` to the cluster count and size their output, + instead of reading and writing out of range when called directly. +- AVX2 and AVX-512 FHT rotation accepts overlapping input, as the other + backends do. +- A failed copy assignment leaves the target object in its previous state, and + HNSW search no longer prefetches addresses outside its allocation. + +## Internal changes + +Index, SIMD and binding internals share more helpers, and identifier naming is +enforced by clang-tidy. Several pull requests optimize query preparation, LUT +quantization and graph search; their measurements are recorded in those pull +requests and do not establish a general result. No performance measurement was +made for k-means sampling or for the 2,000-cluster threshold at which index +`train` switches to `QGKMeans`. The published [benchmarks](../benchmarks/bench.md) +retain their recorded 0.7.1 provenance. + +The package declares its license as an SPDX expression with expanded PyPI +metadata, and the repository adds a security policy and a software citation. + +## Upgrading + +Python code written for 0.7 runs unchanged apart from the behavior changes +above; follow the deprecation warnings and the [migration guide](../python_migration.md) +before 0.9. Index files from 0.7 load as trained indexes, and no conversion is +required. + +C++ consumers must rename the three query-factor accessors, replace +`rabitqlib::Array`, and must not save empty indexes. For example, zvec, which +uses the core and FastScan components, built and passed its RaBitQ tests after +renaming one `kbxsumq()` call in a test. The stable core API v1 and component +targets are unchanged. +Rebuild C++ consumers with matching headers and libraries; no cross-version +C++ binary ABI is promised. + +Use the [installation guide](../installation.md) for supported platforms and +the [quick start](../quick_start.md) for a complete Python example. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index abb11e5b..753e30b2 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -13,6 +13,7 @@ nav: - Installation: installation.md - C++ Integration: cpp_integration.md - Release notes: + - "0.8.0": releases/0.8.0.md - "0.7.2": releases/0.7.2.md - "0.7.1": releases/0.7.1.md - "0.7.0": releases/0.7.0.md diff --git a/pyproject.toml b/pyproject.toml index 2c0f9892..80094eff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build" [project] name = "rabitqlib" -version = "0.7.2" +version = "0.8.0" description = "Official RaBitQ library for vector quantization and approximate nearest-neighbor search" readme = "README.md" requires-python = ">=3.11" diff --git a/sample/python/README.md b/sample/python/README.md index a7a523f4..532caaf0 100644 --- a/sample/python/README.md +++ b/sample/python/README.md @@ -1,7 +1,7 @@ # Python examples Run these commands from the repository root in your project Python environment -with `rabitqlib>=0.7.0` and NumPy installed. When working from source, +with `rabitqlib>=0.8.0` and NumPy installed. When working from source, [install the library from the same checkout](../../.github/CONTRIBUTING.md#python-changes) so the examples and native extension use matching APIs. Clustering and indexing both use RaBitQ Library; FAISS is needed only for `compare_with_faiss.py`.