fix: bound IVF initializers, QG degree sums, HNSW capacity errors and ISA helper linkage

docs/development.md

@@ -602,7 +602,7 @@ preprocessing objects, `visited` for visited state.
 
 **Search windows**
 
-`ef_search` for a per-call graph window, `default_ef_search_` for a stored default, and
+`ef_search` for a per-call graph window, `kDefaultEfSearch` for HNSW's fixed fallback, and
 `ef_construction` for graph-building search breadth. IVF's `nprobe` counts clusters and
 remains distinct.
 
@@ -809,7 +809,7 @@ This mark is unrelated to the `SearchBuffer` ID marker. Keep these rules when to
 
 - Read cluster IDs only through `get_clusterid_by_internalid`, which masks the bit. Estimators,
   routing, and the `add` reconstruction index centroids with it.
-- A removed point stays in the graph: base-layer search still inserts it into `candidate_set`, and
+- A removed point stays in the graph: base-layer search still inserts it into `search_pool`, and
   `add` may link to it. Only the result buffer skips it, including when it is the entry point.
 - `load` masks the bit before range-checking the cluster ID.
 - `HnswRemoveTest.SearchReachesLivePointsThroughRemovedOnes`,

include/rabitqlib/clustering/detail/kmeans.hpp

@@ -244,7 +244,7 @@ class LloydKMeans : public Parameters {
         }
         if (n > static_cast<size_t>(std::numeric_limits<std::ptrdiff_t>::max()) / d) {
             throw std::invalid_argument(
-                std::string(Assignment::kName) + " input dimensions overflow size_t"
+                std::string(Assignment::kName) + " input dimensions overflow ptrdiff_t"
             );
         }
         const uint32_t num_workers = effective_num_threads();

include/rabitqlib/core/detail/rabitq_impl.hpp

@@ -102,7 +102,8 @@ inline T direct_rescale(T l2_sqr, T residual_code_ip, MetricType metric_type) {
 
 /**
  * Inverse of pack_binary_to_bytes<uint64_t>: write each coordinate's sign bit (0 or
- * 1) to signs. Full 64-coordinate words come first, followed by a final 32-bit word.
+ * 1) to signs. Full 64-coordinate words come first, followed by a final partial word
+ * of any multiple of 8 coordinates.
  */
 inline void unpack_sign_bits(const uint8_t* packed, size_t dim, uint16_t* signs) {
     for (size_t start = 0; start < dim; start += 64) {

include/rabitqlib/fastscan/data_layout.hpp

@@ -28,8 +28,8 @@ template <typename T>
 struct BatchDataMap {
    public:
     explicit BatchDataMap(char* data, size_t padded_dim)
-        : batch_bin_code_(reinterpret_cast<uint8_t*>(data))
-        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))  // 1 bit code
+        : batch_bin_code_(reinterpret_cast<uint8_t*>(data))  // 1-bit codes
+        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))
         , f_rescale_(data + detail::BatchCodeOffsets<T>::rescale(padded_dim))
         , f_error_(data + detail::BatchCodeOffsets<T>::error(padded_dim)) {}
 
@@ -54,8 +54,8 @@ template <typename T>
 struct ConstBatchDataMap {
    public:
     explicit ConstBatchDataMap(const char* data, size_t padded_dim)
-        : batch_bin_code_(reinterpret_cast<const uint8_t*>(data))
-        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))  // 1 bit code
+        : batch_bin_code_(reinterpret_cast<const uint8_t*>(data))  // 1-bit codes
+        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))
         , f_rescale_(data + detail::BatchCodeOffsets<T>::rescale(padded_dim))
         , f_error_(data + detail::BatchCodeOffsets<T>::error(padded_dim)) {}
 
@@ -76,8 +76,8 @@ template <typename T>
 struct QGBatchDataMap {
    public:
     explicit QGBatchDataMap(char* data, size_t padded_dim)
-        : batch_bin_code_(reinterpret_cast<uint8_t*>(data))
-        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))  // 1 bit code
+        : batch_bin_code_(reinterpret_cast<uint8_t*>(data))  // 1-bit codes
+        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))
         , f_rescale_(data + detail::BatchCodeOffsets<T>::rescale(padded_dim)) {}
 
     [[nodiscard]] uint8_t* bin_code() { return batch_bin_code_; }
@@ -107,8 +107,8 @@ template <typename T>
 struct ConstQGBatchDataMap {
    public:
     explicit ConstQGBatchDataMap(const char* data, size_t padded_dim)
-        : batch_bin_code_(reinterpret_cast<const uint8_t*>(data))
-        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))  // 1 bit code
+        : batch_bin_code_(reinterpret_cast<const uint8_t*>(data))  // 1-bit codes
+        , f_add_(data + detail::BatchCodeOffsets<T>::add(padded_dim))
         , f_rescale_(data + detail::BatchCodeOffsets<T>::rescale(padded_dim)) {}
 
     [[nodiscard]] const uint8_t* bin_code() const { return batch_bin_code_; }

include/rabitqlib/fastscan/fastscan.hpp

@@ -26,7 +26,7 @@ constexpr static std::array<int, 16> kPos = {
     3 /*1101*/,
     2 /*1110*/,
     3 /*1111*/,
-};  // all possible combination for a 4 bit string
+};  // kPos[j]: query coordinate of j's lowest set bit (bit 3 is q0, bit 0 is q3)
 
 // Accumulate one batch into int32_t results. Throws if a result exceeds int32_t.
 // dim must be a positive multiple of 16.

include/rabitqlib/index/hnsw/hnsw.hpp

@@ -169,12 +169,12 @@ class HierarchicalNSW {
     // Borrows construct() input only while its workers run; cleared on success or failure.
     const float* construction_data_{nullptr};
 
-    // Bounded priority queue implemented as a sorted vector.
     struct Candidate {
         float est_dist;
         PID id;
     };
 
+    // Bounded priority queue implemented as a sorted vector.
     class BoundedKNN {
        public:
         explicit BoundedKNN(size_t capacity) : capacity_(capacity) {}
@@ -268,7 +268,8 @@ class HierarchicalNSW {
 
     // Layout: (# of edges + edges) + (cluster_id) + (External_id) + (BinData) + (ExData)
     AlignedStorage data_level0_memory_;
-    // Reserve capacity without initializing unused slots; size tracks live nodes.
+    // Reserve capacity without initializing unused slots; size tracks stored nodes,
+    // including removed ones.
     std::vector<LinkStorage> link_lists_;
     std::vector<int> element_levels_;
 
@@ -447,8 +448,7 @@ class HierarchicalNSW {
         BoundedKNN& nearest_candidates
     ) const;
 
-    // Construction
-    // Currently only support index construction with non-quantized vectors
+    // Raw-vector distance used by construct; add scores links with get_quant_dist.
     float get_data_dist(PID obj1, PID obj2) const {
         PID label1 = get_external_label(obj1);
         PID label2 = get_external_label(obj2);

include/rabitqlib/index/ivf/initializer.hpp

@@ -52,6 +52,8 @@ class Initializer {
     virtual ~Initializer() = 0;
     [[nodiscard]] virtual const float* centroid(PID) const = 0;
     virtual void add_vectors(const float*, size_t) = 0;
+    // Resize the output to min(nprobe, number of clusters) and fill it with the
+    // nearest centroids in no particular order; the caller need not presize it.
     virtual void centroids_distances(
         const float*, size_t, std::vector<AnnCandidate<float>>&
     ) const = 0;
@@ -96,20 +98,18 @@ class FlatInitializer : public Initializer {
                     ? dot_product_dis(query, centroid(i), dim_)
                     : std::sqrt(euclidean_sqr(query, centroid(i), dim_));
         }
+        const size_t num_probes = std::min(nprobe, num_cluster_);
         std::partial_sort(
             centroid_candidates.begin(),
-            centroid_candidates.begin() + static_cast<long>(nprobe),
+            centroid_candidates.begin() + static_cast<std::ptrdiff_t>(num_probes),
             centroid_candidates.end()
         );
 
-        std::memcpy(
-            candidates.data(),
-            centroid_candidates.data(),
-            sizeof(AnnCandidate<float>) * nprobe
-        );
+        candidates.resize(num_probes);
+        std::copy_n(centroid_candidates.begin(), num_probes, candidates.begin());
     }
 
-    // for flat initer, we save & load into the ifstream
+    // The flat initializer stores its centroids inline in the index file stream.
     void save(std::ofstream& output, const char*) const override {
         output.write(
             reinterpret_cast<const char*>(centroids_.data()),
@@ -258,13 +258,20 @@ class HNSWInitializer : public Initializer {
     void centroids_distances(
         const float* query, size_t nprobe, std::vector<AnnCandidate<float>>& candidates
     ) const override {
+        const size_t num_probes = std::min(nprobe, num_cluster_);
+        candidates.resize(num_probes);
+        if (num_probes == 0) {
+            return;
+        }
         std::priority_queue<std::pair<float, hnswlib::labeltype>> result =
-            alg_hnsw_->searchKnnWithEf(query, nprobe, std::max(size_t{768}, 2 * nprobe));
-        if (result.size() != nprobe) {
+            alg_hnsw_->searchKnnWithEf(
+                query, num_probes, std::max(size_t{768}, 2 * num_probes)
+            );
+        if (result.size() != num_probes) {
             throw std::runtime_error("HNSW centroid graph returned too few candidates");
         }
 
-        for (size_t i = 0; i < nprobe; ++i) {
+        for (size_t i = 0; i < num_probes; ++i) {
             candidates[i].distance = metric_type_ == METRIC_IP
                                          ? result.top().first
                                          : std::sqrt(result.top().first);
@@ -273,7 +280,7 @@ class HNSWInitializer : public Initializer {
         }
     }
 
-    // for hnsw initer, we save & load into a separate file by hnswlib
+    // The HNSW initializer stores its graph in a separate hnswlib sidecar file.
     void save(std::ofstream&, const char* filename) const override {
         std::string hnsw(filename);
         hnsw += ".hnsw";

include/rabitqlib/index/ivf/ivf.hpp

@@ -83,7 +83,7 @@ class IVF {
     size_t num_clusters_ = 0;
     bool raw_reranking_ = false;  // raw vectors replace extra-bit codes
     bool ready_ = false;          // construction or loading completed
-    size_t ex_bits_ = 0;          // total bits = ex_bits_ + 1
+    size_t ex_bits_ = 0;          // total bits = ex_bits_ + 1; 0 with raw reranking
     RotatorType rotator_type_ = RotatorType::FhtKacRotator;
     std::unique_ptr<Rotator<float>> rotator_;
     std::vector<Cluster> clusters_;

include/rabitqlib/utils/bitops.hpp

@@ -10,8 +10,11 @@
 
 namespace rabitqlib::bitops {
 
+// ISA-specific translation units include these helpers; internal linkage keeps each
+// backend's compiled body from being merged with another backend's.
+
 // Load 1..8 bytes of a compact word, zero-extending a final partial word.
-inline uint64_t load_word(const void* data, size_t bytes) {
+static inline uint64_t load_word(const void* data, size_t bytes) {
     if (bytes == 8) {
         uint64_t word;
         std::memcpy(&word, data, sizeof(word));
@@ -27,7 +30,7 @@ inline uint64_t load_word(const void* data, size_t bytes) {
     return word;
 }
 
-inline unsigned popcount32(uint32_t value) {
+static inline unsigned popcount32(uint32_t value) {
 #if defined(_MSC_VER)
     value -= (value >> 1) & 0x55555555U;
     value = (value & 0x33333333U) + ((value >> 2) & 0x33333333U);
@@ -53,7 +56,7 @@ static inline unsigned popcount64(uint64_t value) {
 }
 
 // Callers pass nonzero masks.
-inline unsigned countr_zero32(uint32_t value) {
+static inline unsigned countr_zero32(uint32_t value) {
 #if defined(_MSC_VER)
     unsigned long index = 0;
     _BitScanForward(&index, value);

python_bindings/__init__.pyi

@@ -371,7 +371,7 @@ class HnswIndex:
     dim : int
         Original vector dimension, from 64 through 65536.
     max_elements : int
-        Storage capacity, including removed points. Use resize to grow it.
+        Storage capacity, at least 1, including removed points. Use resize to grow it.
     M : int, optional
         Graph connectivity, at least 2. Default is 16.
     ef_construction : int, optional
@@ -407,7 +407,7 @@ class HnswIndex:
         dim : int
             Original vector dimension, from 64 through 65536.
         max_elements : int
-            Storage capacity, including removed points. Use resize to grow it.
+            Storage capacity, at least 1, including removed points. Use resize to grow it.
         M : int, optional
             Graph connectivity, at least 2. Default is 16.
         ef_construction : int, optional

python_bindings/hnsw_bindings.cpp

@@ -293,7 +293,7 @@ Parameters
 dim : int
     Original vector dimension, from 64 through 65536.
 max_elements : int
-    Storage capacity, including removed points. Use resize to grow it.
+    Storage capacity, at least 1, including removed points. Use resize to grow it.
 M : int, optional
     Graph connectivity, at least 2. Default is 16.
 ef_construction : int, optional

src/fastscan/fastscan_generic.cpp

@@ -60,7 +60,7 @@ void pack_lut_range_generic(
 
 namespace rabitqlib::fastscan::simd {
 void transfer_lut_hacc_generic(const uint16_t* lut, size_t dim, uint8_t* hc_lut) {
-    // Match the AVX2 layout: two low-byte tables, then two high-byte tables.
+    // HaccLutLayout::Pair32: two low-byte tables, then two high-byte tables.
     for (size_t group = 0; group < dim / 4; ++group) {
         const size_t offset = group / 2 * 64 + group % 2 * 16;
         for (size_t i = 0; i < 16; ++i) {

src/index/hnsw.cpp

@@ -79,6 +79,9 @@ HierarchicalNSW::HierarchicalNSW(
     if (M < 2) {
         throw std::invalid_argument("HNSW M must be at least 2");
     }
+    if (max_elements == 0) {
+        throw std::invalid_argument("HNSW max_elements must be greater than zero");
+    }
     max_elements_ = max_elements;
     dim_ = dim;
     rotator_.reset(
@@ -117,9 +120,6 @@ HierarchicalNSW::HierarchicalNSW(
     data_level0_memory_.reset(
         memory::huge_page_allocate<char>(max_elements_ * level0_record_bytes_)
     );
-    if (data_level0_memory_ == nullptr) {
-        throw std::runtime_error("Not enough memory");
-    }
 
     level_generator_.seed(random_seed);
 
@@ -346,15 +346,9 @@ void HierarchicalNSW::load(const char* filename) {
 
     if (centroids_bytes != 0) {
         loaded.centroids_memory_.reset(memory::huge_page_allocate<char>(centroids_bytes));
-        if (loaded.centroids_memory_ == nullptr) {
-            throw std::bad_alloc();
-        }
         read_payload(loaded.centroids_memory_.get(), centroids_bytes);
     }
     loaded.data_level0_memory_.reset(memory::huge_page_allocate<char>(data_capacity_bytes));
-    if (loaded.data_level0_memory_ == nullptr) {
-        throw std::bad_alloc();
-    }
     read_payload(loaded.data_level0_memory_.get(), data_bytes);
 
     loaded.element_levels_ = std::vector<int>(loaded.max_elements_);
@@ -543,11 +537,6 @@ void HierarchicalNSW::construct(
         num_clusters_ = num_clusters;
         const size_t centroids_bytes = num_clusters_ * padded_dim_ * sizeof(float);
         centroids_memory_.reset(memory::huge_page_allocate<char>(centroids_bytes));
-        if (centroids_memory_ == nullptr) {
-            throw std::runtime_error(
-                "Not enough memory: HNSW failed to allocate centroids"
-            );
-        }
 
         for (size_t i = 0; i < num_clusters; ++i) {
             this->rotator_->rotate(
@@ -734,9 +723,7 @@ void HierarchicalNSW::add_point(
             static_cast<char*>(std::malloc((upper_layer_links_bytes_ * node_level) + 1))
         );
         if (link_lists_[new_node_id] == nullptr) {
-            throw std::runtime_error(
-                "Not enough memory: add_point failed to allocate linklist"
-            );
+            throw std::bad_alloc();
         }
         memset(
             link_lists_[new_node_id].get(), 0, (upper_layer_links_bytes_ * node_level) + 1
@@ -767,7 +754,7 @@ void HierarchicalNSW::add_point(
         }
     }
 
-    // Releasing lock for the maximum level
+    // A taller node becomes the entry point; entry_point_lock is still held then.
     if (node_level > previous_max_level) {
         entry_point_id_ = new_node_id;
         max_level_ = node_level;
@@ -1170,6 +1157,9 @@ void HierarchicalNSW::resize(size_t new_max_elements) {
     if (level0_record_bytes_ == 0) {
         throw std::logic_error("HNSW index must be initialized before resize");
     }
+    if (new_max_elements == 0) {
+        throw std::invalid_argument("HNSW resize capacity must be greater than zero");
+    }
     if (new_max_elements < num_stored_points_) {
         throw std::invalid_argument("HNSW resize: capacity below the element count");
     }
@@ -1188,9 +1178,6 @@ void HierarchicalNSW::resize(size_t new_max_elements) {
     // comes from aligned_alloc, which realloc does not keep aligned.
     const size_t level0_bytes = new_max_elements * level0_record_bytes_;
     AlignedStorage level0(memory::huge_page_allocate<char>(level0_bytes));
-    if (level0 == nullptr) {
-        throw std::bad_alloc();
-    }
     std::memset(level0.get(), 0, level0_bytes);
     std::memcpy(
         level0.get(), data_level0_memory_.get(), num_stored_points_ * level0_record_bytes_
@@ -1294,9 +1281,7 @@ std::vector<PID> HierarchicalNSW::add(
                 std::calloc(1, (upper_layer_links_bytes_ * levels[i]) + 1)
             );
             if (raw == nullptr) {
-                throw std::runtime_error(
-                    "Not enough memory: HNSW add failed to allocate a link list"
-                );
+                throw std::bad_alloc();
             }
         }
         link_lists.emplace_back(raw);

src/index/hnsw_dispatch.cpp

@@ -13,12 +13,12 @@ namespace rabitqlib::hnsw::detail {
 namespace {
 using SearchKnnFn =
     std::vector<std::pair<float, PID>> (*)(HierarchicalNSW&, const float*, size_t, size_t);
-// The core variant uses AVX2 warmup; the popcount variant has its own stricter tier.
 }  // namespace
 std::vector<std::pair<float, PID>> search_knn_sorted(
     HierarchicalNSW& index, const float* query, size_t k, size_t ef_search
 ) {
 #if defined(__x86_64__) || defined(_M_X64)
+    // The core variant uses AVX2 warmup; the popcount variant has its own stricter tier.
     static const SearchKnnFn kSearchKnnFn =
         cpu::has_avx512_popcnt() ? search_knn_sorted_avx512_popcnt
                                  : RABITQ_RESOLVE(

src/index/hnsw_search_impl.hpp

@@ -203,7 +203,7 @@ inline void HierarchicalNSW::search_base_layer_direct(
 ) const {
     auto visited = visited_list_pool_->get_free_vislist();
 
-    // Use our bounded priority queue instead of the maxheap.
+    // Keep the ef_search best estimates in a bounded sorted buffer.
     buffer::SearchBuffer<float> search_pool(ef_search);
 
     float dist_k = 1e10;