diff --git a/include/rabitqlib/clustering/detail/kmeans.hpp b/include/rabitqlib/clustering/detail/kmeans.hpp index 77ff701f..714b2035 100644 --- a/include/rabitqlib/clustering/detail/kmeans.hpp +++ b/include/rabitqlib/clustering/detail/kmeans.hpp @@ -55,6 +55,25 @@ namespace detail { return change <= early_stop_threshold; } +// Float vectors accumulated in double, in index order. +[[nodiscard]] inline double squared_norm_f64(const float* vector, size_t d) { + double result = 0.0; + for (size_t dim = 0; dim < d; ++dim) { + const double value = vector[dim]; + result += value * value; + } + return result; +} + +[[nodiscard]] inline double squared_l2_f64(const float* a, const float* b, size_t d) { + double result = 0.0; + for (size_t dim = 0; dim < d; ++dim) { + const double difference = static_cast(a[dim]) - b[dim]; + result += difference * difference; + } + return result; +} + // Inputs have already passed clustering validation. GEMM only screens candidates; // final distances and tie breaking retain the scalar double-precision arithmetic. // Optional previous assignments contain n valid centroid IDs and may alias the @@ -96,10 +115,7 @@ inline void exact_assign( } distance += 1.0; } else { - for (size_t dim = 0; dim < d; ++dim) { - const double difference = static_cast(vector[dim]) - centroid[dim]; - distance += difference * difference; - } + distance = squared_l2_f64(vector, centroid, d); } return distance; }; @@ -126,12 +142,8 @@ inline void exact_assign( #pragma omp for schedule(static) for (std::ptrdiff_t cluster = 0; cluster < static_cast(k); ++cluster) { - double norm = 0.0; - for (size_t dim = 0; dim < d; ++dim) { - const double value = centroids[static_cast(cluster) * d + dim]; - norm += value * value; - } - centroid_norms[static_cast(cluster)] = norm; + centroid_norms[static_cast(cluster)] = + squared_norm_f64(centroids + static_cast(cluster) * d, d); } #pragma omp for schedule(static) for (std::ptrdiff_t block = 0; block < static_cast(blocks); @@ -144,12 +156,7 @@ inline void exact_assign( const size_t rows = std::min(point_block, n - first_point); for (size_t row = 0; row < rows; ++row) { const float* vector = x + (first_point + row) * d; - double norm = 0.0; - for (size_t dim = 0; dim < d; ++dim) { - const double value = vector[dim]; - norm += value * value; - } - point_norms[row] = norm; + point_norms[row] = squared_norm_f64(vector, d); best_distances[row] = std::numeric_limits::infinity(); if (previous_assignments != nullptr) { const size_t point = first_point + row; @@ -484,14 +491,9 @@ class LloydKMeans : public Parameters { // Means, normalization, and displacement share one worker team. Preserve // the per-centroid arithmetic; empty rows are measured after refilling. const auto cluster_shift = [&](size_t cluster) { - double result = 0; - for (size_t dim = 0; dim < d; ++dim) { - const size_t i = cluster * d + dim; - const double difference = - static_cast(centroids[i]) - previous_centroids[i]; - result += difference * difference; - } - return result; + return squared_l2_f64( + centroids.data() + cluster * d, previous_centroids.data() + cluster * d, d + ); }; double shift = 0; #pragma omp parallel for num_threads(num_workers) reduction(+ : shift) @@ -604,10 +606,7 @@ class LloydKMeans : public Parameters { } void normalize_centroid(float* centroid) const { - double squared_norm = 0; - for (size_t dim = 0; dim < d; ++dim) { - squared_norm += static_cast(centroid[dim]) * centroid[dim]; - } + const double squared_norm = squared_norm_f64(centroid, d); if (squared_norm == 0) { return; } diff --git a/include/rabitqlib/index/hnsw/hnsw.hpp b/include/rabitqlib/index/hnsw/hnsw.hpp index e2a8b8c5..a520c3b2 100644 --- a/include/rabitqlib/index/hnsw/hnsw.hpp +++ b/include/rabitqlib/index/hnsw/hnsw.hpp @@ -2,6 +2,7 @@ #pragma once #include +#include #include #include #include @@ -228,6 +229,8 @@ class HierarchicalNSW { // `remove` sets the high bit of a point's stored cluster ID. construct and load // keep num_clusters_ at or below 2^31, so a real cluster ID never has it. static constexpr PID kRemovedMask = 0x80000000U; + // Used when a search passes ef_search == 0. + static constexpr size_t kDefaultEfSearch = 10; size_t max_elements_{0}; mutable std::atomic num_stored_points_{0}; size_t num_removed_{0}; // Recovered from stored cluster-ID markers when loading. @@ -237,7 +240,6 @@ class HierarchicalNSW { size_t upper_layer_max_degree_{0}; size_t base_layer_max_degree_{0}; size_t ef_construction_{0}; - size_t default_ef_search_{0}; MetricType metric_type_{METRIC_L2}; double mult_{0.0}; @@ -314,12 +316,6 @@ class HierarchicalNSW { ); } - PID* get_external_label_pt(PID internal_id) { - return reinterpret_cast( - data_level0_memory_.get() + (internal_id * level0_record_bytes_) + label_offset_ - ); - } - char* get_bindata_by_internalid(PID internal_id) { return data_level0_memory_.get() + (internal_id * level0_record_bytes_) + bin_data_offset_; @@ -356,13 +352,32 @@ class HierarchicalNSW { return (get_raw_clusterid(internal_id) & kRemovedMask) != 0; } - char* get_clusterid_pt(PID internal_id) { - return reinterpret_cast( + // Store a cluster ID, including any removal mark. + void set_raw_clusterid(PID internal_id, PID cluster_id) { + memcpy( data_level0_memory_.get() + (internal_id * level0_record_bytes_) + - level0_links_bytes_ + level0_links_bytes_, + &cluster_id, + sizeof(PID) ); } + // Derive record sizes and offsets from padded_dim_, ex_bits_ and the layer degrees. + void set_record_layout(); + + // The fields set_record_layout derives, in a fixed order for comparison. + [[nodiscard]] std::array record_layout() const { + return { + bin_data_bytes_, + ex_data_bytes_, + level0_links_bytes_, + label_offset_, + bin_data_offset_, + ex_data_offset_, + level0_record_bytes_, + upper_layer_links_bytes_}; + } + int get_random_level(double reverse_size); // Write the index file at exactly this path. diff --git a/include/rabitqlib/utils/memory.hpp b/include/rabitqlib/utils/memory.hpp index 70047f1a..238af43e 100644 --- a/include/rabitqlib/utils/memory.hpp +++ b/include/rabitqlib/utils/memory.hpp @@ -175,90 +175,92 @@ static inline void prefetch_l2(const void* addr) { #endif } -inline void mem_prefetch_l1(const char* ptr, size_t num_lines) { +// Prefetch num_lines 64-byte lines (at most 20) starting at ptr. +template +inline void mem_prefetch_lines(const char* ptr, size_t num_lines) { // The repeated fallthrough branches intentionally unroll up to 20 prefetches. // NOLINTBEGIN(bugprone-branch-clone) switch (num_lines) { default: [[fallthrough]]; case 20: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 19: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 18: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 17: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 16: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 15: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 14: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 13: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 12: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 11: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 10: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 9: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 8: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 7: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 6: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 5: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 4: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 3: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 2: - prefetch_l1(ptr); + Prefetch(ptr); ptr += 64; [[fallthrough]]; case 1: - prefetch_l1(ptr); + Prefetch(ptr); [[fallthrough]]; case 0: break; @@ -266,94 +268,11 @@ inline void mem_prefetch_l1(const char* ptr, size_t num_lines) { // NOLINTEND(bugprone-branch-clone) } +inline void mem_prefetch_l1(const char* ptr, size_t num_lines) { + mem_prefetch_lines(ptr, num_lines); +} + inline void mem_prefetch_l2(const char* ptr, size_t num_lines) { - // The repeated fallthrough branches intentionally unroll up to 20 prefetches. - // NOLINTBEGIN(bugprone-branch-clone) - switch (num_lines) { - default: - [[fallthrough]]; - case 20: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 19: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 18: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 17: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 16: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 15: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 14: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 13: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 12: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 11: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 10: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 9: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 8: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 7: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 6: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 5: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 4: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 3: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 2: - prefetch_l2(ptr); - ptr += 64; - [[fallthrough]]; - case 1: - prefetch_l2(ptr); - [[fallthrough]]; - case 0: - break; - } - // NOLINTEND(bugprone-branch-clone) + mem_prefetch_lines(ptr, num_lines); } } // namespace rabitqlib::memory diff --git a/src/core/api.cpp b/src/core/api.cpp index 22afd727..6f89faab 100644 --- a/src/core/api.cpp +++ b/src/core/api.cpp @@ -186,17 +186,21 @@ uint16_t read_u16(const uint8_t* bytes) { return static_cast(bytes[0] | (uint16_t{bytes[1]} << 8U)); } +// The i-th UnpackedU16 code, rejected if it does not fit the configured bit width. +uint16_t checked_unpacked_code(const uint8_t* bytes, size_t i, uint16_t maximum) { + const uint16_t value = read_u16(bytes + 2 * i); + if (value > maximum) { + throw std::invalid_argument("Unpacked code exceeds the configured bit width"); + } + return value; +} + void read_codes(const Parameters& p, CodeView input, ScratchStorage& s) { const auto* bytes = static_cast(input.codes.data); if (input.layout == CodeLayout::UnpackedU16) { const auto maximum = static_cast((1U << p.total_bits) - 1); for (size_t i = 0; i < p.dim; ++i) { - s.codes[i] = read_u16(bytes + 2 * i); - if (s.codes[i] > maximum) { - throw std::invalid_argument( - "Unpacked code exceeds the configured bit width" - ); - } + s.codes[i] = checked_unpacked_code(bytes, i, maximum); } return; } @@ -390,13 +394,8 @@ float QueryState::compute_checked( const auto* bytes = static_cast(code.codes.data); const auto maximum = static_cast((1U << p.total_bits) - 1); for (size_t i = 0; i < p.dim; ++i) { - const uint16_t value = read_u16(bytes + 2 * i); - if (value > maximum) { - throw std::invalid_argument( - "Unpacked code exceeds the configured bit width" - ); - } - ip += query.query_[i] * static_cast(value); + ip += query.query_[i] * + static_cast(checked_unpacked_code(bytes, i, maximum)); } } else { const size_t ex_bits = p.total_bits - 1; diff --git a/src/index/hnsw.cpp b/src/index/hnsw.cpp index 87447731..4f6f1d2c 100644 --- a/src/index/hnsw.cpp +++ b/src/index/hnsw.cpp @@ -112,20 +112,8 @@ HierarchicalNSW::HierarchicalNSW( upper_layer_max_degree_ = M_; base_layer_max_degree_ = M_ * 2; ef_construction_ = std::max(ef_construction, M_); - default_ef_search_ = 10; - bin_data_bytes_ = BinDataMap::data_bytes(padded_dim_); - ex_data_bytes_ = ExDataMap::data_bytes(padded_dim_, ex_bits_); - level0_links_bytes_ = base_layer_max_degree_ * sizeof(PID) + sizeof(PID); - label_offset_ = - level0_links_bytes_ + sizeof(PID); // (# of edges + edges) + (cluster_id) - bin_data_offset_ = - label_offset_ + - sizeof(PID); // (# of edges + edges) + (cluster_id) + (external label) - ex_data_offset_ = - bin_data_offset_ + bin_data_bytes_; // (# of edges + edges) + (cluster_id) - // + (external label) + (BinData) - level0_record_bytes_ = ex_data_offset_ + ex_data_bytes_; + set_record_layout(); data_level0_memory_.reset( memory::huge_page_allocate(max_elements_ * level0_record_bytes_) ); @@ -144,13 +132,24 @@ HierarchicalNSW::HierarchicalNSW( max_level_ = -1; link_lists_.reserve(max_elements_); - upper_layer_links_bytes_ = upper_layer_max_degree_ * sizeof(PID) + sizeof(PID); mult_ = 1 / log(1.0 * static_cast(M_)); this->query_config_ = quant::faster_config(padded_dim_, SplitSingleQuery::kNumBits); } +void HierarchicalNSW::set_record_layout() { + bin_data_bytes_ = BinDataMap::data_bytes(padded_dim_); + ex_data_bytes_ = ExDataMap::data_bytes(padded_dim_, ex_bits_); + // Level 0: link count and links, cluster ID, external label, then the codes. + level0_links_bytes_ = (base_layer_max_degree_ + 1) * sizeof(PID); + label_offset_ = level0_links_bytes_ + sizeof(PID); + bin_data_offset_ = label_offset_ + sizeof(PID); + ex_data_offset_ = bin_data_offset_ + bin_data_bytes_; + level0_record_bytes_ = ex_data_offset_ + ex_data_bytes_; + upper_layer_links_bytes_ = (upper_layer_max_degree_ + 1) * sizeof(PID); +} + void HierarchicalNSW::save(const char* filename) const { if (level0_record_bytes_ == 0) { throw std::logic_error("HNSW index must be initialized before save"); @@ -296,23 +295,11 @@ void HierarchicalNSW::load(const char* filename) { invalid_file(); } - const size_t expected_level0 = (loaded.base_layer_max_degree_ + 1) * sizeof(PID); - const size_t expected_bin = BinDataMap::data_bytes(loaded.padded_dim_); - const size_t expected_ex = - ExDataMap::data_bytes(loaded.padded_dim_, loaded.ex_bits_); - const size_t expected_label = expected_level0 + sizeof(PID); - const size_t expected_bin_offset = expected_label + sizeof(PID); - const size_t expected_ex_offset = expected_bin_offset + expected_bin; - const size_t expected_level0_record_bytes = expected_ex_offset + expected_ex; - if (loaded.bin_data_bytes_ != expected_bin || loaded.ex_data_bytes_ != expected_ex || - loaded.level0_links_bytes_ != expected_level0 || - loaded.label_offset_ != expected_label || - loaded.bin_data_offset_ != expected_bin_offset || - loaded.ex_data_offset_ != expected_ex_offset || - loaded.level0_record_bytes_ != expected_level0_record_bytes || - loaded.upper_layer_links_bytes_ != - (loaded.upper_layer_max_degree_ + 1) * sizeof(PID) || - !std::isfinite(loaded.mult_) || loaded.mult_ <= 0.0) { + // The stored layout must be the one the validated parameters derive. + const auto stored_layout = loaded.record_layout(); + loaded.set_record_layout(); + if (loaded.record_layout() != stored_layout || !std::isfinite(loaded.mult_) || + loaded.mult_ <= 0.0) { invalid_file(); } @@ -477,7 +464,6 @@ void HierarchicalNSW::load(const char* filename) { loaded.raw_dist_func_ = (loaded.metric_type_ == METRIC_IP) ? dot_product_dis : euclidean_sqr; loaded.ip_func_ = select_excode_ipfunc(loaded.ex_bits_); - loaded.default_ef_search_ = 10; loaded.query_config_ = quant::faster_config(loaded.padded_dim_, SplitSingleQuery::kNumBits); @@ -495,7 +481,6 @@ void HierarchicalNSW::load(const char* filename) { swap(upper_layer_max_degree_, loaded.upper_layer_max_degree_); swap(base_layer_max_degree_, loaded.base_layer_max_degree_); swap(ef_construction_, loaded.ef_construction_); - swap(default_ef_search_, loaded.default_ef_search_); swap(metric_type_, loaded.metric_type_); swap(mult_, loaded.mult_); swap(max_level_, loaded.max_level_); @@ -726,8 +711,8 @@ void HierarchicalNSW::add_point( ); // Initialisation of label and cluster id - memcpy(get_external_label_pt(new_node_id), &label, sizeof(PID)); - memcpy(get_clusterid_pt(new_node_id), &cluster_id, sizeof(PID)); + set_external_label(new_node_id, label); + set_raw_clusterid(new_node_id, cluster_id); // Quantize raw data and initialize quantized data std::vector rotated_data(padded_dim_); @@ -1005,7 +990,7 @@ size_t HierarchicalNSW::remove(const PID* labels, size_t num_points) { for (const PID id : internal_ids) { if (!is_removed(id)) { const PID marked = get_raw_clusterid(id) | kRemovedMask; - std::memcpy(get_clusterid_pt(id), &marked, sizeof(PID)); + set_raw_clusterid(id, marked); ++removed; } } @@ -1337,8 +1322,8 @@ std::vector HierarchicalNSW::add( 0, level0_record_bytes_ ); - std::memcpy(get_external_label_pt(id), &id, sizeof(PID)); - std::memcpy(get_clusterid_pt(id), &assigned[i], sizeof(PID)); + set_external_label(id, id); + set_raw_clusterid(id, assigned[i]); std::vector rotated_data(padded_dim_); rotator_->rotate(data + (i * dim_), rotated_data.data()); quant::quantize_split_single( diff --git a/src/index/hnsw_search_avx2.cpp b/src/index/hnsw_search_avx2.cpp index 820982af..0bccdb29 100644 --- a/src/index/hnsw_search_avx2.cpp +++ b/src/index/hnsw_search_avx2.cpp @@ -32,9 +32,7 @@ struct HnswAvx2Kernel { std::vector> search_knn_sorted_avx2( HierarchicalNSW& index, const float* rotated_query, size_t k, size_t ef_search ) { - return index.search_knn_sorted_direct( - rotated_query, k, ef_search == 0 ? index.default_ef_search_ : ef_search - ); + return index.search_knn_sorted_direct(rotated_query, k, ef_search); } } // namespace rabitqlib::hnsw::detail diff --git a/src/index/hnsw_search_avx512_core.cpp b/src/index/hnsw_search_avx512_core.cpp index c6c80002..485f76f7 100644 --- a/src/index/hnsw_search_avx512_core.cpp +++ b/src/index/hnsw_search_avx512_core.cpp @@ -34,7 +34,7 @@ std::vector> search_knn_sorted_avx512_core( HierarchicalNSW& index, const float* rotated_query, size_t k, size_t ef_search ) { return index.search_knn_sorted_direct( - rotated_query, k, ef_search == 0 ? index.default_ef_search_ : ef_search + rotated_query, k, ef_search ); } diff --git a/src/index/hnsw_search_avx512_popcnt.cpp b/src/index/hnsw_search_avx512_popcnt.cpp index 0602e087..630a4e3b 100644 --- a/src/index/hnsw_search_avx512_popcnt.cpp +++ b/src/index/hnsw_search_avx512_popcnt.cpp @@ -33,7 +33,7 @@ std::vector> search_knn_sorted_avx512_popcnt( HierarchicalNSW& index, const float* rotated_query, size_t k, size_t ef_search ) { return index.search_knn_sorted_direct( - rotated_query, k, ef_search == 0 ? index.default_ef_search_ : ef_search + rotated_query, k, ef_search ); } diff --git a/src/index/hnsw_search_generic.cpp b/src/index/hnsw_search_generic.cpp index 77b7fba1..9d7ef5f5 100644 --- a/src/index/hnsw_search_generic.cpp +++ b/src/index/hnsw_search_generic.cpp @@ -35,9 +35,7 @@ struct HnswGenericKernel { std::vector> search_knn_sorted_generic( HierarchicalNSW& index, const float* rotated_query, size_t k, size_t ef_search ) { - return index.search_knn_sorted_direct( - rotated_query, k, ef_search == 0 ? index.default_ef_search_ : ef_search - ); + return index.search_knn_sorted_direct(rotated_query, k, ef_search); } } // namespace rabitqlib::hnsw::detail diff --git a/src/index/hnsw_search_impl.hpp b/src/index/hnsw_search_impl.hpp index 15a60c91..0995e4a3 100644 --- a/src/index/hnsw_search_impl.hpp +++ b/src/index/hnsw_search_impl.hpp @@ -99,6 +99,9 @@ inline std::vector> HierarchicalNSW::search_knn_sorted_dir if (num_stored_points_ == 0) { return result; } + if (ef_search == 0) { + ef_search = kDefaultEfSearch; + } SplitSingleQuery query_wrapper( rotated_query, padded_dim_, ex_bits_, query_config_, metric_type_ diff --git a/src/index/hnsw_search_neon.cpp b/src/index/hnsw_search_neon.cpp index a07e9506..d613e074 100644 --- a/src/index/hnsw_search_neon.cpp +++ b/src/index/hnsw_search_neon.cpp @@ -36,9 +36,7 @@ struct HnswNeonKernel { std::vector> search_knn_sorted_neon( HierarchicalNSW& index, const float* rotated_query, size_t k, size_t ef_search ) { - return index.search_knn_sorted_direct( - rotated_query, k, ef_search == 0 ? index.default_ef_search_ : ef_search - ); + return index.search_knn_sorted_direct(rotated_query, k, ef_search); } } // namespace rabitqlib::hnsw::detail diff --git a/src/index/ivf.cpp b/src/index/ivf.cpp index b3b6cec0..1efbb382 100644 --- a/src/index/ivf.cpp +++ b/src/index/ivf.cpp @@ -44,6 +44,11 @@ namespace rabitqlib::ivf { +namespace { +// remove() marks a point by this f_add; save and load keep the marker. +constexpr float kRemovedFAdd = std::numeric_limits::infinity(); +} // namespace + IVF::IVF() noexcept = default; IVF::~IVF() = default; @@ -670,7 +675,6 @@ size_t IVF::remove(const PID* ids_to_remove, size_t num_points) { // A point whose f_add is +inf has +inf estimated and lower-bound distances in every // scan path, so it never enters the result buffer or triggers reranking. - constexpr float kRemoved = std::numeric_limits::infinity(); const size_t batch_bytes = BatchDataMap::data_bytes(padded_dim_); size_t removed = 0; for (const auto& cluster : clusters_) { @@ -684,8 +688,8 @@ size_t IVF::remove(const PID* ids_to_remove, size_t num_points) { padded_dim_ ); const size_t lane = pos % fastscan::kBatchSize; - if (static_cast(cur_batch.f_add()[lane]) != kRemoved) { - cur_batch.f_add()[lane] = kRemoved; + if (static_cast(cur_batch.f_add()[lane]) != kRemovedFAdd) { + cur_batch.f_add()[lane] = kRemovedFAdd; ++removed; } } @@ -960,8 +964,7 @@ void IVF::load(const char* filename) { ); const size_t count = std::min(fastscan::kBatchSize, cluster.num() - first); for (size_t lane = 0; lane < count; ++lane) { - loaded.num_removed_ += - batch.f_add()[lane] == std::numeric_limits::infinity(); + loaded.num_removed_ += batch.f_add()[lane] == kRemovedFAdd; } } } diff --git a/src/index/pipnn.cpp b/src/index/pipnn.cpp index a00c6cbd..dc8d66fd 100644 --- a/src/index/pipnn.cpp +++ b/src/index/pipnn.cpp @@ -177,11 +177,18 @@ Workspace ScratchPool::worker(size_t id) { {}}; } -void gather(const float* data, size_t dim, const Bucket& ids, Workspace& work) { - for (size_t i = 0; i < ids.size(); ++i) { - std::copy_n(data + ids[i] * dim, dim, work.points + i * dim); +namespace { +// Copy the rows named by ids[0..count) to consecutive rows of output. +void copy_rows(const float* data, size_t dim, const PID* ids, size_t count, float* output) { + for (size_t i = 0; i < count; ++i) { + std::copy_n(data + ids[i] * dim, dim, output + i * dim); } } +} // namespace + +void gather(const float* data, size_t dim, const Bucket& ids, Workspace& work) { + copy_rows(data, dim, ids.data(), ids.size(), work.points); +} void pairwise(Workspace& work, size_t size, size_t dim, MetricType metric_type) { simd::pairwise_distances_lower( @@ -248,9 +255,7 @@ std::vector partition( leaders.reserve(leader_count); std::sample(ids.begin(), ids.end(), std::back_inserter(leaders), leader_count, random); RowMajorMatrix leader_data(leader_count, dim); - for (size_t i = 0; i < leader_count; ++i) { - std::copy_n(data + leaders[i] * dim, dim, leader_data.data() + i * dim); - } + copy_rows(data, dim, leaders.data(), leader_count, leader_data.data()); Vector leader_norms(leader_count); simd::row_norms(leader_data.data(), leader_norms.data(), leader_count, dim); std::vector assignments(ids.size() * fanout); @@ -268,9 +273,7 @@ std::vector partition( static_cast(leader_count) ); VectorMap norms(work.norms, static_cast(count)); - for (size_t i = 0; i < count; ++i) { - std::copy_n(data + ids[begin + i] * dim, dim, points.data() + i * dim); - } + copy_rows(data, dim, ids.data() + begin, count, points.data()); simd::matrix_product_transposed( points.data(), leader_data.data(), distances.data(), count, dim, leader_count ); @@ -362,6 +365,13 @@ std::vector cluster( return a.size() < b.size(); }); Bucket small; + // Merged small children become one deduplicated leaf. + const auto finish_small = [&] { + std::sort(small.begin(), small.end()); + small.erase(std::unique(small.begin(), small.end()), small.end()); + finished[i].push_back(std::move(small)); + small = Bucket(); + }; for (size_t j = 0; j < children.size(); ++j) { auto& child = children[j]; if (child.empty()) { @@ -372,10 +382,7 @@ std::vector cluster( if (small.size() < kLeafSize / 10) { continue; } - std::sort(small.begin(), small.end()); - small.erase(std::unique(small.begin(), small.end()), small.end()); - finished[i].push_back(std::move(small)); - small = Bucket(); + finish_small(); } else if (child.size() <= kLeafSize) { finished[i].push_back(std::move(child)); } else if ( @@ -402,9 +409,7 @@ std::vector cluster( } } if (!small.empty()) { - std::sort(small.begin(), small.end()); - small.erase(std::unique(small.begin(), small.end()), small.end()); - finished[i].push_back(std::move(small)); + finish_small(); } Bucket().swap(job.ids); }; diff --git a/src/index/qg.cpp b/src/index/qg.cpp index 2fcc1bfc..b3da8f70 100644 --- a/src/index/qg.cpp +++ b/src/index/qg.cpp @@ -133,9 +133,6 @@ QuantizedGraph::QuantizedGraph( , max_degree_(max_degree) , dim_(dim) , padded_dim_(dim) - , raw_dist_func_( - (metric_type == METRIC_IP) ? dot_product_dis : euclidean_sqr - ) , metric_type_(metric_type) , rotator_type_(rotator_type) , quantization_bits_(quantization_bits) @@ -372,8 +369,6 @@ void QuantizedGraph::load(const char* filename) { loaded.quantization_bits_ = 0; } - loaded.raw_dist_func_ = - (loaded.metric_type_ == METRIC_IP) ? dot_product_dis : euclidean_sqr; loaded.validate_configuration(); if (stored_padded_dim < loaded.dim_ || stored_padded_dim % 32 != 0) { throw std::runtime_error("Invalid padded dimension in quantized graph file"); @@ -675,6 +670,8 @@ void QuantizedGraph::initialize(bool randomize_rotation) { assert(record_bytes_ % sizeof(float) == 0); data_ = RowStorage(data_bytes / sizeof(float)); + raw_dist_func_ = + metric_type_ == METRIC_IP ? dot_product_dis : euclidean_sqr; if (quantization_bits_ != 0) { quantized_ip_func_ = select_excode_ipfunc(quantization_bits_); } diff --git a/src/index/qg_builder.cpp b/src/index/qg_builder.cpp index 9f2c6001..a091269f 100644 --- a/src/index/qg_builder.cpp +++ b/src/index/qg_builder.cpp @@ -227,33 +227,22 @@ void QGBuilder::write_row(PID id, BuildWorker& worker) { auto& rotated_data = worker.rotated_neighbors; rotated_data.resize((num_neighbors + 1) * padded_dim); float* rotated_centroid = rotated_data.data() + num_neighbors * padded_dim; - for (size_t i = 0; i < num_neighbors; ++i) { + // Cached rows, rotated raw vectors or reconstructed codes, as configured. + const auto rotated_row = [&](PID source, float* row) { if (cache != nullptr) { std::copy_n( - cache->rotated_vectors.data() + new_neighbors[i] * padded_dim, - padded_dim, - rotated_data.data() + i * padded_dim + cache->rotated_vectors.data() + source * padded_dim, padded_dim, row ); } else if (qg_.quantization_bits_ == 0) { - const float* neighbor_vec = qg_.get_vector(new_neighbors[i]); - qg_.rotator_->rotate(neighbor_vec, &rotated_data[i * padded_dim]); + qg_.rotator_->rotate(qg_.get_vector(source), row); } else { - qg_.reconstruct_quantized_vector( - new_neighbors[i], &rotated_data[i * padded_dim] - ); + qg_.reconstruct_quantized_vector(source, row); } + }; + for (size_t i = 0; i < num_neighbors; ++i) { + rotated_row(new_neighbors[i], rotated_data.data() + i * padded_dim); } - if (cache != nullptr) { - std::copy_n( - cache->rotated_vectors.data() + id * padded_dim, - padded_dim, - rotated_centroid - ); - } else if (qg_.quantization_bits_ == 0) { - qg_.rotator_->rotate(qg_.get_vector(id), rotated_centroid); - } else { - qg_.reconstruct_quantized_vector(id, rotated_centroid); - } + rotated_row(id, rotated_centroid); qg_.encode_neighbors(id, new_neighbors, rotated_data.data(), rotated_centroid); } degrees_[id] = worker.neighbor_ids.size(); diff --git a/tests/unit/rabitqlib/index/hnsw_test.cpp b/tests/unit/rabitqlib/index/hnsw_test.cpp index c979a844..8bd72a66 100644 --- a/tests/unit/rabitqlib/index/hnsw_test.cpp +++ b/tests/unit/rabitqlib/index/hnsw_test.cpp @@ -69,9 +69,7 @@ struct HnswPruningTestAccess { } static void set_raw_cluster_id(HierarchicalNSW& index, PID label, PID value) { - std::memcpy( - index.get_clusterid_pt(index.label_lookup_.at(label)), &value, sizeof(PID) - ); + index.set_raw_clusterid(index.label_lookup_.at(label), value); } // Give internal slot i the label count - 1 - i, as parallel construction can