diff --git a/include/rabitqlib/clustering/detail/kmeans.hpp b/include/rabitqlib/clustering/detail/kmeans.hpp index 5cde801b..2c7f835a 100644 --- a/include/rabitqlib/clustering/detail/kmeans.hpp +++ b/include/rabitqlib/clustering/detail/kmeans.hpp @@ -245,7 +245,6 @@ class LloydKMeans : public Parameters { ); } const uint32_t num_workers = effective_num_threads(); - static_cast(num_workers); // Leave headroom for products of squared distances in graph pruning, // as well as quantized reconstruction and rotated residuals. const double max_coordinate = std::sqrt( @@ -486,7 +485,6 @@ class LloydKMeans : public Parameters { const std::vector& previous_centroids ) { const uint32_t num_workers = effective_num_threads(); - static_cast(num_workers); // Means, normalization, and displacement share one worker team. Preserve // the per-centroid arithmetic; empty rows are measured after refilling. const auto cluster_shift = [&](size_t cluster) { diff --git a/include/rabitqlib/core/detail/rabitq_impl.hpp b/include/rabitqlib/core/detail/rabitq_impl.hpp index 16de73e5..359f2aee 100644 --- a/include/rabitqlib/core/detail/rabitq_impl.hpp +++ b/include/rabitqlib/core/detail/rabitq_impl.hpp @@ -93,28 +93,9 @@ inline CodeFactorTerms code_factor_terms( namespace one_bit { /** - * Return data - centroid and write its sign code: 1 for strictly positive - * coordinates, 0 otherwise (including zero). All buffers contain dim elements. - */ -template -inline RowMajorArray one_bit_code( - const T* data, const T* centroid, size_t dim, int* binary_code -) { - ConstRowMajorArrayMap data_arr(data, 1, dim); - ConstRowMajorArrayMap cent_arr(centroid, 1, dim); - - RowMajorArray residual_arr = data_arr - cent_arr; - - RowMajorArrayMap x_u(binary_code, 1, static_cast(dim)); - x_u = (residual_arr > 0).template cast(); - - return residual_arr; -} - -/** - * Write the sign code and estimation factors for data - centroid, using the - * same zero convention as one_bit_code. Inputs, binary_code, residual and - * centered scratch each contain dim elements. + * Write the sign code and estimation factors for data - centroid: 1 for strictly + * positive coordinates, 0 otherwise (including zero). Inputs, binary_code, residual + * and centered scratch each contain dim elements. * f_add/f_rescale supply the additive/scaling terms for squared L2 or 1 - dot. * f_error is query-independent: multiplying it by the query residual norm gives * a probabilistic error margin, not a deterministic bound for every query. diff --git a/include/rabitqlib/index/hnsw/hnsw.hpp b/include/rabitqlib/index/hnsw/hnsw.hpp index 5c0ed5c5..5d9554a6 100644 --- a/include/rabitqlib/index/hnsw/hnsw.hpp +++ b/include/rabitqlib/index/hnsw/hnsw.hpp @@ -222,7 +222,6 @@ class HierarchicalNSW { HierarchicalNSW&, const float*, size_t, size_t ); - static constexpr PID kMaxLabelOperationLock = 65536; // `remove` sets the high bit of a point's stored cluster ID. construct and load // keep num_clusters_ at or below 2^31, so a real cluster ID never has it. static constexpr PID kRemovedMask = 0x80000000U; @@ -241,9 +240,6 @@ class HierarchicalNSW { double mult_{0.0}; int max_level_{0}; - // Locks operations with element by label value - mutable std::vector label_op_locks_; - std::mutex global_; std::vector link_list_locks_; @@ -295,11 +291,6 @@ class HierarchicalNSW { float (*raw_dist_func_)(const float* __restrict__, const float* __restrict__, size_t){ nullptr}; - std::mutex& get_label_op_mutex(PID label) const { - size_t lock_id = label & (kMaxLabelOperationLock - 1); - return label_op_locks_[lock_id]; - } - PID get_external_label(PID internal_id) const { PID return_label; memcpy( diff --git a/src/index/hnsw.cpp b/src/index/hnsw.cpp index 67736a6d..8570eeb8 100644 --- a/src/index/hnsw.cpp +++ b/src/index/hnsw.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -65,7 +64,6 @@ HierarchicalNSW::HierarchicalNSW( MetricType metric_type ) : metric_type_(metric_type) - , label_op_locks_(kMaxLabelOperationLock) , link_list_locks_(max_elements) , element_levels_(max_elements) , raw_dist_func_( @@ -361,7 +359,6 @@ void HierarchicalNSW::load(const char* filename) { loaded.element_levels_ = std::vector(loaded.max_elements_); loaded.link_list_locks_ = std::vector(loaded.max_elements_); - loaded.label_op_locks_ = std::vector(kMaxLabelOperationLock); loaded.link_lists_.reserve(loaded.max_elements_); loaded.link_lists_.resize(element_count); for (size_t i = 0; i < element_count; ++i) { @@ -489,7 +486,6 @@ void HierarchicalNSW::load(const char* filename) { swap(metric_type_, loaded.metric_type_); swap(mult_, loaded.mult_); swap(max_level_, loaded.max_level_); - label_op_locks_.swap(loaded.label_op_locks_); link_list_locks_.swap(loaded.link_list_locks_); swap(entry_point_id_, loaded.entry_point_id_); swap(level0_links_bytes_, loaded.level0_links_bytes_); @@ -594,22 +590,13 @@ void HierarchicalNSW::construct( void HierarchicalNSW::add_point( PID label, PID cluster_id, const quant::RabitqConfig& config ) { - std::unique_lock label_lock(get_label_op_mutex(label)); - int node_level = 0; PID new_node_id = 0; { + // construct passes distinct labels below its validated capacity. std::unique_lock label_lookup_lock(label_lookup_lock_); - if (label_lookup_.find(label) != label_lookup_.end()) { - throw std::runtime_error( - "Currently not support replacement of existing elements, only support " - "inserting elements with distinct labels" - ); - } - - if (num_stored_points_ >= max_elements_) { - throw std::runtime_error("The number of elements exceeds the specified limit"); - } + assert(label_lookup_.find(label) == label_lookup_.end()); + assert(num_stored_points_ < max_elements_); new_node_id = num_stored_points_; link_lists_.emplace_back(); @@ -682,15 +669,7 @@ void HierarchicalNSW::add_point( const PID* datal = data + 1; for (size_t i = 0; i < size; i++) { PID cand = datal[i]; - if (cand > max_elements_) { - throw std::runtime_error( - "HNSW construct upper-layer traversal: invalid candidate " - "ID " + - std::to_string(cand) + " at level " + - std::to_string(level) + - " (capacity=" + std::to_string(max_elements_) + ")" - ); - } + assert(cand < num_stored_points_); float d = get_data_dist(cand, new_node_id); if (d < current_distance) { current_distance = d; @@ -707,10 +686,6 @@ void HierarchicalNSW::add_point( search_base_layer(current_id, new_node_id, level); current_id = mutually_connect_new_element(new_node_id, top_candidates, level); } - } else { - // Do nothing for the first element - entry_point_id_ = 0; - max_level_ = node_level; } // Releasing lock for the maximum level @@ -1145,13 +1120,7 @@ maxheap> HierarchicalNSW::search_base_layer_quant( for (size_t j = 0; j < size; j++) { PID candidate_id = datal[j]; - if (candidate_id >= num_stored_points_) { - throw std::runtime_error( - "HNSW add layer search: invalid candidate ID " + - std::to_string(candidate_id) + " at level " + std::to_string(layer) + - " (stored_points=" + std::to_string(num_stored_points_.load()) + ")" - ); - } + assert(candidate_id < num_stored_points_); if (visited->get(candidate_id)) { continue; } @@ -1227,15 +1196,7 @@ void HierarchicalNSW::add_point_quant( const PID* datal = data + 1; for (size_t i = 0; i < size; i++) { PID cand = datal[i]; - if (cand >= num_stored_points_) { - throw std::runtime_error( - "HNSW add upper-layer traversal: invalid candidate " - "ID " + - std::to_string(cand) + " at level " + - std::to_string(level) + " (stored_points=" + - std::to_string(num_stored_points_.load()) + ")" - ); - } + assert(cand < num_stored_points_); float d = get_quant_dist(cand, new_node_id, query); if (d < current_distance) { current_distance = d; @@ -1254,9 +1215,6 @@ void HierarchicalNSW::add_point_quant( current_id = mutually_connect_quant(new_node_id, top_candidates, level, query); } - } else { - entry_point_id_ = new_node_id; - max_level_ = node_level; } if (node_level > previous_max_level) { diff --git a/src/index/hnsw_search_impl.hpp b/src/index/hnsw_search_impl.hpp index fc6759dd..15a60c91 100644 --- a/src/index/hnsw_search_impl.hpp +++ b/src/index/hnsw_search_impl.hpp @@ -1,10 +1,9 @@ #pragma once #include +#include #include #include -#include -#include #include #include @@ -148,13 +147,7 @@ inline std::vector> HierarchicalNSW::search_knn_sorted_dir const PID* datal = data + 1; for (size_t i = 0; i < size; i++) { PID cand = datal[i]; - if (cand > max_elements_) { - throw std::runtime_error( - "HNSW search upper-layer traversal: invalid candidate ID " + - std::to_string(cand) + " at level " + std::to_string(level) + - " (capacity=" + std::to_string(max_elements_) + ")" - ); - } + assert(cand < num_stored_points_); EstimateRecord candidate_estimate; get_bin_est_direct( diff --git a/src/index/ivf.cpp b/src/index/ivf.cpp index 7b920533..642cb5eb 100644 --- a/src/index/ivf.cpp +++ b/src/index/ivf.cpp @@ -313,9 +313,7 @@ void IVF::quantize_cluster( const quant::RabitqConfig& config ) { size_t num_points = point_ids.size(); - if (cp.num() != num_points) { - throw std::invalid_argument("Cluster size and ID count differ"); - } + assert(cp.num() == num_points); std::copy(point_ids.begin(), point_ids.end(), cp.ids()); this->rotator_->rotate(cur_centroid, rotated_centroid); @@ -361,8 +359,7 @@ void IVF::add( bool fast_quantization, size_t num_threads ) { - if (!ready_ || initializer_ == nullptr || rotator_ == nullptr || - clusters_.size() != num_clusters_ || cluster_storage_.size() != num_clusters_) { + if (!ready_) { throw std::logic_error("IVF index must be constructed or loaded before add"); } if (num_points == 0) { @@ -622,8 +619,7 @@ void IVF::extend_cluster( } void IVF::reserve(PID cluster_id, size_t num_points) { - if (!ready_ || clusters_.size() != num_clusters_ || - cluster_storage_.size() != num_clusters_) { + if (!ready_) { throw std::logic_error("IVF index must be constructed or loaded before reserve"); } if (cluster_id >= num_clusters_) { @@ -662,8 +658,7 @@ void IVF::reserve(PID cluster_id, size_t num_points) { } size_t IVF::remove(const PID* ids_to_remove, size_t num_points) { - if (!ready_ || initializer_ == nullptr || rotator_ == nullptr || - clusters_.size() != num_clusters_ || cluster_storage_.size() != num_clusters_) { + if (!ready_) { throw std::logic_error("IVF index must be constructed or loaded before remove"); } if (num_points == 0) { @@ -709,8 +704,7 @@ size_t IVF::remove(const PID* ids_to_remove, size_t num_points) { } void IVF::save(const char* filename) const { - if (!ready_ || initializer_ == nullptr || rotator_ == nullptr || - clusters_.size() != num_clusters_ || cluster_storage_.size() != num_clusters_) { + if (!ready_) { throw std::logic_error("Cannot save an unconstructed IVF index"); } if (filename == nullptr || filename[0] == '\0') { @@ -1025,7 +1019,7 @@ void IVF::search( if (metric_type_ == METRIC_L2) { query_wrapper.set_g_add(dist); - } else if (metric_type_ == METRIC_IP) { + } else { const float residual_norm = std::sqrt(euclidean_sqr( rotated_query.data(), initializer_->centroid(cluster_id), padded_dim_ )); @@ -1033,9 +1027,6 @@ void IVF::search( rotated_query.data(), initializer_->centroid(cluster_id), padded_dim_ ); query_wrapper.set_g_add(residual_norm, query_centroid_ip); - } else { - // unsupported - throw std::invalid_argument("Quantization only supports L2 and IP metrics"); } search_cluster(cluster, query_wrapper, result_pool, use_hacc, query); } @@ -1113,8 +1104,7 @@ void IVF::search_batch( inline void IVF::validate_search( const float* query, size_t k, size_t nprobe, const PID* result_ids ) const { - if (!ready_ || initializer_ == nullptr || rotator_ == nullptr || - clusters_.size() != num_clusters_ || cluster_storage_.size() != num_clusters_) { + if (!ready_) { throw std::logic_error("IVF index must be constructed or loaded before search"); } if (query == nullptr) { diff --git a/src/simd/dispatch.cpp b/src/simd/dispatch.cpp index 66c9772e..5a96a5a8 100644 --- a/src/simd/dispatch.cpp +++ b/src/simd/dispatch.cpp @@ -1,9 +1,9 @@ +#include #include #include #include #include "dispatch_resolver.hpp" -#include "excode_dispatch.hpp" #include "rabitqlib/core/detail/simd/matrix_dispatch.hpp" #include "rabitqlib/core/detail/simd/pack_excode_dispatch.hpp" #include "rabitqlib/core/detail/simd/quantization_dispatch.hpp" @@ -197,7 +197,9 @@ void fht_rotate( kFhtRotateFn(data, rotated_data, dim, padded_dim, trunc_dim, scale_factor, flip); } -ExcodeIpTable resolve_excode_ip_table() { +using ExcodeIpTable = std::array; + +static ExcodeIpTable resolve_excode_ip_table() { const auto generic = ExcodeIpTable{ ip_fxu0, excode_ipimpl::ip16_fxu1_generic, diff --git a/src/simd/dispatch_resolver.hpp b/src/simd/dispatch_resolver.hpp index 2aa2e2e6..b00409d1 100644 --- a/src/simd/dispatch_resolver.hpp +++ b/src/simd/dispatch_resolver.hpp @@ -22,14 +22,6 @@ Function resolve_kernel( return fallback; } -template -Function resolve_kernel(Function avx2, Function fallback) { - if (cpu::has_avx2()) { - return avx2; - } - return fallback; -} - // Use a distinct name: a fallback function pointer can implicitly convert to bool. // Overloading the x86 resolver would let MSVC select this overload when __restrict // makes the SIMD and scalar function-pointer types differ. @@ -53,8 +45,6 @@ Function resolve_optional_kernel(Function preferred, Function fallback, bool sup // The calling function's local static still provides thread-safe initialization. #define RABITQ_RESOLVE(...) \ [&]() RABITQ_COLD_INIT { return rabitqlib::simd::resolve_kernel(__VA_ARGS__); }() -#define RABITQ_RESOLVE2(...) RABITQ_RESOLVE(__VA_ARGS__) #else #define RABITQ_RESOLVE(avx512, avx2, fallback, ...) fallback -#define RABITQ_RESOLVE2(avx2, fallback) fallback #endif diff --git a/src/simd/excode_dispatch.hpp b/src/simd/excode_dispatch.hpp deleted file mode 100644 index 08857603..00000000 --- a/src/simd/excode_dispatch.hpp +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace rabitqlib::simd { - -using ExcodeIpTable = std::array; - -ExcodeIpTable resolve_excode_ip_table(); - -} // namespace rabitqlib::simd diff --git a/tests/unit/rabitqlib/quantization/rabitq_test.cpp b/tests/unit/rabitqlib/quantization/rabitq_test.cpp index 20de7373..d5b13a87 100644 --- a/tests/unit/rabitqlib/quantization/rabitq_test.cpp +++ b/tests/unit/rabitqlib/quantization/rabitq_test.cpp @@ -595,10 +595,10 @@ TEST(RabitqFullQuantizationTest, ReconstructsFromCodeAndEstimatorFactors) { } // An exactly-zero residual coordinate must be encoded consistently by both halves of -// the split code. one_bit_code() uses (residual > 0) and so calls zero negative; if -// ex_bits_code() calls it positive, the assembled code lands 2^ex_bits away from where -// the factors assume it is. The == ||r||^2 invariant cannot see this, -// because a zero coordinate contributes nothing to that inner product. +// the split code. one_bit_code_with_factor() uses (residual > 0) and so calls zero +// negative; if ex_bits_code() calls it positive, the assembled code lands 2^ex_bits away +// from where the factors assume it is. The == ||r||^2 invariant cannot see +// this, because a zero coordinate contributes nothing to that inner product. TEST(RabitqSignConventionTest, ZeroResidualCoordinateGetsSmallestMagnitudeCode) { constexpr size_t kDim = 64; constexpr size_t kTotalBits = 8; diff --git a/tests/unit/rabitqlib/quantization/xy_quantization_test.cpp b/tests/unit/rabitqlib/quantization/xy_quantization_test.cpp index a3fa8cfd..70fdcf5d 100644 --- a/tests/unit/rabitqlib/quantization/xy_quantization_test.cpp +++ b/tests/unit/rabitqlib/quantization/xy_quantization_test.cpp @@ -175,7 +175,7 @@ TEST(XyQuantization, BaseBitsOneMatchesExistingOneBitPlusExBits) { // ip(base)*2^extra_bits + ip(extra) must equal ip(total_code) -- the // linearity the boosting step relies on. // The sign bit must be assigned the same way here as in the 1-bit path, -// including for an exactly-zero residual component. one_bit_code() uses +// including for an exactly-zero residual component. one_bit_code_with_factor() uses // `residual > 0`, so a zero lands on the 0 side; combined_code() has to agree // or the same vector gets different sign bits from the two paths. // @@ -200,8 +200,11 @@ TEST(XyQuantization, SignBitMatchesOneBitPathOnZeroResiduals) { ASSERT_GT(zero_dims, 0U); std::vector ref_bin(kDim, -1); - quant::rabitq_impl::one_bit::one_bit_code( - data.data(), centroid.data(), kDim, ref_bin.data() + float ref_add = 0; + float ref_rescale = 0; + float ref_error = 0; + quant::rabitq_impl::one_bit::one_bit_code_with_factor( + data.data(), centroid.data(), kDim, ref_bin.data(), ref_add, ref_rescale, ref_error ); std::vector residual(kDim);