diff --git a/CMakeLists.txt b/CMakeLists.txt index 480409d8..958f8432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,7 +296,7 @@ endif() if(RABITQ_BUILD_INDEX) install(DIRECTORY include/rabitqlib/index include/rabitqlib/clustering DESTINATION "${rabitq_include_dir}") - install(FILES include/rabitqlib/utils/array.hpp include/rabitqlib/utils/io.hpp + install(FILES include/rabitqlib/utils/io.hpp include/rabitqlib/utils/stopw.hpp DESTINATION "${rabitq_include_dir}/utils") install(DIRECTORY include/rabitqlib/third/hnswlib DESTINATION "${rabitq_include_dir}/third") diff --git a/include/rabitqlib/core/rotator.hpp b/include/rabitqlib/core/rotator.hpp index 49e833e8..ee0f9c7c 100644 --- a/include/rabitqlib/core/rotator.hpp +++ b/include/rabitqlib/core/rotator.hpp @@ -112,23 +112,23 @@ class MatrixRotator : public Rotator { void load(std::ifstream& input) override { input.read( reinterpret_cast(rand_mat_.data()), - static_cast(sizeof(T) * this->dim_ * this->padded_dim_) + static_cast(MatrixRotator::dump_bytes()) ); } void save(std::ofstream& output) const override { output.write( reinterpret_cast(rand_mat_.data()), - (sizeof(T) * this->dim_ * this->padded_dim_) + static_cast(MatrixRotator::dump_bytes()) ); } void load(const char* data) override { - std::memcpy(rand_mat_.data(), data, sizeof(T) * this->dim_ * this->padded_dim_); + std::memcpy(rand_mat_.data(), data, MatrixRotator::dump_bytes()); } void save(char* data) const override { - std::memcpy(data, rand_mat_.data(), sizeof(T) * this->dim_ * this->padded_dim_); + std::memcpy(data, rand_mat_.data(), MatrixRotator::dump_bytes()); } size_t dump_bytes() const override { @@ -193,35 +193,28 @@ class FhtKacRotator : public Rotator { void load(std::ifstream& input) override { input.read( reinterpret_cast(flip_.data()), - static_cast(sizeof(uint8_t) * flip_.size()) + static_cast(FhtKacRotator::dump_bytes()) ); } void save(std::ofstream& output) const override { output.write( reinterpret_cast(flip_.data()), - static_cast(sizeof(uint8_t) * flip_.size()) + static_cast(FhtKacRotator::dump_bytes()) ); } void load(const char* data) override { - std::memcpy(flip_.data(), data, sizeof(uint8_t) * flip_.size()); + std::memcpy(flip_.data(), data, FhtKacRotator::dump_bytes()); } void save(char* data) const override { - std::memcpy(data, flip_.data(), sizeof(uint8_t) * flip_.size()); + std::memcpy(data, flip_.data(), FhtKacRotator::dump_bytes()); } size_t dump_bytes() const override { return sizeof(uint8_t) * flip_.size(); } - FhtKacRotator& operator=(const FhtKacRotator& other) { - this->dim_ = other.dim_; - this->padded_dim_ = other.padded_dim_; - this->flip_ = other.flip_; - this->trunc_dim_ = other.trunc_dim_; - this->fac_ = other.fac_; - return *this; - } + FhtKacRotator& operator=(const FhtKacRotator&) = default; static void kacs_walk(float* data, size_t len) { simd::kacs_walk(data, len); } diff --git a/include/rabitqlib/core/types.hpp b/include/rabitqlib/core/types.hpp index d1638c52..ad2fd582 100644 --- a/include/rabitqlib/core/types.hpp +++ b/include/rabitqlib/core/types.hpp @@ -10,10 +10,6 @@ namespace quant { struct RabitqConfig { double t_const = -1; explicit RabitqConfig() = default; - RabitqConfig(RabitqConfig const&) = default; - RabitqConfig(RabitqConfig&&) = default; - RabitqConfig& operator=(const RabitqConfig&) = default; - RabitqConfig& operator=(RabitqConfig&&) = default; }; } // namespace quant } // namespace rabitqlib diff --git a/include/rabitqlib/utils/array.hpp b/include/rabitqlib/utils/array.hpp deleted file mode 100644 index 1e87c363..00000000 --- a/include/rabitqlib/utils/array.hpp +++ /dev/null @@ -1,172 +0,0 @@ -// The implementation is largely based on the implementation of SVS. -// https://github.com/intel/ScalableVectorSearch - -/* - * Copyright 2023 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rabitqlib/utils/memory.hpp" - -namespace rabitqlib { -namespace array_impl { -/** - * @brief get size of array - */ -template -[[nodiscard]] constexpr auto size(const Dims& dims) -> size_t { - static_assert(std::is_same_v); - - size_t res = 1; - for (const size_t cur_d : dims) { - if (cur_d != 0 && res > std::numeric_limits::max() / cur_d) { - throw std::bad_array_new_length(); - } - res *= cur_d; - } - return res; -} -} // namespace array_impl - -template < - typename T, - typename Dims = std::vector, - typename Alloc = memory::Allocator> -class Array { - private: - static_assert(std::is_trivial_v); // only handle trivial types - - /// @brief num of data objects - [[nodiscard]] constexpr auto size() const -> size_t { return array_impl::size(dims_); } - - /// @brief num of bytes for all data objects - [[nodiscard]] constexpr auto bytes() const -> size_t { - const size_t num_elements = size(); - if (num_elements > std::numeric_limits::max() / sizeof(T)) { - throw std::bad_array_new_length(); - } - return sizeof(T) * num_elements; - } - - void destroy() { - size_t num_elements = size(); - atraits::deallocate(allocator_, pointer_, num_elements); - pointer_ = nullptr; - } - - public: - using allocator_type = Alloc; - using atraits = std::allocator_traits; - using pointer = typename atraits::pointer; - using const_pointer = typename atraits::const_pointer; - - using value_type = T; - using reference = T&; - using const_reference = const T&; - - Array() = default; - - explicit Array(Dims dims, const Alloc& allocator) - : dims_(std::move(dims)), allocator_(allocator) { - size_t num_elements = size(); - pointer_ = atraits::allocate(allocator_, num_elements); - } - - explicit Array(Dims dims) : Array(std::move(dims), Alloc()) {} - - ~Array() noexcept { - if (pointer_ != nullptr) { - destroy(); - } - } - - /// @brief move constructor - Array(Array&& other) noexcept - : pointer_{std::exchange(other.pointer_, nullptr)} - , dims_{std::move(other.dims_)} - , allocator_{std::move(other.allocator_)} {} - - // Unequal, non-propagating allocators require allocation and may throw. - // NOLINTBEGIN(performance-noexcept-move-constructor) - Array& operator=(Array&& other) noexcept( - std::is_nothrow_move_assignable_v && - (atraits::propagate_on_container_move_assignment::value - ? std::is_nothrow_move_assignable_v - : atraits::is_always_equal::value) - ) { - if (this == &other) { - return *this; - } - if constexpr ( - !atraits::propagate_on_container_move_assignment::value && - !atraits::is_always_equal::value - ) { - if (allocator_ != other.allocator_ && other.pointer_ != nullptr) { - // Allocate with the destination's allocator before releasing either - // array. Unequal, non-propagating allocators cannot transfer storage. - Array replacement(other.dims_, allocator_); - std::copy_n(other.pointer_, other.size(), replacement.pointer_); - *this = std::move(replacement); - other.destroy(); - return *this; - } - } - if (pointer_ != nullptr) { - destroy(); - } - if constexpr (atraits::propagate_on_container_move_assignment::value) { - allocator_ = std::move(other.allocator_); - } - dims_ = std::move(other.dims_); - pointer_ = std::exchange(other.pointer_, nullptr); - return *this; - } - - // NOLINTEND(performance-noexcept-move-constructor) - - [[nodiscard]] pointer data() { return pointer_; } - [[nodiscard]] const_pointer data() const { return pointer_; } - - [[nodiscard]] reference at(size_t idx) { return pointer_[idx]; } - [[nodiscard]] const_reference at(size_t idx) const { return pointer_[idx]; } - - void save(std::ofstream& output) const { - if (output.good()) { - output.write(reinterpret_cast(pointer_), bytes()); - } - } - void load(std::ifstream& input) { - input.read(reinterpret_cast(pointer_), bytes()); - } - - reference operator[](size_t idx) { return pointer_[idx]; } - - private: - pointer pointer_ = nullptr; - [[no_unique_address]] Dims dims_; - [[no_unique_address]] Alloc allocator_; -}; -} // namespace rabitqlib diff --git a/include/rabitqlib/utils/memory.hpp b/include/rabitqlib/utils/memory.hpp index 9a953f88..70047f1a 100644 --- a/include/rabitqlib/utils/memory.hpp +++ b/include/rabitqlib/utils/memory.hpp @@ -151,29 +151,6 @@ class DefaultInitAlignedAllocator : public AlignedAllocator -struct Allocator { - public: - using value_type = T; - - constexpr Allocator() noexcept = default; - - template - explicit constexpr Allocator(const Allocator&) noexcept {} - - [[nodiscard]] constexpr T* allocate(std::size_t n) { return ::new T[n]; } - - constexpr void deallocate(T* ptr, [[maybe_unused]] size_t n) noexcept { - ::delete[] ptr; - } - - // Intercept zero-argument construction to do default initialization. - template - void construct(U* ptr) noexcept(std::is_nothrow_default_constructible_v) { - ::new (static_cast(ptr)) U; - } -}; - // Use the same size-aware policy as huge-page-enabled STL storage. Advice is // best-effort; actual page sizes remain under the operating system's control. template diff --git a/include/rabitqlib/utils/space.hpp b/include/rabitqlib/utils/space.hpp index 76186447..af064813 100644 --- a/include/rabitqlib/utils/space.hpp +++ b/include/rabitqlib/utils/space.hpp @@ -58,29 +58,13 @@ void scalar_quantize_optimized( float lo, float delta ) { - scalar_quantize_normal(result, vec0, dim, lo, delta); -} - -template <> -inline void scalar_quantize_optimized( - uint8_t* __restrict__ result, - const float* __restrict__ vec0, - size_t dim, - float lo, - float delta -) { - simd::scalar_quantize_uint8(result, vec0, dim, lo, delta); -} - -template <> -inline void scalar_quantize_optimized( - uint16_t* __restrict__ result, - const float* __restrict__ vec0, - size_t dim, - float lo, - float delta -) { - simd::scalar_quantize_uint16(result, vec0, dim, lo, delta); + if constexpr (std::is_same_v) { + simd::scalar_quantize_uint8(result, vec0, dim, lo, delta); + } else if constexpr (std::is_same_v) { + simd::scalar_quantize_uint16(result, vec0, dim, lo, delta); + } else { + scalar_quantize_normal(result, vec0, dim, lo, delta); + } } } // namespace scalar_impl @@ -197,7 +181,7 @@ template void scalar_quantize( T* __restrict__ result, const TD* __restrict__ vec0, size_t dim, TD lo, TD delta ) { - assert_integral(); + static_assert(std::is_integral_v, "T must be an integral type"); scalar_impl::scalar_quantize_optimized(result, vec0, dim, lo, delta); } diff --git a/include/rabitqlib/utils/tools.hpp b/include/rabitqlib/utils/tools.hpp index a653b463..a36a847e 100644 --- a/include/rabitqlib/utils/tools.hpp +++ b/include/rabitqlib/utils/tools.hpp @@ -46,12 +46,23 @@ inline T rand_integer(T min, T max) { return distribution(generator); } +template +constexpr T div_round_up(T x, T divisor) { + static_assert(std::is_integral_v, "T must be an integral type"); + return (x / divisor) + static_cast((x % divisor) != 0); +} + constexpr size_t div_round_up(size_t val, size_t div) { - return (val / div) + static_cast((val % div) != 0); + return div_round_up(val, div); } constexpr size_t round_up_to_multiple(size_t val, size_t multiple_of) { - return multiple_of * (div_round_up(val, multiple_of)); + return multiple_of * div_round_up(val, multiple_of); +} + +template +constexpr T round_up_to_multiple_of(size_t x, size_t multiple_of) { + return round_up_to_multiple(x, multiple_of); } inline size_t floor_log2(size_t x) { @@ -70,17 +81,6 @@ inline size_t ceil_log2(size_t x) { inline bool is_powerof2(size_t n) { return n > 0 && (n & (n - 1)) == 0; } -template -constexpr T div_round_up(T x, T divisor) { - static_assert(std::is_integral_v, "T must be an integral type"); - return (x / divisor) + static_cast((x % divisor) != 0); -} - -template -constexpr T round_up_to_multiple_of(size_t x, size_t multiple_of) { - return multiple_of * (div_round_up(x, multiple_of)); -} - // Available logical CPUs in the calling context, with a hardware-count fallback. size_t total_threads(); diff --git a/src/core/api.cpp b/src/core/api.cpp index 2eda58f2..a23a12a6 100644 --- a/src/core/api.cpp +++ b/src/core/api.cpp @@ -245,9 +245,9 @@ Requirements requirements(const Parameters& p, CodeLayout layout) { return sizing; }(); Requirements r{0, 0, budget.bytes(p.dim), kWorkspaceAlignment}; - const auto sizes = code_sizes(p, layout); - r.code_bytes = sizes.first; - r.extra_bytes = sizes.second; + const auto [code_bytes, extra_bytes] = code_sizes(p, layout); + r.code_bytes = code_bytes; + r.extra_bytes = extra_bytes; return r; } @@ -376,9 +376,9 @@ float QueryState::compute_checked( throw std::invalid_argument("QueryState has not been prepared"); } const Parameters p{query.dim_, query.total_bits_, query.metric_}; - const auto sizes = code_sizes(p, code.layout); - check_buffer(code.codes, sizes.first); - check_buffer(code.extra, sizes.second); + const auto [code_bytes, extra_bytes] = code_sizes(p, code.layout); + check_buffer(code.codes, code_bytes); + check_buffer(code.extra, extra_bytes); if (!std::isfinite(factors.add) || !std::isfinite(factors.rescale) || !std::isfinite(factors.error)) { throw std::invalid_argument("Quantization factors must be finite"); diff --git a/tests/unit/rabitqlib/utils/array_test.cpp b/tests/unit/rabitqlib/utils/array_test.cpp deleted file mode 100644 index 36912b91..00000000 --- a/tests/unit/rabitqlib/utils/array_test.cpp +++ /dev/null @@ -1,101 +0,0 @@ -#include "rabitqlib/utils/array.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace { -struct Allocations { - std::map owners; - bool mismatch = false; - bool fail = false; -}; - -template -struct TrackedAllocator { - using value_type = int; - using propagate_on_container_move_assignment = std::bool_constant; - using is_always_equal = std::false_type; - Allocations* allocations; - int id; - - int* allocate(size_t count) { - if (allocations->fail) { - throw std::bad_alloc(); - } - auto* ptr = std::allocator{}.allocate(count); - allocations->owners[ptr] = id; - return ptr; - } - void deallocate(int* ptr, size_t count) { - allocations->mismatch |= allocations->owners.at(ptr) != id; - allocations->owners.erase(ptr); - std::allocator{}.deallocate(ptr, count); - } - bool operator!=(const TrackedAllocator& other) const { return id != other.id; } -}; - -template -void check_move(int destination_id) { - Allocations state; - using Array = rabitqlib::Array, TrackedAllocator>; - static_assert(std::is_nothrow_move_assignable_v == Propagate); - { - Array source({3}, {&state, 1}); - Array target({2}, {&state, destination_id}); - source[0] = 17; - source[1] = 23; - source[2] = 42; - auto* previous = source.data(); - target = std::move(source); - EXPECT_EQ(target[0], 17); - EXPECT_EQ(target[1], 23); - EXPECT_EQ(target[2], 42); - // A successful move explicitly leaves the source pointer null. - // NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - EXPECT_EQ(source.data(), nullptr); - EXPECT_EQ(target.data() == previous, Propagate || destination_id == 1); - auto& alias = target; - target = std::move(alias); - EXPECT_EQ(target[2], 42); - } - EXPECT_TRUE(state.owners.empty()); - EXPECT_FALSE(state.mismatch); -} - -TEST(ArrayOwnership, MoveAssignmentHonorsAllocator) { - check_move(1); - check_move(2); - check_move(1); - check_move(2); -} - -TEST(ArrayOwnership, FailedUnequalMovePreservesBothArrays) { - Allocations state; - using Array = rabitqlib::Array, TrackedAllocator>; - { - Array source({3}, {&state, 1}), target({2}, {&state, 2}); - source[0] = 17; - target[0] = 23; - const auto* source_ptr = source.data(); - const auto* target_ptr = target.data(); - state.fail = true; - EXPECT_THROW(target = std::move(source), std::bad_alloc); - state.fail = false; - // The failed allocation must leave the source untouched. - // NOLINTNEXTLINE(bugprone-use-after-move,clang-analyzer-cplusplus.Move) - EXPECT_EQ(source.data(), source_ptr); - EXPECT_EQ(target.data(), target_ptr); - EXPECT_EQ(source[0], 17); - EXPECT_EQ(target[0], 23); - } - EXPECT_TRUE(state.owners.empty()); - EXPECT_FALSE(state.mismatch); -} -} // namespace diff --git a/tests/unit/rabitqlib/utils/memory_test.cpp b/tests/unit/rabitqlib/utils/memory_test.cpp index 567b1cb2..a1b84437 100644 --- a/tests/unit/rabitqlib/utils/memory_test.cpp +++ b/tests/unit/rabitqlib/utils/memory_test.cpp @@ -15,8 +15,6 @@ #include #include -#include "rabitqlib/utils/array.hpp" - namespace rabitqlib::memory { namespace { @@ -145,12 +143,5 @@ TEST(AlignedAllocationTest, HugePageStoragePreservesExplicitAlignmentAndOverflow #endif } -TEST(ArrayTest, RejectsDimensionProductOverflow) { - EXPECT_THROW( - (Array(std::vector{std::numeric_limits::max(), 2})), - std::bad_array_new_length - ); -} - } // namespace } // namespace rabitqlib::memory