Reduce CI duplication and strengthen clustering and SIMD coverage

.github/scripts/ci_changes.py

@@ -72,8 +72,12 @@ def classify(paths):
             selected.add("cpp")
             if path.endswith(CPP_SUFFIXES):
                 selected.add("cpp_format")
+        elif path == "tests/python/test_cpp_clustering_examples.py":
+            selected.update(("cpp", "python_quality"))
         elif path.startswith("tests/python/"):
             selected.update(("python_build", "wheels"))
+            if path == "tests/python/conftest.py":
+                selected.add("cpp")
             if path.endswith(".py"):
                 selected.add("python_quality")
         elif path.startswith("sample/python/"):

.github/scripts/test_ci_changes.py

@@ -44,6 +44,13 @@ def test_language_specific_changes(self):
             },
             "python_bindings/__init__.py": {"python_quality", "python_build", "wheels"},
             "python_bindings/licenses/LLVM-OpenMP.txt": {"python_build", "wheels"},
+            "tests/python/test_cpp_clustering_examples.py": {"cpp", "python_quality"},
+            "tests/python/conftest.py": {
+                "cpp",
+                "python_quality",
+                "python_build",
+                "wheels",
+            },
             "tests/python/test_ivf.py": {"python_quality", "python_build", "wheels"},
             "tests/python/fixtures/ivf_legacy_4bit.index": {
                 "python_build",

.github/workflows/python.yml

@@ -67,8 +67,16 @@ jobs:
       - name: Build and install wheel
         run: pip install ".[test]" -v
 
-      - name: Run pytest
-        run: pytest --tb=short -q
+      # Repaired-wheel jobs run the full Python suite.
+      - name: Smoke-test source installation
+        run: >-
+          python -m pytest --tb=short -q
+          tests/python/test_import.py
+          tests/python/test_ivf.py::test_save_load_roundtrip
+          tests/python/test_hnsw.py::test_save_load_roundtrip
+          tests/python/test_symqg.py::test_save_load_roundtrip
+          tests/python/test_qgkmeans.py::test_train_populates_faiss_style_outputs
+          tests/python/test_rabitqkmeans.py::test_flat_parameters_and_outputs
 
   python-windows:
     needs: changes
@@ -84,5 +92,33 @@ jobs:
           cache: pip
       - name: Build and install package
         run: python -m pip install ".[test]" -v
-      - name: Test Python bindings
-        run: python -m pytest tests/python --tb=short -q
+      - name: Smoke-test source installation
+        run: >-
+          python -m pytest --tb=short -q
+          tests/python/test_import.py
+          tests/python/test_ivf.py::test_save_load_roundtrip
+          tests/python/test_hnsw.py::test_save_load_roundtrip
+          tests/python/test_symqg.py::test_save_load_roundtrip
+          tests/python/test_qgkmeans.py::test_train_populates_faiss_style_outputs
+          tests/python/test_rabitqkmeans.py::test_flat_parameters_and_outputs
+
+  faiss-comparison:
+    needs: changes
+    if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.python_build == 'true') }}
+    name: FAISS comparison script
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      OMP_NUM_THREADS: "2"
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install system dependencies
+        run: sudo apt-get update -qq && sudo apt-get install -y cmake
+      - name: Install comparison dependencies
+        run: python -m pip install ".[test]" faiss-cpu
+      - name: Test comparison script
+        run: python -m pytest tests/python/test_compare_with_faiss.py --tb=short -q

.github/workflows/test.yaml

@@ -98,14 +98,26 @@ jobs:
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.11"
       - name: Install dependencies
         run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build
       - name: Configure
-        run: cmake -S . -B build -G Ninja -DRABITQ_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release
+        run: cmake -S . -B build -G Ninja -DRABITQ_BUILD_TESTS=ON -DRABITQ_BUILD_SAMPLES=ON -DCMAKE_BUILD_TYPE=Release
       - name: Build
         run: cmake --build build
-      - name: Test
-        run: ctest --test-dir build --output-on-failure
+      # The installed-consumer job runs the full portable Release suite.
+      - name: Test native build smoke coverage
+        run: >-
+          ctest --test-dir build --output-on-failure --no-tests=error
+          -R 'MatrixDispatchTest|FloatDistance|FastScan|QGKMeansTest.TrainsAndReturnsAssignmentsForFinalCentroids|RaBitQKMeansTest.FinalModesKeepFlatLloydAssignmentsAcrossClusterCounts'
+      - name: Install example test dependencies
+        run: python -m pip install numpy pytest
+      - name: Test C++ examples
+        env:
+          RABITQ_REQUIRE_CPP_EXAMPLES: "1"
+        run: python -m pytest tests/python/test_cpp_clustering_examples.py --tb=short -q
 
   windows:
     needs: changes
@@ -135,6 +147,7 @@ jobs:
       - uses: actions/checkout@v7
       - name: Install dependencies
         run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build
+      # Keep assertions/debug symbols while avoiding unoptimized sanitizer loops.
       - name: Configure
         run: >-
           cmake -S . -B build-sanitizers -G Ninja
@@ -143,6 +156,7 @@ jobs:
           -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF
           -DRABITQ_ENABLE_SANITIZERS=ON
           -DCMAKE_BUILD_TYPE=Debug
+          -DCMAKE_CXX_FLAGS_DEBUG="-O1 -g -fno-optimize-sibling-calls"
       - name: Build
         run: cmake --build build-sanitizers
       - name: Test
@@ -172,6 +186,30 @@ jobs:
           cmake --build build-install
       - name: Test portable Release build
         run: ctest --test-dir build-install --output-on-failure
+      - name: Download pinned Intel SDE
+        run: |
+          curl --fail --location --retry 3 --max-time 120 \
+            https://downloadmirror.intel.com/924984/sde-external-10.13.1-2026-07-28-lin.tar.xz \
+            -o "$RUNNER_TEMP/sde.tar.xz"
+          echo "94e97d623fec54385686e1e7ba65ebc9941748c05ee451423948334892bf2b50  $RUNNER_TEMP/sde.tar.xz" | sha256sum --check
+          mkdir -p "$RUNNER_TEMP/sde"
+          tar -xJf "$RUNNER_TEMP/sde.tar.xz" -C "$RUNNER_TEMP/sde" --strip-components=1
+      - name: Test AVX2-only and AVX-512 dispatch under emulation
+        env:
+          OMP_NUM_THREADS: "2"
+        run: |
+          simd_filter='CpuFeatures.*:FloatDistance.*:MatrixDispatchTest.*:FastScan*.*:BatchEstimatorTest.*'
+          simd_filter+=':FhtDispatchTest.*:FlipSignTest.*:KacsWalkTest.*:ExcodeIp.*:MaskIpX0Q.*'
+          simd_filter+=':PortableKernels.*:QueryPreparation.*:ScalarQuantize.*:WarmupIpX0Q.*:PackBinary.*'
+          simd_filter+=':ExplicitSimd/RabitqRescaleBackendTest.*:QGEstimatorTest.*'
+          simd_filter+=':QGKMeansTest.TrainsAndReturnsAssignmentsForFinalCentroids'
+          simd_filter+=':RaBitQKMeansTest.FinalModesKeepFlatLloydAssignmentsAcrossClusterCounts'
+          simd_filter+=':IvfPersistenceTest.ReloadsRawAndQuantizedStorage'
+          simd_filter+=':HnswSaveTest.SuccessfulSavePreservesSearchAfterLoading'
+          for cpu_model in hsw icl; do
+            RABITQ_TEST_CPU="$cpu_model" "$RUNNER_TEMP/sde/sde64" "-$cpu_model" -- \
+              build-install/tests/rabitq_tests --gtest_filter="$simd_filter"
+          done
       - name: Install RaBitQ-Library
         run: cmake --install build-install
       - name: Build downstream consumer

pyproject.toml

@@ -44,8 +44,8 @@ cmake.args = [
 build = "cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64 cp311-manylinux_aarch64 cp312-manylinux_aarch64 cp313-manylinux_aarch64 cp314-manylinux_aarch64 cp311-win_amd64 cp312-win_amd64 cp313-win_amd64 cp314-win_amd64 cp311-macosx_arm64 cp312-macosx_arm64 cp313-macosx_arm64 cp314-macosx_arm64"
 build-verbosity = 1
 environment = { CMAKE_ARGS = "-DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF" }
-test-requires = ["pytest", "faiss-cpu"]
-test-command = "pytest {project}/tests/python --tb=short -q"
+test-requires = ["pytest"]
+test-command = "pytest {project}/tests/python --ignore={project}/tests/python/test_compare_with_faiss.py --ignore={project}/tests/python/test_cpp_clustering_examples.py --tb=short -q"
 
 [tool.cibuildwheel.linux]
 manylinux-aarch64-image = "manylinux_2_28"

tests/README.md

@@ -98,6 +98,27 @@ AVX-512 tests skip when the CPU or OS lacks the required features. Tests using
 `/dev/full` skip on Windows; native POSIX path tests compile only on their
 applicable platforms. Passing on an AVX2 machine does not verify AVX-512 execution.
 
+## CI coverage
+
+The installed-consumer job runs the full portable Linux Release suite. The Ubuntu
+native build runs SIMD and clustering smoke tests; Windows, ARM64, and ASan/UBSan
+jobs retain their full C++ suites. The sanitizer job uses Debug with
+`-O1 -g -fno-optimize-sibling-calls`, retaining assertions, frame pointers, leak
+checking, and both ASan/UBSan while avoiding unoptimized clustering loops.
+
+The portable Linux job also reuses its test binary under pinned
+[Intel SDE](https://www.intel.com/content/www/us/en/download/684897/intel-software-development-emulator.html)
+CPU models: Haswell (`-hsw`, AVX2 only) and Ice Lake (`-icl`, AVX-512 including
+VPOPCNTDQ). Kernel reference tests and small clustering/index tests exercise real
+runtime dispatch under emulation. `RABITQ_TEST_CPU` checks the emulated features
+and prints them; an incorrect CPU model fails instead of silently losing coverage.
+These are correctness checks, not hardware performance measurements.
+
+Source-install Python jobs check imports, index persistence, and both clustering
+APIs. Repaired-wheel jobs run the full core Python suite. Large-dimension Python
+cases check representative binding/persistence paths; C++ retains the exhaustive
+clustering dimension/storage matrix.
+
 ## Python tests
 
 Use an activated [project environment](../CONTRIBUTING.md#python-environment).
@@ -117,16 +138,20 @@ The [Python example tests](python/test_examples.py) cover RaBitQKMeans and
 QGKMeans clustering, L2/IP indexing, saved cluster files, and queries without
 importing FAISS. The [FAISS comparison tests](python/test_compare_with_faiss.py)
 cover both clustering methods and skip FAISS-dependent cases when `faiss-cpu`
-is absent; wheel CI includes it as a test dependency.
+is absent. CI runs them in a dedicated Linux job; wheel tests do not install FAISS.
 
 The [C++ clustering example tests](python/test_cpp_clustering_examples.py) check
 saved vectors, exact labels, and compatibility with the C++ index builders.
-Build the corresponding examples first; cases skip when their executables are absent.
+The Ubuntu C++ job runs them against its sample binaries, with
+`RABITQ_REQUIRE_CPP_EXAMPLES=1` so missing executables fail. Wheel jobs exclude
+these tests. Locally, build the examples first; missing executables otherwise skip.
 
 ## Installed CMake package test
 
 After building the library, verify that another project can consume its installed
-headers and library. Use an absolute path for `<install-prefix>`:
+headers and library. The consumer trains QGKMeans and RaBitQKMeans and checks
+final assignments, distances, and objective against a scalar L2 reference.
+Use an absolute path for `<install-prefix>`:
 
 ```text
 cmake --install build --config Release --prefix "<install-prefix>"

tests/consumer/main.cpp

@@ -1,6 +1,93 @@
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <iostream>
+#include <limits>
+#include <vector>
+
+#include "rabitqlib/clustering/qgkmeans.hpp"
+#include "rabitqlib/clustering/rabitqkmeans.hpp"
 #include "rabitqlib/utils/cpu_features.hpp"
 
+namespace {
+constexpr size_t kDimension = 65;
+constexpr size_t kPoints = 67;
+
+// Use public results and an independent scalar reference. Checks must also run
+// in Release builds, where assert() would be disabled.
+template <typename Model>
+bool check_clustering(Model& model, const std::vector<float>& data) {
+    model.train(kPoints, data.data());
+    if (model.centroids.size() != model.k * kDimension ||
+        model.assignments.size() != kPoints || model.distances.size() != kPoints ||
+        model.iteration_stats.empty()) {
+        return false;
+    }
+    for (float value : model.centroids) {
+        if (!std::isfinite(value)) {
+            return false;
+        }
+    }
+    double objective = 0;
+    for (size_t point = 0; point < kPoints; ++point) {
+        if (model.assignments[point] >= model.k) {
+            return false;
+        }
+        double nearest = std::numeric_limits<double>::infinity();
+        double selected = 0;
+        for (size_t cluster = 0; cluster < model.k; ++cluster) {
+            double distance = 0;
+            for (size_t dim = 0; dim < kDimension; ++dim) {
+                const double delta = static_cast<double>(data[point * kDimension + dim]) -
+                                     model.centroids[cluster * kDimension + dim];
+                distance += delta * delta;
+            }
+            nearest = std::min(nearest, distance);
+            if (cluster == model.assignments[point]) {
+                selected = distance;
+            }
+        }
+        const double tolerance = 1e-4 * std::max(1.0, nearest);
+        if (!(std::abs(selected - nearest) <= tolerance) ||
+            !(std::abs(model.distances[point] - selected) <= tolerance)) {
+            return false;
+        }
+        objective += selected;
+    }
+    return std::abs(model.final_obj - objective) <= 1e-4 * std::max(1.0, objective);
+}
+}  // namespace
+
 int main() {
     const auto& features = rabitqlib::cpu::features();
-    return features.avx512vpopcntdq && !features.avx512f ? 1 : 0;
+    if (features.avx512vpopcntdq && !features.avx512f) {
+        return 1;
+    }
+    std::vector<float> data(kPoints * kDimension);
+    for (size_t i = 0; i < data.size(); ++i) {
+        data[i] = std::sin(static_cast<float>(i) * 0.17F);
+    }
+
+    rabitqlib::qgkmeans::QGKMeansParameters graph_parameters;
+    graph_parameters.niter = 2;
+    graph_parameters.num_threads = 2;
+    graph_parameters.ef_build = 33;
+    graph_parameters.ef_search = 33;
+    graph_parameters.final_assignment = rabitqlib::clustering::FinalAssignmentMode::Exact;
+    rabitqlib::qgkmeans::QGKMeans graph(kDimension, 33, graph_parameters);
+    if (!check_clustering(graph, data)) {
+        std::cerr << "Installed QGKMeans returned invalid clustering results\n";
+        return 1;
+    }
+
+    rabitqlib::rabitqkmeans::RaBitQKMeansParameters flat_parameters;
+    flat_parameters.niter = 2;
+    flat_parameters.num_threads = 2;
+    flat_parameters.final_assignment = rabitqlib::clustering::FinalAssignmentMode::Exact;
+    rabitqlib::rabitqkmeans::RaBitQKMeans flat(kDimension, 4, flat_parameters);
+    if (!check_clustering(flat, data)) {
+        std::cerr << "Installed RaBitQKMeans returned invalid clustering results\n";
+        return 1;
+    }
+    return 0;
 }

tests/python/test_cpp_clustering_examples.py

@@ -13,7 +13,10 @@
 def run_cpp(name, *args):
     executable = BIN / (name + (".exe" if os.name == "nt" else ""))
     if not executable.is_file():
-        pytest.skip(f"Build the C++ example target {name} first")
+        message = f"Build the C++ example target {name} first"
+        if os.environ.get("RABITQ_REQUIRE_CPP_EXAMPLES") == "1":
+            pytest.fail(message)
+        pytest.skip(message)
     return subprocess.run(
         [str(executable), *map(str, args)],
         env=dict(os.environ, OMP_THREAD_LIMIT="2", OMP_NUM_THREADS="2"),
@@ -119,3 +122,14 @@ def test_cpp_clustering_usage_and_errors(tmp_path, method):
     result = run_cpp(method, *args)
     assert result.returncode == 1
     assert "File does not exist:" in result.stderr
+
+
+@pytest.mark.parametrize("required", [False, True])
+def test_missing_cpp_example_fails_when_required(tmp_path, monkeypatch, required):
+    monkeypatch.setitem(run_cpp.__globals__, "BIN", tmp_path)
+    monkeypatch.setenv("RABITQ_REQUIRE_CPP_EXAMPLES", "1" if required else "0")
+    expected = pytest.fail.Exception if required else pytest.skip.Exception
+    with pytest.raises(
+        expected, match=r"Build the C\+\+ example target qgkmeans first"
+    ):
+        run_cpp("qgkmeans")

tests/python/test_large_dimensions.py

@@ -5,7 +5,8 @@
 from rabitqlib import HnswIndex, IvfIndex, SymqgIndex
 
 
-@pytest.mark.parametrize("dim", [4096, 16384, 16385, 65536])
+# Keep a padded dimension and the maximum across every index and metric.
+@pytest.mark.parametrize("dim", [16385, 65536])
 @pytest.mark.parametrize("metric", ["l2", "ip"])
 @pytest.mark.parametrize("kind", ["ivf", "hnsw", "qg"])
 def test_large_dimension_round_trip(tmp_path, dim, metric, kind):

tests/python/test_qgkmeans.py

@@ -411,8 +411,8 @@ def test_training_size_warning_is_opt_in(capfd, verbose):
     assert ("please provide at least" in captured.err) == verbose
 
 
-@pytest.mark.parametrize("dim", [4096, 4097, 65535, 65536])
-@pytest.mark.parametrize("bits", [0, 4, 8])
+# C++ covers the full dimension/storage matrix; check binding outputs here.
+@pytest.mark.parametrize("dim,bits", [(4097, 0), (65535, 4), (65536, 8)])
 def test_large_dimensions(dim, bits):
     x = np.zeros((33, dim), dtype=np.float32)
     x[np.arange(33), np.arange(33)] = 1