From e8ae1862294dfbac2cce134aa03dbb4e1e25cbf4 Mon Sep 17 00:00:00 2001 From: gouyt13clear Date: Wed, 21 Oct 2026 22:08:09 +0800 Subject: [PATCH] ci: compile benchmark smoke targets and validate workflow edits --- .github/scripts/ci_changes.py | 37 ++++++++++++--- .github/scripts/test_ci_changes.py | 72 +++++++++++++++++++++++++++--- .github/workflows/changes.yml | 3 ++ .github/workflows/test.yaml | 34 ++++++++++++++ benchmarks/core/README.md | 4 +- benchmarks/hotpaths/README.md | 2 + docs/development.md | 4 +- docs/maintaining.md | 16 +++++++ scripts/check-python.sh | 2 +- 9 files changed, 157 insertions(+), 17 deletions(-) diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 1160eb5f..007e965f 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -14,6 +14,7 @@ "python_build", "shell", "wheels", + "benchmarks", ) CPP_SUFFIXES = (".c", ".cc", ".cpp", ".h", ".hpp") @@ -49,7 +50,13 @@ def classify(paths): ".editorconfig", ): return set(GROUPS) - if path == "benchmarks/plot_homepage.py": + if path.startswith(("benchmarks/core/", "benchmarks/hotpaths/")): + selected.add("benchmarks") + if path.endswith(CPP_SUFFIXES): + selected.add("cpp_format") + elif path.endswith(".py"): + selected.add("python_quality") + elif path == "benchmarks/plot_homepage.py": selected.add("python_quality") elif path in ( "sample/run_cpp.sh", @@ -67,13 +74,22 @@ def classify(paths): # Frozen compatibility fixture; excluded from clang-format by design. selected.add("cpp") elif path == "pyproject.toml": - selected.update(("python_quality", "python_build", "wheels")) + selected.update(("python_quality", "python_build", "wheels", "benchmarks")) elif path == "CMakeLists.txt" or path.startswith("cmake/"): - selected.update(("cpp", "cpp_tidy", "python_build", "wheels")) + selected.update(("cpp", "cpp_tidy", "python_build", "wheels", "benchmarks")) elif path.startswith(("include/", "src/")): - selected.update(("cpp", "cpp_format", "cpp_tidy", "python_build", "wheels")) + selected.update( + ( + "cpp", + "cpp_format", + "cpp_tidy", + "python_build", + "wheels", + "benchmarks", + ) + ) elif path.startswith("python_bindings/"): - selected.update(("python_build", "wheels")) + selected.update(("python_build", "wheels", "benchmarks")) if path.endswith(".py"): selected.add("python_quality") elif path.endswith(CPP_SUFFIXES): @@ -108,7 +124,16 @@ def classify(paths): elif path.startswith(".github/scripts/"): selected.add("python_quality") elif path == ".github/workflows/test.yaml": - selected.update(("cpp", "cpp_format", "cpp_tidy")) + selected.update( + ( + "cpp", + "cpp_format", + "cpp_tidy", + "shell", + "python_quality", + "benchmarks", + ) + ) elif path == ".github/workflows/python.yml": selected.update(("python_quality", "python_build")) elif path == ".github/workflows/release.yml": diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index 3e7506e0..eff50074 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -8,6 +8,39 @@ class ChangeSelectionTest(unittest.TestCase): + def test_benchmark_and_workflow_self_coverage(self): + cases = { + ".github/workflows/test.yaml": { + "cpp", + "cpp_format", + "cpp_tidy", + "shell", + "python_quality", + "benchmarks", + }, + "benchmarks/core/main.cpp": {"benchmarks", "cpp_format"}, + "benchmarks/core/CMakeLists.txt": {"benchmarks"}, + "benchmarks/hotpaths/fastscan.cpp": {"benchmarks", "cpp_format"}, + "benchmarks/hotpaths/common.hpp": {"benchmarks", "cpp_format"}, + "benchmarks/hotpaths/ids.py": {"benchmarks", "python_quality"}, + "benchmarks/hotpaths/compare.py": {"benchmarks", "python_quality"}, + } + for path, expected in cases.items(): + with self.subTest(path=path): + self.assertEqual(classify([path]), expected) + for path in ( + "include/rabitqlib/core/api.hpp", + "src/core/api.cpp", + "python_bindings/bindings_common.hpp", + "pyproject.toml", + "CMakeLists.txt", + "cmake/rabitqOpenMP.cmake", + ): + with self.subTest(dependency=path): + self.assertIn("benchmarks", classify([path])) + self.assertEqual(classify(["benchmarks/core/README.md"]), set()) + self.assertEqual(classify(["benchmarks/new_kernel.cpp"]), set(GROUPS)) + def test_docs_only_skip_builds(self): self.assertEqual( classify( @@ -48,6 +81,7 @@ def test_support_paths_do_not_mask_source_or_unknown_changes(self): self.assertEqual( classify(support + ["src/core/api.cpp"]), { + "benchmarks", "cpp", "cpp_format", "cpp_tidy", @@ -64,7 +98,14 @@ def test_shared_code_runs_cpp_and_python(self): with self.subTest(path=path): self.assertEqual( classify([path]), - {"cpp", "cpp_format", "cpp_tidy", "python_build", "wheels"}, + { + "cpp", + "cpp_format", + "cpp_tidy", + "python_build", + "wheels", + "benchmarks", + }, ) def test_language_specific_changes(self): @@ -73,13 +114,23 @@ def test_language_specific_changes(self): "tests/dispatch/fastscan_no_simd_test.cpp": {"cpp", "cpp_format"}, "sample/cpp/ivf_rabitq_querying.cpp": {"cpp", "cpp_format"}, "python_bindings/ivf_bindings.cpp": { + "benchmarks", "cpp_format", "cpp_tidy", "python_build", "wheels", }, - "python_bindings/__init__.py": {"python_quality", "python_build", "wheels"}, - "python_bindings/licenses/LLVM-OpenMP.txt": {"python_build", "wheels"}, + "python_bindings/__init__.py": { + "python_quality", + "python_build", + "wheels", + "benchmarks", + }, + "python_bindings/licenses/LLVM-OpenMP.txt": { + "python_build", + "wheels", + "benchmarks", + }, "tests/python/test_cpp_clustering_examples.py": {"cpp", "python_quality"}, "tests/python/conftest.py": { "cpp", @@ -115,7 +166,14 @@ def test_language_specific_changes(self): "scripts/check-includes.sh": {"shell"}, ".clang-format": {"cpp_format"}, ".clang-tidy": {"cpp_tidy"}, - ".github/workflows/test.yaml": {"cpp", "cpp_format", "cpp_tidy"}, + ".github/workflows/test.yaml": { + "cpp", + "cpp_format", + "cpp_tidy", + "python_quality", + "shell", + "benchmarks", + }, } for path, expected in cases.items(): with self.subTest(path=path): @@ -124,12 +182,13 @@ def test_language_specific_changes(self): def test_build_configuration_selects_affected_checks(self): self.assertEqual( classify(["pyproject.toml"]), - {"python_quality", "python_build", "wheels"}, + {"python_quality", "python_build", "wheels", "benchmarks"}, ) for path in ("CMakeLists.txt", "cmake/RaBitQLibConfig.cmake.in"): with self.subTest(path=path): self.assertEqual( - classify([path]), {"cpp", "cpp_tidy", "python_build", "wheels"} + classify([path]), + {"cpp", "cpp_tidy", "python_build", "wheels", "benchmarks"}, ) def test_mixed_changes_and_renamed_source(self): @@ -237,6 +296,7 @@ def test_tagged_main_routing_changes_preserve_lightweight_companion_checks(self) (["python_bindings/__init__.py"], set(GROUPS)), (["scripts/build-macos-openmp.sh"], set(GROUPS)), (["scripts/check-tidy.sh"], set(GROUPS)), + (["benchmarks/core/main.cpp"], set(GROUPS)), (["CMakeLists.txt"], set(GROUPS)), (["pyproject.toml"], set(GROUPS)), (["unknown-build-input"], set(GROUPS)), diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index ea98d24e..6d18267e 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -3,6 +3,8 @@ name: Changed files on: workflow_call: outputs: + benchmarks: + value: ${{ jobs.detect.outputs.benchmarks }} cpp: value: ${{ jobs.detect.outputs.cpp }} cpp_format: @@ -26,6 +28,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 outputs: + benchmarks: ${{ steps.paths.outputs.benchmarks }} cpp: ${{ steps.paths.outputs.cpp }} cpp_format: ${{ steps.paths.outputs.cpp_format }} cpp_tidy: ${{ steps.paths.outputs.cpp_tidy }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 118f0a99..616d9c01 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -95,6 +95,40 @@ jobs: done shellcheck "${files[@]}" + benchmark-smoke: + name: Benchmark build and smoke + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.benchmarks == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + OMP_NUM_THREADS: "2" + OPENBLAS_NUM_THREADS: "1" + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.11" + - name: Install build dependencies + run: | + sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build + python -m pip install "numpy>=1.23" "pybind11>=2.12" + - name: Configure benchmark targets only + run: >- + cmake -S . -B build-benchmarks -G Ninja + -DCMAKE_BUILD_TYPE=Release + -DRABITQ_BUILD_BENCHMARKS=ON + -DRABITQ_BUILD_TESTS=OFF + -DRABITQ_BUILD_SAMPLES=OFF + -DRABITQ_BUILD_PYTHON_BINDINGS=ON + -DRABITQ_ENABLE_NATIVE_OPTIMIZATION=OFF + -DPython_EXECUTABLE="$(python -c 'import sys; print(sys.executable)')" + -Dpybind11_DIR="$(python -m pybind11 --cmakedir)" + - name: Compile benchmarks and dependencies + run: cmake --build build-benchmarks --parallel 2 + - name: Run bounded benchmark smoke tests + run: ctest --test-dir build-benchmarks --output-on-failure --no-tests=error --timeout 60 + ubuntu: needs: changes if: ${{ !cancelled() && (needs.changes.result != 'success' || needs.changes.outputs.cpp == 'true') }} diff --git a/benchmarks/core/README.md b/benchmarks/core/README.md index efa244cf..9f460873 100644 --- a/benchmarks/core/README.md +++ b/benchmarks/core/README.md @@ -56,8 +56,8 @@ For comparisons, retain all raw repetitions and record CPU model, compiler/versi build flags, affinity/SMT policy, OS and dataset parameters beside the CSV. Keep these fixed and alternate baseline/candidate runs; compare quality before claiming a speedup. Linux users can pin the executable with `taskset -c `. The program reports available -ISA features, not a trace of every selected kernel. CI only smoke-tests functionality; -shared CI timing is not a performance gate. +ISA features, not a trace of every selected kernel. The Ubuntu CI `Benchmark build and smoke` job compiles these targets and runs +the registered small cases. Shared CI timing is not a performance gate. ## Scoring-call overhead diff --git a/benchmarks/hotpaths/README.md b/benchmarks/hotpaths/README.md index 84a4f0fe..cee2a754 100644 --- a/benchmarks/hotpaths/README.md +++ b/benchmarks/hotpaths/README.md @@ -17,6 +17,8 @@ No additional production dependencies are introduced. Core/FastScan benchmarks s OpenMP OFF. Python benchmarks use the existing Python/NumPy/pybind11 environment and are available only with `RABITQ_BUILD_PYTHON_BINDINGS=ON`. The helper module is not installed in the package. `ctest` smoke tests verify functionality, never shared-host speed thresholds. +The CI `Benchmark build and smoke` job builds these targets on Ubuntu, including +the Python helper, and runs the registered small cases without external datasets. ## Build and run diff --git a/docs/development.md b/docs/development.md index ae66bdac..3980a863 100644 --- a/docs/development.md +++ b/docs/development.md @@ -234,8 +234,8 @@ To apply Python formatting and safe automatic lint fixes before running the check: ```bash -ruff check --fix python_bindings sample/python tests/python benchmarks/plot_homepage.py -ruff format python_bindings sample/python tests/python benchmarks/plot_homepage.py +ruff check --fix python_bindings sample/python tests/python benchmarks/hotpaths benchmarks/plot_homepage.py +ruff format python_bindings sample/python tests/python benchmarks/hotpaths benchmarks/plot_homepage.py ``` ## Shell scripts diff --git a/docs/maintaining.md b/docs/maintaining.md index f18d00b6..39889a1f 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -12,6 +12,8 @@ only the affected job groups: | --- | --- | | Documentation and Markdown only | Documentation workflow for `docs/`; no C++ or Python builds | | Issue forms/configuration and CODEOWNERS | No C++/Python builds | +| `benchmarks/core/`, `benchmarks/hotpaths/` code/build files | Ubuntu benchmark build and bounded smoke tests; C++ formatting or Python lint as applicable, without the wheel/platform matrix for benchmark-only edits | +| `.github/workflows/test.yaml` | C++ checks, ShellCheck, Python quality (including summary-job tests), and benchmark smoke | | `benchmarks/plot_homepage.py` | Python lint, including the plotting script; no index or wheel builds | | `sample/run_cpp.sh`, `sample/run_python.sh` | ShellCheck, including these runners | | `tests/core_consumer/`, `tests/fixtures/core_api_v1/` | C++ checks; the frozen v1 fixture is excluded from formatting | @@ -52,6 +54,20 @@ routing plus library code, build configuration, wheel-related inputs or unknown paths keeps the full selection. PR routing checks remain broad. An untagged version on `main` still forces full C++ and Python CI, even for a subsequent docs-only commit, before automatic publishing can proceed. +## Benchmark smoke coverage + +`Benchmark build and smoke` uses one Ubuntu runner with benchmarks enabled, +ordinary unit tests and samples disabled, native optimization disabled, and Python +bindings enabled for the ID-conversion helper. It builds the targets and runs +only their registered small CTest cases, with a 60-second per-test timeout. +No dataset download, timing threshold, or performance comparison is run. + +The job also runs for shared library sources/headers, root/CMake build settings, +Python bindings/package configuration, and changes to the Test workflow. Other +unrecognized benchmark paths retain conservative full selection. The hot-path +Python scripts are included in Python lint. This is Linux build/functionality +coverage, not a performance result or proof of every SIMD backend executing. + ## Matrix summary checks `Clang validation` and `Core and FastScan validation` provide stable check names diff --git a/scripts/check-python.sh b/scripts/check-python.sh index eb71fa52..f278c22e 100755 --- a/scripts/check-python.sh +++ b/scripts/check-python.sh @@ -18,7 +18,7 @@ if [[ "$($ruff --version)" != "ruff 0.16.1" ]]; then fi cd "$repo_root" -paths=(python_bindings sample/python tests/python) +paths=(python_bindings sample/python tests/python benchmarks/hotpaths) if [[ -f benchmarks/plot_homepage.py ]]; then paths+=(benchmarks/plot_homepage.py) fi