From 29700abf26234b22a11b04b4495692f1f9d60339 Mon Sep 17 00:00:00 2001 From: gouyt13clear Date: Fri, 23 Oct 2026 18:56:32 +0800 Subject: [PATCH] ci: reduce setup overhead and parallelize Windows compilation --- .github/actions/setup-clang/action.yml | 20 +++++- .github/scripts/test_setup_clang.py | 99 ++++++++++++++++++++++++++ .github/workflows/python.yml | 8 +-- .github/workflows/test.yaml | 27 +++++-- 4 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 .github/scripts/test_setup_clang.py diff --git a/.github/actions/setup-clang/action.yml b/.github/actions/setup-clang/action.yml index 8e7bed18..7e071a3d 100644 --- a/.github/actions/setup-clang/action.yml +++ b/.github/actions/setup-clang/action.yml @@ -1,10 +1,16 @@ name: Set up Clang 23 description: Install the pinned compiler, analysis, formatting and OpenMP tools on Ubuntu. +inputs: + format-only: + description: Install only clang-format for formatting checks. + default: "false" runs: using: composite steps: - name: Install LLVM 23 tools shell: bash + env: + FORMAT_ONLY: ${{ inputs.format-only }} run: | set -euo pipefail # shellcheck source=/dev/null @@ -14,7 +20,15 @@ runs: echo "deb [signed-by=/usr/share/keyrings/apt.llvm.org.asc] https://apt.llvm.org/$VERSION_CODENAME/ llvm-toolchain-$VERSION_CODENAME-23 main" \ | sudo tee /etc/apt/sources.list.d/llvm-23.list >/dev/null sudo apt-get update -qq - sudo apt-get install -y clang-23 clang-format-23 clang-tidy-23 libomp-23-dev cmake ninja-build - clang++-23 --version + packages=(clang-format-23) + if [[ "$FORMAT_ONLY" != true ]]; then + packages+=(clang-23 clang-tidy-23 libomp-23-dev) + fi + sudo apt-get install -y "${packages[@]}" clang-format-23 --version - clang-tidy-23 --version + if [[ "$FORMAT_ONLY" != true ]]; then + clang++-23 --version + clang-tidy-23 --version + cmake --version + ninja --version + fi diff --git a/.github/scripts/test_setup_clang.py b/.github/scripts/test_setup_clang.py new file mode 100644 index 00000000..b402c7c3 --- /dev/null +++ b/.github/scripts/test_setup_clang.py @@ -0,0 +1,99 @@ +"""Exercise LLVM package selection and failure propagation without installing tools.""" + +import json +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class ClangSetupTest(unittest.TestCase): + def test_selected_tools_and_install_failures(self): + action = (ROOT / ".github/actions/setup-clang/action.yml").read_text() + self.assertIn('default: "false"', action) + self.assertIn("FORMAT_ONLY: ${{ inputs.format-only }}", action) + command = textwrap.dedent(action.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "commands.jsonl" + for tool in ( + "sudo", + "curl", + "clang-format-23", + "clang++-23", + "clang-tidy-23", + "cmake", + "ninja", + ): + helper = root / tool + helper.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "name = Path(sys.argv[0]).name\n" + "with open(os.environ['SETUP_LOG'], 'a') as stream:\n" + " stream.write(json.dumps([name, *sys.argv[1:]]) + '\\n')\n" + "if name == os.environ.get('FAIL_TOOL'):\n" + " sys.exit(7)\n" + "if os.environ.get('FAIL_TOOL') == 'install' and " + "sys.argv[1:3] == ['apt-get', 'install']:\n" + " sys.exit(7)\n" + "if name == 'curl':\n" + " print('test-key')\n" + "if name == 'sudo' and sys.argv[1] == 'tee':\n" + " sys.stdin.read()\n" + ) + helper.chmod(0o755) + for mode, failure in ( + ("false", ""), + ("true", ""), + ("false", "install"), + ("true", "install"), + ("true", "clang-format-23"), + ("false", "clang-tidy-23"), + ): + with self.subTest(format_only=mode, failure=failure): + log.unlink(missing_ok=True) + result = subprocess.run( + ["bash", "-c", command], + env=dict( + os.environ, + PATH=str(root) + os.pathsep + os.environ["PATH"], + FORMAT_ONLY=mode, + SETUP_LOG=str(log), + FAIL_TOOL=failure, + ), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode == 0, not failure) + calls = [json.loads(line) for line in log.read_text().splitlines()] + if failure: + self.assertFalse(any(call[0] == "ninja" for call in calls)) + continue + packages = next( + call[4:] + for call in calls + if call[:4] == ["sudo", "apt-get", "install", "-y"] + ) + expected = {"clang-format-23"} + if mode == "false": + expected.update(("clang-23", "clang-tidy-23", "libomp-23-dev")) + self.assertEqual(set(packages), expected) + versions = {call[0] for call in calls if call[1:] == ["--version"]} + expected = {"clang-format-23"} + if mode == "false": + expected.update( + ("clang++-23", "clang-tidy-23", "cmake", "ninja") + ) + self.assertEqual(versions, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ab9a9cf6..d65bd7b7 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -59,8 +59,8 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - - name: Install system dependencies - run: sudo apt-get update -qq && sudo apt-get install -y cmake + - name: Verify runner CMake + run: cmake --version - name: Upgrade pip run: pip install --upgrade pip @@ -136,8 +136,8 @@ jobs: with: python-version: "3.11" cache: pip - - name: Install system dependencies - run: sudo apt-get update -qq && sudo apt-get install -y cmake + - name: Verify runner CMake + run: cmake --version - name: Install comparison dependencies run: python -m pip install ".[test]" faiss-cpu - name: Test comparison script diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 02d77614..072ab591 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: ./.github/actions/setup-clang + with: + format-only: "true" - name: Check formatting run: ./scripts/check-format.sh @@ -111,7 +113,8 @@ jobs: python-version: "3.11" - name: Install build dependencies run: | - sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build + cmake --version + ninja --version python -m pip install "numpy>=1.23" "pybind11>=2.12" - name: Configure benchmark targets only run: >- @@ -140,8 +143,10 @@ jobs: - uses: actions/setup-python@v7 with: python-version: "3.11" - - name: Install dependencies - run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build + - name: Verify runner build tools + run: | + cmake --version + ninja --version - name: Configure run: cmake -S . -B build -G Ninja -DRABITQ_BUILD_TESTS=ON -DRABITQ_BUILD_SAMPLES=ON -DCMAKE_BUILD_TYPE=Release - name: Build @@ -251,6 +256,9 @@ jobs: name: Windows x86-64 (VS 2026) runs-on: windows-2025-vs2026 timeout-minutes: 45 + env: + # MSBuild's project parallelism alone does not parallelize source files. + CL: /MP2 steps: - uses: actions/checkout@v7 - name: Configure @@ -278,8 +286,10 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v7 - - name: Install dependencies - run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build + - name: Verify runner build tools + run: | + cmake --version + ninja --version # Keep assertions/debug symbols while avoiding unoptimized sanitizer loops. - name: Configure run: >- @@ -317,8 +327,10 @@ jobs: timeout-minutes: 15 steps: - uses: actions/checkout@v7 - - name: Install dependencies - run: sudo apt-get update -qq && sudo apt-get install -y cmake ninja-build + - name: Verify runner build tools + run: | + cmake --version + ninja --version - name: Build RaBitQ-Library with portable tests run: | cmake -S . -B build-install -G Ninja \ @@ -413,6 +425,7 @@ jobs: env: CMAKE_GENERATOR: Visual Studio 18 2026 CMAKE_GENERATOR_PLATFORM: x64 + CL: /MP2 run: >- cmake "-DRABITQ_SOURCE_DIR=$PWD" "-DWORK_DIR=$env:RUNNER_TEMP/rabitq-core"