diff --git a/.github/scripts/ci_changes.py b/.github/scripts/ci_changes.py index 8bcea765..92fe4526 100644 --- a/.github/scripts/ci_changes.py +++ b/.github/scripts/ci_changes.py @@ -126,6 +126,8 @@ def classify(paths): "tests/unit/", "tests/integration/", "tests/dispatch/", + "tests/allocation/", + "tests/failures/", "tests/common/", "tests/consumer/", "tests/core_consumer/", diff --git a/.github/scripts/test_ci_changes.py b/.github/scripts/test_ci_changes.py index f5b65a1a..82cae3d9 100644 --- a/.github/scripts/test_ci_changes.py +++ b/.github/scripts/test_ci_changes.py @@ -115,6 +115,8 @@ def test_language_specific_changes(self): cases = { "tests/unit/rabitqlib/index/ivf_test.cpp": {"cpp", "cpp_format"}, "tests/dispatch/fastscan_no_simd_test.cpp": {"cpp", "cpp_format"}, + "tests/allocation/core_api_no_alloc.cpp": {"cpp", "cpp_format"}, + "tests/failures/construction_failure_test.cpp": {"cpp", "cpp_format"}, "sample/cpp/ivf_rabitq_querying.cpp": {"cpp", "cpp_format"}, "python_bindings/ivf_bindings.cpp": { "benchmarks", diff --git a/.github/scripts/test_ci_workflow_contracts.py b/.github/scripts/test_ci_workflow_contracts.py index a147dfe1..5223163b 100644 --- a/.github/scripts/test_ci_workflow_contracts.py +++ b/.github/scripts/test_ci_workflow_contracts.py @@ -15,6 +15,74 @@ class WorkflowContractsTest(unittest.TestCase): + def test_format_checker_propagates_file_list_failures(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "scripts").mkdir() + (root / "bin").mkdir() + git = root / "bin/git" + git.write_text(f"#!{sys.executable}\nprint({str(root)!r})\n") + git.chmod(0o755) + formatter = root / "bin/formatter" + log = root / "formatted.json" + formatter.write_text( + f"#!{sys.executable}\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "if sys.argv[1:] == ['--version']:\n" + " print('clang-format version 23.1.4')\n" + "else:\n" + " Path(os.environ['FORMAT_LOG']).write_text(json.dumps(sys.argv[1:]))\n" + " sys.exit(int(os.environ.get('FORMAT_STATUS', '0')))\n" + ) + formatter.chmod(0o755) + env = dict( + os.environ, + PATH=str(root / "bin") + os.pathsep + os.environ["PATH"], + CLANG_FORMAT=str(formatter), + FORMAT_LOG=str(log), + ) + helper = root / "scripts/clang-format-files.sh" + filename = str(root / "space and\nnewline.cpp") + for mode, content, status, expected in ( + ("missing", None, "0", False), + ("failed", "sys.exit(7)\n", "0", False), + ( + "partial", + f"sys.stdout.write({filename!r} + '\\0')\nsys.exit(7)\n", + "0", + False, + ), + ("empty", "sys.exit(0)\n", "0", True), + ("valid", f"sys.stdout.write({filename!r} + '\\0')\n", "0", True), + ( + "format-error", + f"sys.stdout.write({filename!r} + '\\0')\n", + "1", + False, + ), + ): + with self.subTest(mode=mode): + if content is not None: + helper.write_text(f"#!{sys.executable}\nimport sys\n" + content) + helper.chmod(0o755) + log.unlink(missing_ok=True) + result = subprocess.run( + ["bash", str(ROOT / "scripts/check-format.sh")], + env=dict(env, FORMAT_STATUS=status), + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode == 0, expected, result.stderr) + if mode in ("valid", "format-error"): + self.assertEqual( + json.loads(log.read_text()), + ["--dry-run", "--Werror", filename], + ) + else: + self.assertFalse(log.exists()) + def test_benchmark_support_changes_keep_protocol_coverage_without_builds(self): for path in sorted((ROOT / "benchmarks").glob("*.py")): with self.subTest(path=path.name): diff --git a/.github/scripts/test_pr_labels.py b/.github/scripts/test_pr_labels.py index 08a3e266..9923a3a2 100644 --- a/.github/scripts/test_pr_labels.py +++ b/.github/scripts/test_pr_labels.py @@ -160,6 +160,8 @@ def test_api_failures_are_not_silenced(self): def test_workflow_uses_metadata_only_and_routes_to_regression_checks(self): text = WORKFLOW.read_text() self.assertIn("pull_request_target:", text) + self.assertIn(" pull-requests: write\n", text) + self.assertIn(" issues: write\n", text) self.assertIn("types: [opened, edited, reopened, ready_for_review]", text) self.assertNotIn("actions/checkout", text) self.assertNotIn("${{", SCRIPT) diff --git a/.github/workflows/pr-labels.yml b/.github/workflows/pr-labels.yml index 41415b30..b432357c 100644 --- a/.github/workflows/pr-labels.yml +++ b/.github/workflows/pr-labels.yml @@ -9,7 +9,8 @@ concurrency: cancel-in-progress: false permissions: - pull-requests: read + # PR label assignment/removal and repository label creation need separate scopes. + pull-requests: write issues: write jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eb3d2aba..02d77614 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -187,9 +187,14 @@ jobs: CC: clang-23 CXX: clang++-23 OMP_NUM_THREADS: "2" + CMAKE_CXX_COMPILER_LAUNCHER: ccache steps: - uses: actions/checkout@v7 - uses: ./.github/actions/setup-clang + - name: Install compiler cache + run: | + sudo apt-get install -y ccache + echo "CCACHE_DIR=$RUNNER_TEMP/ccache" >> "$GITHUB_ENV" - name: Configure portable build run: >- cmake -S . -B build-clang -G Ninja @@ -217,6 +222,9 @@ jobs: -DWORK_DIR="$RUNNER_TEMP/rabitq-clang-core" -DRABITQ_USE_OPENMP=${{ matrix.openmp }} -P tests/core_consumer/run.cmake + - name: Report compiler cache statistics + if: ${{ always() }} + run: ccache --show-stats clang-result: name: Clang validation @@ -370,9 +378,21 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v7 - - name: Install Apple OpenMP - if: runner.os == 'macOS' && matrix.openmp == 'ON' - run: brew install libomp + - name: Install Linux compiler cache + if: runner.os == 'Linux' + run: sudo apt-get update -qq && sudo apt-get install -y ccache + - name: Install Apple compiler cache and optional OpenMP + if: runner.os == 'macOS' + run: | + brew install ccache + if [ "${{ matrix.openmp }}" = ON ]; then + brew install libomp + fi + - name: Configure compiler cache + if: runner.os != 'Windows' + run: | + echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> "$GITHUB_ENV" + echo "CCACHE_DIR=$RUNNER_TEMP/ccache" >> "$GITHUB_ENV" - name: Test external core-only integrations (Linux) if: runner.os == 'Linux' run: >- @@ -398,6 +418,9 @@ jobs: "-DWORK_DIR=$env:RUNNER_TEMP/rabitq-core" -DRABITQ_USE_OPENMP=${{ matrix.openmp }} -P tests/core_consumer/run.cmake + - name: Report compiler cache statistics + if: ${{ always() && runner.os != 'Windows' }} + run: ccache --show-stats core-consumer-result: name: Core and FastScan validation diff --git a/docs/maintaining.md b/docs/maintaining.md index 0fd77d12..c6a74fa5 100644 --- a/docs/maintaining.md +++ b/docs/maintaining.md @@ -22,6 +22,7 @@ only the affected job groups: | `tests/core_consumer/`, `tests/fixtures/core_api_v1/` | C++ checks; the frozen v1 fixture is excluded from formatting | | C++ headers or library sources | C++ formatting, clang-tidy, platform tests, sanitizers, consumer build, and Python builds/wheels | | C++ tests or examples | C++ formatting and platform tests; no clang-tidy | +| `tests/allocation/`, `tests/failures/` | C++ formatting and platform tests, including sanitizer and allocation-failure coverage; no Python builds or wheels | | Compiled Python bindings | C++ formatting and clang-tidy, plus Python builds and wheel tests | | Python tests | Python lint, source-install smoke tests, and wheel tests | | `tests/python/test_cpp_clustering_examples.py` | Python lint and C++ checks; examples run in the Ubuntu job | @@ -46,6 +47,13 @@ Pushes compare the complete pushed range; pull requests compare against their base. Renames and deletions are included. Relevant jobs also run if change detection fails, so a detection error cannot silently waive a check. +Linux and macOS core/FastScan consumer jobs and the Clang matrix use a job-local +`ccache` through CMake's compiler launcher. Independent producer, installed-package, +`add_subdirectory`, and FetchContent builds and tests are retained; identical +compilations can reuse compiler output within the job. Cache statistics are printed +after testing. Compiler caches are not uploaded or shared between workflow runs. +Windows keeps its existing build path without a compiler cache. + Release tags and manual wheel builds always run the full wheel build. CI routing edits run the full relevant matrix on pull requests. After an already released version reaches `main`, a push containing routing edits avoids repeated diff --git a/scripts/check-format.sh b/scripts/check-format.sh index f196075d..0170fcc8 100755 --- a/scripts/check-format.sh +++ b/scripts/check-format.sh @@ -17,7 +17,14 @@ if [[ "$($formatter --version)" != *"version 23."* ]]; then exit 1 fi -mapfile -d '' files < <("$repo_root/scripts/clang-format-files.sh" "$@") +# Process substitution would discard the file-list helper's exit status. +file_list="$(mktemp)" +trap 'rm -f -- "$file_list"' EXIT +if ! "$repo_root/scripts/clang-format-files.sh" "$@" > "$file_list"; then + echo "error: could not list files for formatting" >&2 + exit 1 +fi +mapfile -d '' files < "$file_list" if printf '%s\n' "${files[@]}" \ | grep -q '/include/rabitqlib/third/'; then