ci: cache compilations and trim redundant CI and test work

.github/actions/save-ccache/action.yml

@@ -0,0 +1,16 @@
+name: Save ccache
+description: >-
+  Report the statistics of the cache set up by setup-ccache and save it on pushes to main.
+  Pull requests only restore, so they cannot evict the cache every pull request starts from.
+runs:
+  using: composite
+  steps:
+    - name: Report compiler cache statistics
+      shell: bash
+      run: ccache --show-stats
+    - name: Save compiler cache
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      uses: actions/cache/save@v6
+      with:
+        path: ${{ runner.temp }}/ccache
+        key: ${{ env.CCACHE_KEY_PREFIX }}${{ github.sha }}

.github/actions/setup-ccache/action.yml

@@ -0,0 +1,45 @@
+name: Set up ccache
+description: >-
+  Install ccache, route CMake compilations through it and restore the newest cache saved
+  from main on Linux and macOS. Pair it with save-ccache as the job's last step. Visual
+  Studio generators do not support compiler launchers.
+inputs:
+  key:
+    description: Cache partition for one compiler and build configuration.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Install ccache
+      shell: bash
+      env:
+        CACHE_PARTITION: ${{ inputs.key }}
+      run: |
+        set -euo pipefail
+        if ! command -v ccache >/dev/null 2>&1; then
+          if [[ "$RUNNER_OS" == Linux ]]; then
+            sudo apt-get update -qq
+            sudo apt-get install -y ccache
+          else
+            brew install ccache
+          fi
+        fi
+        ccache --version | head -n 1
+        {
+          echo "CCACHE_DIR=$RUNNER_TEMP/ccache"
+          echo "CCACHE_BASEDIR=$GITHUB_WORKSPACE"
+          echo "CCACHE_MAXSIZE=500M"
+          echo "CCACHE_KEY_PREFIX=ccache-$RUNNER_OS-$RUNNER_ARCH-$CACHE_PARTITION-"
+          echo "CMAKE_C_COMPILER_LAUNCHER=ccache"
+          echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache"
+        } >> "$GITHUB_ENV"
+    # Only main saves (see save-ccache); every run restores the newest matching partition.
+    - name: Restore compiler cache
+      uses: actions/cache/restore@v6
+      with:
+        path: ${{ runner.temp }}/ccache
+        key: ${{ env.CCACHE_KEY_PREFIX }}${{ github.sha }}
+        restore-keys: ${{ env.CCACHE_KEY_PREFIX }}
+    - name: Reset compiler cache statistics
+      shell: bash
+      run: ccache --zero-stats

.github/scripts/ci_changes.py

@@ -8,6 +8,7 @@
 
 GROUPS = (
     "cpp",
+    "cpp_core",
     "cpp_format",
     "cpp_tidy",
     "python_quality",
@@ -20,6 +21,34 @@
 
 CPP_SUFFIXES = (".c", ".cc", ".cpp", ".h", ".hpp")
 
+# Core and FastScan consumer builds exclude ANN indexes, clustering and their tests.
+# Anything not listed here keeps those consumer checks selected.
+INDEX_ONLY_PREFIXES = (
+    "include/rabitqlib/clustering/",
+    "include/rabitqlib/index/",
+    "include/rabitqlib/third/hnswlib/",
+    "src/index/",
+    "tests/consumer/",
+    "tests/integration/",
+    "tests/unit/rabitqlib/clustering/",
+    "tests/unit/rabitqlib/index/",
+)
+CORE_SAMPLE_PREFIXES = ("sample/cpp/core/", "sample/cpp/fastscan/")
+INDEX_ONLY_TESTS = frozenset(
+    (
+        "tests/failures/construction_failure_test.cpp",
+        "tests/failures/hnsw_add_allocation_test.cpp",
+    )
+)
+
+
+def selects_core(path):
+    """Whether a C++ change can affect the core and FastScan consumer builds."""
+    if path.startswith(INDEX_ONLY_PREFIXES) or path in INDEX_ONLY_TESTS:
+        return False
+    return not path.startswith("sample/cpp/") or path.startswith(CORE_SAMPLE_PREFIXES)
+
+
 BENCHMARK_PYTHON_PATHS = frozenset(
     f"benchmarks/{name}.py"
     for name in (
@@ -107,11 +136,13 @@ def classify(paths):
             selected.update(("python_quality", "python_build", "wheels"))
         elif path.startswith("tests/fixtures/core_api_v1/"):
             # Frozen compatibility fixture; excluded from clang-format by design.
-            selected.add("cpp")
+            selected.update(("cpp", "cpp_core"))
         elif path == "pyproject.toml":
             selected.update(("python_quality", "python_build", "wheels", "benchmarks"))
         elif path == "CMakeLists.txt" or path.startswith("cmake/"):
-            selected.update(("cpp", "cpp_tidy", "python_build", "wheels", "benchmarks"))
+            selected.update(
+                ("cpp", "cpp_core", "cpp_tidy", "python_build", "wheels", "benchmarks")
+            )
         elif path.startswith(("include/", "src/")):
             selected.update(
                 (
@@ -123,6 +154,8 @@ def classify(paths):
                     "benchmarks",
                 )
             )
+            if selects_core(path):
+                selected.add("cpp_core")
         elif path.startswith("python_bindings/"):
             selected.update(("python_build", "wheels", "benchmarks"))
             if path.endswith((".py", ".pyi")):
@@ -146,6 +179,8 @@ def classify(paths):
             or path == "tests/CMakeLists.txt"
         ):
             selected.add("cpp")
+            if selects_core(path):
+                selected.add("cpp_core")
             if path.endswith(CPP_SUFFIXES):
                 selected.add("cpp_format")
         elif path == "tests/python/test_cpp_clustering_examples.py":
@@ -175,6 +210,7 @@ def classify(paths):
             selected.update(
                 (
                     "cpp",
+                    "cpp_core",
                     "cpp_format",
                     "cpp_tidy",
                     "shell",

.github/scripts/test_ci_aggregates.py

@@ -1,24 +1,21 @@
 """Execute the matrix verdict commands with success, skip and failure inputs."""
 
 import os
-import re
 import subprocess
-import textwrap
 import unittest
-from pathlib import Path
 
-WORKFLOWS = Path(__file__).resolve().parents[1] / "workflows"
+from workflow_data import job, load_workflow, needs, steps
 
 
 class MatrixSummaryTest(unittest.TestCase):
     def test_matrix_verdicts_and_dependencies(self):
-        for filename, job, dependency, output, selection_var in (
+        for filename, job_name, dependency, output, selection_var in (
             ("test.yaml", "clang-result", "clang", "cpp", "CPP_SELECTED"),
             (
                 "test.yaml",
                 "core-consumer-result",
                 "core-consumer",
-                "cpp",
+                "cpp_core",
                 "CPP_SELECTED",
             ),
             (
@@ -30,20 +27,19 @@ def test_matrix_verdicts_and_dependencies(self):
             ),
             ("docs.yml", "docs-result", "build", "docs", "DOCS_SELECTED"),
         ):
-            workflow = (WORKFLOWS / filename).read_text()
-            block = re.search(
-                rf"^  {job}:\n(.*?)(?=^  [\w-]+:|\Z)",
-                workflow,
-                re.MULTILINE | re.DOTALL,
-            ).group(1)
-            self.assertIn(f"needs: [changes, {dependency}]", block)
-            self.assertIn("if: ${{ always() }}", block)
-            self.assertIn(f"MATRIX_RESULT: ${{{{ needs.{dependency}.result }}}}", block)
-            self.assertIn("CHANGES_RESULT: ${{ needs.changes.result }}", block)
-            self.assertIn(
-                f"{selection_var}: ${{{{ needs.changes.outputs.{output} }}}}", block
+            summary = job(load_workflow(filename), job_name)
+            self.assertEqual(needs(summary), ["changes", dependency])
+            self.assertEqual(summary["if"], "${{ always() }}")
+            (verdict,) = steps(summary)
+            self.assertEqual(
+                verdict["env"],
+                {
+                    "CHANGES_RESULT": "${{ needs.changes.result }}",
+                    selection_var: f"${{{{ needs.changes.outputs.{output} }}}}",
+                    "MATRIX_RESULT": f"${{{{ needs.{dependency}.result }}}}",
+                },
             )
-            command = textwrap.dedent(block.split("        run: |\n", 1)[1])
+            command = verdict["run"]
             for detection in ("success", "failure", "cancelled", "skipped"):
                 for selected in ("true", "false", ""):
                     for matrix in ("success", "failure", "cancelled", "skipped", ""):
@@ -53,7 +49,7 @@ def test_matrix_verdicts_and_dependencies(self):
                             and matrix == "skipped"
                         )
                         with self.subTest(
-                            job=job,
+                            job=job_name,
                             detection=detection,
                             selected=selected,
                             matrix=matrix,

.github/scripts/test_ci_changes.py

@@ -12,6 +12,7 @@ def test_benchmark_and_workflow_self_coverage(self):
         cases = {
             ".github/workflows/test.yaml": {
                 "cpp",
+                "cpp_core",
                 "cpp_format",
                 "cpp_tidy",
                 "shell",
@@ -73,8 +74,8 @@ def test_known_support_paths_do_not_select_unrelated_builds(self):
             "example_py.sh": {"shell"},
             "python/ivf.py": {"python_quality", "python_build", "wheels"},
             "python/utils/io.py": {"python_quality", "python_build", "wheels"},
-            "tests/core_consumer/run.cmake": {"cpp", "python_quality"},
-            "tests/fixtures/core_api_v1/consumer.cpp": {"cpp"},
+            "tests/core_consumer/run.cmake": {"cpp", "cpp_core", "python_quality"},
+            "tests/fixtures/core_api_v1/consumer.cpp": {"cpp", "cpp_core"},
             ".github/scripts/repair_windows_wheel.py": {"python_quality", "wheels"},
             ".github/scripts/test_repair_windows_wheel.py": {
                 "python_quality",
@@ -92,6 +93,7 @@ def test_support_paths_do_not_mask_source_or_unknown_changes(self):
             {
                 "benchmarks",
                 "cpp",
+                "cpp_core",
                 "cpp_format",
                 "cpp_tidy",
                 "python_build",
@@ -103,25 +105,70 @@ def test_support_paths_do_not_mask_source_or_unknown_changes(self):
         self.assertEqual(classify(["benchmarks/new_kernel.cpp"]), set(GROUPS))
 
     def test_shared_code_runs_cpp_and_python(self):
-        for path in ("include/rabitqlib/index/ivf/ivf.hpp", "src/simd/dispatch.cpp"):
+        library = {
+            "cpp",
+            "cpp_format",
+            "cpp_tidy",
+            "python_build",
+            "wheels",
+            "benchmarks",
+        }
+        for path, core in (
+            ("include/rabitqlib/index/ivf/ivf.hpp", False),
+            ("src/simd/dispatch.cpp", True),
+        ):
             with self.subTest(path=path):
                 self.assertEqual(
-                    classify([path]),
-                    {
-                        "cpp",
-                        "cpp_format",
-                        "cpp_tidy",
-                        "python_build",
-                        "wheels",
-                        "benchmarks",
-                    },
+                    classify([path]), library | ({"cpp_core"} if core else set())
                 )
 
+    def test_index_only_cpp_changes_skip_core_consumers(self):
+        for path in (
+            "src/index/ivf.cpp",
+            "include/rabitqlib/index/hnsw/hnsw.hpp",
+            "include/rabitqlib/clustering/qgkmeans.hpp",
+            "include/rabitqlib/third/hnswlib/hnswalg.h",
+            "tests/unit/rabitqlib/index/qg_test.cpp",
+            "tests/unit/rabitqlib/clustering/qgkmeans_test.cpp",
+            "tests/failures/construction_failure_test.cpp",
+            "tests/failures/hnsw_add_allocation_test.cpp",
+            "tests/consumer/main.cpp",
+            "tests/integration/cpp_example_index_check.cpp",
+            "sample/cpp/ivf_rabitq_querying.cpp",
+        ):
+            with self.subTest(path=path):
+                selected = classify([path])
+                self.assertIn("cpp", selected)
+                self.assertNotIn("cpp_core", selected)
+        for path in (
+            "src/core/api.cpp",
+            "src/fastscan/fastscan_avx2.cpp",
+            "include/rabitqlib/core/api.hpp",
+            "include/rabitqlib/utils/space.hpp",
+            "include/rabitqlib/third/Eigen/Core",
+            "tests/unit/rabitqlib/quantization/core_api_test.cpp",
+            "tests/unit/rabitqlib/utils/thread_count_test.cpp",
+            "tests/common/temp_file.hpp",
+            "tests/failures/compact_codes_failure_test.cpp",
+            "tests/CMakeLists.txt",
+            "tests/core_consumer/CMakeLists.txt",
+            "sample/cpp/core/main.cpp",
+            "sample/cpp/fastscan/main.cpp",
+            "CMakeLists.txt",
+        ):
+            with self.subTest(path=path):
+                self.assertLessEqual({"cpp", "cpp_core"}, classify([path]))
+        self.assertIn("cpp_core", classify(["src/index/ivf.cpp", "src/core/api.cpp"]))
+
     def test_language_specific_changes(self):
         cases = {
             "tests/unit/rabitqlib/index/ivf_test.cpp": {"cpp", "cpp_format"},
-            "tests/dispatch/fastscan_no_simd_test.cpp": {"cpp", "cpp_format"},
-            "tests/allocation/core_api_no_alloc.cpp": {"cpp", "cpp_format"},
+            "tests/dispatch/fastscan_no_simd_test.cpp": {
+                "cpp",
+                "cpp_core",
+                "cpp_format",
+            },
+            "tests/allocation/core_api_no_alloc.cpp": {"cpp", "cpp_core", "cpp_format"},
             "tests/failures/construction_failure_test.cpp": {"cpp", "cpp_format"},
             "sample/cpp/ivf_rabitq_querying.cpp": {"cpp", "cpp_format"},
             "python_bindings/ivf_bindings.cpp": {
@@ -179,6 +226,7 @@ def test_language_specific_changes(self):
             ".clang-tidy": {"cpp_tidy"},
             ".github/workflows/test.yaml": {
                 "cpp",
+                "cpp_core",
                 "cpp_format",
                 "cpp_tidy",
                 "python_quality",
@@ -199,14 +247,28 @@ def test_build_configuration_selects_affected_checks(self):
             with self.subTest(path=path):
                 self.assertEqual(
                     classify([path]),
-                    {"cpp", "cpp_tidy", "python_build", "wheels", "benchmarks"},
+                    {
+                        "cpp",
+                        "cpp_core",
+                        "cpp_tidy",
+                        "python_build",
+                        "wheels",
+                        "benchmarks",
+                    },
                 )
 
     def test_mixed_changes_and_renamed_source(self):
         self.assertIn("cpp", classify(["docs/new.md", "src/deleted.cpp"]))
         self.assertEqual(
             classify(["sample/python/tool.py", "tests/unit/example.cpp"]),
-            {"cpp", "cpp_format", "python_quality", "python_build", "wheels"},
+            {
+                "cpp",
+                "cpp_core",
+                "cpp_format",
+                "python_quality",
+                "python_build",
+                "wheels",
+            },
         )
 
     def test_unknown_inputs_and_filter_edits_run_everything(self):