CI: LLVM source-based C/C++ coverage on Linux pytest

.github/workflows/build_ubuntu-24.04.sh

@@ -30,6 +30,49 @@ set -u
 
 export INSTALL_PREFIX=$1
 
+# Extra configure args added under coverage; keep declared under `set -u`.
+OPENMP_ARGS=()
+
+# LLVM source-based coverage instrumentation, opt-in via env var. Requires
+# clang (installed only when opted in, see pytest.yml). Kept out of the
+# default O2 path so non-coverage runs are unaffected.
+if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ]; then
+    export CC="${CC:-clang}"
+    export CXX="${CXX:-clang++}"
+    # LLVM source-based coverage generates its region mapping before the
+    # optimizer runs, so optimization does not affect coverage quality
+    # (clang doc: SourceBasedCodeCoverage, "Impact of llvm optimizations
+    # on coverage reports"). Keep the project's normal -O2 to hold the
+    # runtime impact small. -fprofile-update=atomic makes counter
+    # increments atomic; required for GRASS's OpenMP-parallel regions
+    # (e.g. r.univar) so threads racing on shared counters do not lose
+    # updates.
+    #
+    # Setting CFLAGS explicitly at configure time suppresses autoconf's
+    # default "-g -O2", so pass -O2 here.
+    COV_FLAGS="-fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic"
+    export CFLAGS="${CFLAGS:--O2} ${COV_FLAGS}"
+    export CXXFLAGS="${CXXFLAGS:--O2} ${COV_FLAGS}"
+    export LDFLAGS="${LDFLAGS:-} -fprofile-instr-generate -fcoverage-mapping"
+    # Clang ships omp.h in its resource-dir instead of /usr/include, so
+    # configure's plain omp.h check misses it. Point configure at both
+    # the include and (from apt libomp-dev) the runtime lib.
+    CLANG_RES_DIR="$("${CC}" -print-resource-dir 2>/dev/null || true)"
+    if [ -n "${CLANG_RES_DIR}" ] && [ -d "${CLANG_RES_DIR}/include" ]; then
+        OPENMP_ARGS+=("--with-openmp-includes=${CLANG_RES_DIR}/include")
+        # Debian's libomp-dev installs libomp.so under /usr/lib/llvm-<N>/lib,
+        # not the linker's default search path, so configure's link probe for
+        # omp_get_num_threads fails and it falls back to -lgomp - which clang's
+        # OpenMP-instrumented code (using __kmpc_*) then can't link against.
+        # The resource-dir sits at <llvm-libdir>/clang/<ver>, so its
+        # grandparent is <llvm-libdir>, where libomp.so lives.
+        LLVM_LIB_DIR="$(cd "${CLANG_RES_DIR}/../.." && pwd)"
+        if [ -e "${LLVM_LIB_DIR}/libomp.so" ]; then
+            OPENMP_ARGS+=("--with-openmp-libs=${LLVM_LIB_DIR}")
+        fi
+    fi
+fi
+
 ./configure \
     --enable-largefile \
     --prefix="$INSTALL_PREFIX/" \
@@ -50,7 +93,8 @@ export INSTALL_PREFIX=$1
     --with-readline \
     --with-sqlite \
     --with-tiff \
-    --with-zstd
+    --with-zstd \
+    ${OPENMP_ARGS[@]+"${OPENMP_ARGS[@]}"}
 
 eval $makecmd
 make install

.github/workflows/pytest.yml

@@ -32,6 +32,9 @@ jobs:
       FORCE_COLOR: 1 # for software including pip: https://force-color.org/
       CLICOLOR_FORCE: 1 # for other software including ninja: https://bixense.com/clicolors/
       PYTHONWARNINGS: always
+      # Only exercise LLVM C coverage on the latest Python; the older one
+      # runs the default gcc/no-instrumentation build for speed.
+      GRASS_LLVM_COVERAGE: ${{ matrix.python-version == '3.13' && '1' || '0' }}
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -54,6 +57,12 @@ jobs:
           xargs -a <(awk '! /^ *(#|$)/' ".github/workflows/apt.txt") -r -- \
               sudo apt-get install -y --no-install-recommends --no-install-suggests
 
+      - name: Install LLVM coverage toolchain
+        if: env.GRASS_LLVM_COVERAGE == '1'
+        run: |
+          sudo apt-get install -y --no-install-recommends --no-install-suggests \
+              clang llvm libomp-dev
+
       - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1
 
       - name: Install Python dependencies
@@ -90,16 +99,29 @@ jobs:
       - name: Test executing of the grass command
         run: .github/workflows/test_simple.sh
 
+      - name: Prepare coverage directories
+        run: |
+          mkdir -p "${PWD}/llvm-cov-raw"
+          # Wipe stray default.profraw the build step left behind (Makefiles
+          # run instrumented tools to dump interface XML), so their build-
+          # time data can't contaminate the test-run coverage.
+          find . -name 'default.profraw' -type f -delete
+          # %p (process id) separates every subprocess; %9m caps the per-binary
+          # pool at 512 files so long suites don't create millions of tiny
+          # .profraw. Together they eliminate the write-collision that made
+          # earlier runs non-deterministic.
+          {
+            echo "LLVM_PROFILE_FILE=${PWD}/llvm-cov-raw/grass-%p-%9m.profraw"
+            echo "INITIAL_GISBASE=$(grass --config path)"
+            echo "INITIAL_PWD=${PWD}"
+          } >> "${GITHUB_ENV}"
+
       - name: Run pytest with multiple workers in parallel
         run: |
           PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
           LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
-          INITIAL_GISBASE="$(grass --config path)"
-          INITIAL_PWD="${PWD}"
           export PYTHONPATH
           export LD_LIBRARY_PATH
-          export INITIAL_GISBASE
-          export INITIAL_PWD
           pytest \
             @.github/workflows/pytest_args_ci.txt \
             @.github/workflows/pytest_args_cov.txt \
@@ -113,12 +135,8 @@ jobs:
         run: |
           PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
           LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
-          INITIAL_GISBASE="$(grass --config path)"
-          INITIAL_PWD="${PWD}"
           export PYTHONPATH
           export LD_LIBRARY_PATH
-          export INITIAL_GISBASE
-          export INITIAL_PWD
           pytest \
             @.github/workflows/pytest_args_ci.txt \
             @.github/workflows/pytest_args_cov.txt \
@@ -129,29 +147,40 @@ jobs:
         run: |
           PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
           LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
-          INITIAL_GISBASE="$(grass --config path)"
-          INITIAL_PWD="${PWD}"
           export PYTHONPATH
           export LD_LIBRARY_PATH
-          export INITIAL_GISBASE
-          export INITIAL_PWD
           pytest \
             @.github/workflows/pytest_args_cov.txt \
             @.github/workflows/pytest_args_gunittest.txt \
             --junitxml=pytest.gunittest.junit.xml
-      - name: Fix non-standard installed script paths in coverage data
+
+      - name: Coverage disk usage (before combine)
+        if: ${{ !cancelled() }}
+        run: utils/coverage_disk_usage.sh
+
+      - name: Combine Python coverage and export XML
+        if: ${{ !cancelled() }}
         run: |
-          PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
-          LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
-          INITIAL_GISBASE="$(grass --config path)"
-          INITIAL_PWD="${PWD}"
-          export PYTHONPATH
-          export LD_LIBRARY_PATH
-          export INITIAL_GISBASE
-          export INITIAL_PWD
-          python utils/coverage_mapper.py
-          coverage combine
-          coverage html
+          # coverage combine must run before coverage_mapper.py: parallel
+          # mode leaves .coverage.<host>.<pid>.<rand> partials and no
+          # .coverage file for the mapper to read. Tolerate "no data" so
+          # a build failure earlier in the job does not mask itself as a
+          # later failure here.
+          coverage combine || true
+          if [ -f .coverage ]; then
+              python utils/coverage_mapper.py
+              if [ -f .coverage.fixed_scripts ]; then
+                  mv .coverage.fixed_scripts .coverage
+              fi
+              coverage html
+              coverage xml -o coverage.xml
+          else
+              echo "No .coverage data collected; skipping Python coverage export."
+          fi
+
+      - name: Merge and export LLVM C/C++ coverage
+        if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }}
+        run: utils/llvm_coverage_export.sh
 
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}
@@ -162,13 +191,25 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           report_type: test_results
 
-      - name: Upload coverage reports to Codecov
+      - name: Upload Python coverage to Codecov
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           verbose: true
+          files: coverage.xml
           flags: pytest-python-${{ matrix.python-version }}
           name: pytest-python-${{ matrix.python-version }}
           token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload LLVM C/C++ coverage to Codecov
+        if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }}
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        with:
+          verbose: true
+          files: coverage-llvm.lcov
+          flags: ubuntu-llvm-c
+          name: ubuntu-llvm-c-${{ matrix.python-version }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Make python-only code coverage test report available
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.gitignore

@@ -87,6 +87,11 @@ lib/*/latex/
 .coverage
 .coverage.*
 coverage.xml
+*.profraw
+*.profdata
+coverage-llvm.lcov
+llvm-cov-raw/
+coverage_html_report/
 
 # Ignore project-local git worktree directories created by AI coding
 # agent workflows (e.g. Claude Code)

utils/coverage_disk_usage.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# Print disk usage of raw coverage artifacts before combining. Emitted to
+# the CI log so we can watch it approach the runner storage budget.
+
+set -u
+
+: "${INITIAL_PWD:=$(pwd)}"
+: "${RAW_DIR:=${INITIAL_PWD}/llvm-cov-raw}"
+
+echo "=== LLVM .profraw files ==="
+if [ -d "${RAW_DIR}" ]; then
+    count=$(find "${RAW_DIR}" -name '*.profraw' -type f | wc -l | tr -d ' ')
+    echo "count: ${count}"
+    du -sh "${RAW_DIR}" 2>/dev/null || true
+else
+    echo "no ${RAW_DIR} (build was not instrumented, or no tests ran)"
+fi
+
+echo "=== Python coverage.py partial files ==="
+py_count=$(find "${INITIAL_PWD}" -maxdepth 2 -name '.coverage.*' -type f 2>/dev/null \
+    | wc -l | tr -d ' ')
+echo "count: ${py_count}"
+if [ "${py_count}" -gt 0 ]; then
+    du -sch "${INITIAL_PWD}"/.coverage.* 2>/dev/null | tail -1
+fi
+
+echo "=== Filesystem ==="
+df -h "${INITIAL_PWD}" 2>/dev/null || df -h

utils/llvm_coverage_export.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Merge LLVM source-based coverage .profraw files into a single .profdata,
+# then export lcov for Codecov. Run from CI after all test steps, but
+# generic enough to invoke by hand too: build with GRASS_LLVM_COVERAGE=1
+# (see build_ubuntu-24.04.sh), run tests with LLVM_PROFILE_FILE pointed at
+# a raw dir (see pytest.yml's "Prepare coverage directories" step), then
+# run this script from the repository root with a `grass` matching that
+# build on PATH.
+#
+# Inputs (env), all optional:
+#   INITIAL_PWD    - repository working directory (default: pwd)
+#   RAW_DIR        - directory holding *.profraw (default: $INITIAL_PWD/llvm-cov-raw)
+#   OUT_PROFDATA   - merged profdata output (default: $INITIAL_PWD/grass.profdata)
+#   OUT_LCOV       - lcov output for Codecov (default: $INITIAL_PWD/coverage-llvm.lcov)
+#   GISBASE        - installed GRASS prefix; falls back to `grass --config path`
+#
+# Requires llvm-profdata and llvm-cov on PATH matching the clang used to build.
+
+set -euo pipefail
+
+: "${INITIAL_PWD:=$(pwd)}"
+: "${RAW_DIR:=${INITIAL_PWD}/llvm-cov-raw}"
+: "${OUT_PROFDATA:=${INITIAL_PWD}/grass.profdata}"
+: "${OUT_LCOV:=${INITIAL_PWD}/coverage-llvm.lcov}"
+: "${GISBASE:=$(grass --config path)}"
+
+if ! compgen -G "${RAW_DIR}/*.profraw" > /dev/null; then
+    echo "llvm_coverage_export: no .profraw files under ${RAW_DIR}; nothing to export."
+    : > "${OUT_LCOV}"
+    exit 0
+fi
+
+# -sparse drops zero-count regions so merges are smaller and deterministic
+# for a given input set. The per-process "%p-%9m" naming can produce
+# hundreds of thousands of .profraw files in a full test run, so pass them
+# via -f (a newline-separated list) rather than shell-globbing them onto
+# argv, which overflows the OS argument-list limit ("Argument list too
+# long").
+PROFRAW_LIST="$(mktemp)"
+trap 'rm -f "${PROFRAW_LIST}"' EXIT
+find "${RAW_DIR}" -name '*.profraw' -print > "${PROFRAW_LIST}"
+llvm-profdata merge -sparse -f "${PROFRAW_LIST}" -o "${OUT_PROFDATA}"
+
+# Enumerate every instrumented ELF binary and shared library the install
+# produced. find -perm -u+x also sweeps in executable Python and shell
+# scripts, so filter to the ELF magic bytes before passing to llvm-cov.
+OBJ_ARGS=()
+while IFS= read -r -d '' obj; do
+    magic=$(head -c4 "${obj}" 2>/dev/null | od -An -tx1 | tr -d ' \n')
+    if [ "${magic}" = "7f454c46" ]; then
+        OBJ_ARGS+=(-object "${obj}")
+    fi
+done < <(
+    find "${GISBASE}" \
+        \( -type f -perm -u+x -o -name '*.so' -o -name '*.so.*' \) \
+        -print0
+)
+
+if [ "${#OBJ_ARGS[@]}" -eq 0 ]; then
+    echo "llvm_coverage_export: no instrumented objects found under ${GISBASE}." >&2
+    exit 1
+fi
+
+# Drop system, conda, and build-tree paths so Codecov only sees repo files.
+llvm-cov export \
+    -format=lcov \
+    -instr-profile="${OUT_PROFDATA}" \
+    -ignore-filename-regex='(^/opt/|^/usr/|/\.cache/|/bin\.[^/]+/|/dist\.[^/]+/|/OBJ\.[^/]+/)' \
+    "${OBJ_ARGS[@]}" \
+    > "${OUT_LCOV}"
+
+echo "llvm_coverage_export: wrote ${OUT_LCOV}"
+du -sh "${OUT_PROFDATA}" "${OUT_LCOV}"