CI: prototype SanitizerCoverage as an alternative to LLVM source-based coverage

.github/workflows/build_ubuntu-24.04.sh

@@ -30,6 +30,156 @@ set -u
 
 export INSTALL_PREFIX=$1
 
+# Extra configure args added under coverage; keep declared under `set -u`.
+OPENMP_ARGS=()
+
+# Clang ships omp.h in its resource-dir instead of /usr/include, so
+# configure's plain omp.h check misses it. Point configure at both the
+# include and (from apt libomp-dev) the runtime lib. Shared by both
+# clang-based coverage mechanisms below.
+configure_clang_openmp_args() {
+    CLANG_RES_DIR="$(clang -print-resource-dir 2>/dev/null || true)"
+    if [ -n "${CLANG_RES_DIR}" ] && [ -d "${CLANG_RES_DIR}/include" ]; then
+        OPENMP_ARGS+=("--with-openmp-includes=${CLANG_RES_DIR}/include")
+        # Debian's libomp-dev installs libomp.so under /usr/lib/llvm-<N>/lib,
+        # not the linker's default search path, so configure's link probe for
+        # omp_get_num_threads fails and it falls back to -lgomp - which clang's
+        # OpenMP-instrumented code (using __kmpc_*) then can't link against.
+        # The resource-dir sits at <llvm-libdir>/clang/<ver>, so its
+        # grandparent is <llvm-libdir>, where libomp.so lives.
+        LLVM_LIB_DIR="$(cd "${CLANG_RES_DIR}/../.." && pwd)"
+        if [ -e "${LLVM_LIB_DIR}/libomp.so" ]; then
+            OPENMP_ARGS+=("--with-openmp-libs=${LLVM_LIB_DIR}")
+        fi
+    fi
+}
+
+if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ] && [ "${GRASS_SANCOV_COVERAGE:-0}" = "1" ]; then
+    echo "GRASS_LLVM_COVERAGE and GRASS_SANCOV_COVERAGE are mutually exclusive" >&2
+    exit 1
+fi
+
+# LLVM source-based coverage instrumentation, opt-in via env var. Requires
+# clang (installed via apt.txt). Kept out of the default O2 path so
+# non-coverage runs are unaffected.
+if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ]; then
+    export CC="${CC:-clang}"
+    export CXX="${CXX:-clang++}"
+    # LLVM source-based coverage generates its region mapping before the
+    # optimizer runs, so optimization does not affect coverage quality
+    # (clang doc: SourceBasedCodeCoverage, "Impact of llvm optimizations
+    # on coverage reports"). Keep the project's normal -O2 to hold the
+    # runtime impact small. -fprofile-update=atomic makes counter
+    # increments atomic; required for GRASS's OpenMP-parallel regions
+    # (e.g. r.univar) so threads racing on shared counters do not lose
+    # updates.
+    #
+    # Setting CFLAGS explicitly at configure time suppresses autoconf's
+    # default "-g -O2", so pass -O2 here.
+    COV_FLAGS="-fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic"
+    export CFLAGS="${CFLAGS:--O2} ${COV_FLAGS}"
+    export CXXFLAGS="${CXXFLAGS:--O2} ${COV_FLAGS}"
+    export LDFLAGS="${LDFLAGS:-} -fprofile-instr-generate -fcoverage-mapping"
+    configure_clang_openmp_args
+fi
+
+# SanitizerCoverage instrumentation, opt-in via env var, compared against
+# GRASS_LLVM_COVERAGE above -- see doc/development/
+# sancov_coverage_experiment.md. GRASS_SANCOV_CC selects gcc (the only
+# -fsanitize-coverage= sub-option GCC 13/14 implement is trace-pc, no
+# per-edge guard: confirmed empirically, see the report) or clang
+# (trace-pc-guard, cheaper steady-state dedup).
+#
+# Unlike -fprofile-instr-generate above, -fsanitize-coverage= is not
+# special-cased by either compiler's driver to auto-link a runtime, so
+# utils/sancov_runtime.c is built here into a small shared library that
+# every GRASS binary must link against. Naively exporting "-lsancovrt"
+# through LDFLAGS or configure's $LIBS does not reliably reach every link
+# GRASS's build performs: GRASS's own Make templates (Shlib.make,
+# Compile.make) place $(LDFLAGS) *before* the object files (GNU ld
+# resolves left to right and never revisits a library once passed, so an
+# undefined reference there is never satisfied), a few special-cased
+# link rules (e.g. lib/init/Makefile's echo/run targets) do not go
+# through the usual $(LIBES)/$(MATHLIB) machinery those templates share
+# at all, and configure's own custom library probes (LOC_CHECK_LIBS,
+# used for BLAS, GDAL, PROJ, and most other optional dependencies) build
+# their own bespoke compile-and-link commands rather than going through
+# autoconf's usual $LIBS (confirmed empirically, in that order, each by
+# a different otherwise-unrelated-looking failure: an "undefined
+# reference" from ld, then a segfault when it was papered over with
+# -Wl,--unresolved-symbols=ignore-all, then an unrelated-looking "unable
+# to link to (C)BLAS", then a silently-unlinked etc/run).
+#
+# Rather than chase each of those individually, $CC/$CXX are pointed at
+# a tiny wrapper that appends the runtime's -L/-l/-rpath flags to
+# whatever argv it is given, verbatim, then execs the real compiler.
+# That covers every link GRASS's build performs, however it constructs
+# the rest of the command line, because they all ultimately invoke
+# $(CC)/$(CXX) (or $(LINK), which itself defaults to $(CC)) as the
+# actual compiler front-end. The same flags are harmless noise on a
+# compile-only invocation (-c): gcc and clang both silently ignore
+# unused -l/-L/-Wl, arguments there (confirmed empirically), so the one
+# wrapper is safe to use for every invocation, compile or link, without
+# needing to tell them apart -- including configure's own, so the
+# instrumentation flags can go back to being set the same way
+# GRASS_LLVM_COVERAGE sets them above, before ./configure runs.
+if [ "${GRASS_SANCOV_COVERAGE:-0}" = "1" ]; then
+    SANCOV_CC="${GRASS_SANCOV_CC:-gcc}"
+    case "${SANCOV_CC}" in
+    gcc)
+        REAL_CC="gcc"
+        REAL_CXX="g++"
+        SANCOV_FLAGS="-fsanitize-coverage=trace-pc"
+        ;;
+    clang)
+        REAL_CC="clang"
+        REAL_CXX="clang++"
+        SANCOV_FLAGS="-fsanitize-coverage=trace-pc-guard"
+        configure_clang_openmp_args
+        ;;
+    *)
+        echo "GRASS_SANCOV_CC must be gcc or clang, got '${SANCOV_CC}'" >&2
+        exit 1
+        ;;
+    esac
+
+    SANCOV_RUNTIME_DIR="${PWD}/sancov-runtime"
+    mkdir -p "${SANCOV_RUNTIME_DIR}"
+    "${REAL_CC}" -O2 -fPIC -shared -pthread \
+        -o "${SANCOV_RUNTIME_DIR}/libsancovrt.so" \
+        utils/sancov_runtime.c -ldl
+
+    # -lm is appended here too, unconditionally, alongside -lsancovrt:
+    # confirmed on a real CI run (GRASS_SANCOV_CC=clang) that GRASS's own
+    # $(MATHLIB) -- normally "-lm", substituted once by configure's
+    # `AC_CHECK_FUNC(atan, MATHLIB=, [AC_CHECK_LIB(m, atan,
+    # MATHLIB=-lm...)])` probe -- comes out empty on this toolchain: glibc
+    # merged libm into libc (since 2.34; this runs on Ubuntu 24.04's
+    # 2.39), so the plain "does atan link without -lm" check succeeds and
+    # MATHLIB is left blank. GNU ld tolerates this for GRASS binaries
+    # that reference cos/sin/floor by transitively finding them in libc
+    # via an already-linked GRASS shared library; this workflow's `Run
+    # rui314/setup-mold` step makes `mold` the default linker, which does
+    # not, so ps.map's link failed with "undefined symbol: cos/sin/floor"
+    # -- a real, otherwise-unrelated-looking failure this wrapper's own
+    # extra library reference happens to be the natural place to fix,
+    # since GRASS's Make templates offer no more reliable hook for it
+    # than they did for the coverage runtime itself.
+    cat > "${SANCOV_RUNTIME_DIR}/cc-wrapper.sh" <<EOF
+#!/usr/bin/env bash
+exec "${REAL_CC}" "\$@" -L"${SANCOV_RUNTIME_DIR}" -lsancovrt -Wl,-rpath,"${SANCOV_RUNTIME_DIR}" -lm
+EOF
+    cat > "${SANCOV_RUNTIME_DIR}/cxx-wrapper.sh" <<EOF
+#!/usr/bin/env bash
+exec "${REAL_CXX}" "\$@" -L"${SANCOV_RUNTIME_DIR}" -lsancovrt -Wl,-rpath,"${SANCOV_RUNTIME_DIR}" -lm
+EOF
+    chmod +x "${SANCOV_RUNTIME_DIR}/cc-wrapper.sh" "${SANCOV_RUNTIME_DIR}/cxx-wrapper.sh"
+    export CC="${SANCOV_RUNTIME_DIR}/cc-wrapper.sh"
+    export CXX="${SANCOV_RUNTIME_DIR}/cxx-wrapper.sh"
+    export CFLAGS="${CFLAGS:--O2} -g ${SANCOV_FLAGS}"
+    export CXXFLAGS="${CXXFLAGS:--O2} -g ${SANCOV_FLAGS}"
+fi
+
 ./configure \
     --enable-largefile \
     --prefix="$INSTALL_PREFIX/" \
@@ -50,7 +200,8 @@ export INSTALL_PREFIX=$1
     --with-readline \
     --with-sqlite \
     --with-tiff \
-    --with-zstd
+    --with-zstd \
+    ${OPENMP_ARGS[@]+"${OPENMP_ARGS[@]}"}
 
 eval $makecmd
 make install

.github/workflows/cmake.yml

@@ -20,10 +20,10 @@ on:
       - "!doc/**"
       - "doc/CMakeLists.txt"
       - "doc/**/CMakeLists.txt"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
-  cancel-in-progress: true
+  # Lets this branch's coverage-mechanism comparison be run on demand
+  # without opening a PR, see doc/development/
+  # sancov_coverage_experiment.md. Remove before merging.
+  workflow_dispatch:
 
 env:
   CMAKE_UNITY_BUILD: OFF
@@ -33,9 +33,26 @@ permissions:
 
 jobs:
   build-cmake:
+    concurrency:
+      group: >-
+        ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}-${{
+        matrix.sancov }}
+      cancel-in-progress: true
+    strategy:
+      matrix:
+        # sancov: gcc builds with -DWITH_SANCOV_COVERAGE=ON (opt-in
+        # SanitizerCoverage experiment, see doc/development/
+        # sancov_coverage_experiment.md) alongside the existing default
+        # (uninstrumented) cell, gcc being what this workflow already
+        # builds with.
+        sancov:
+          - none
+          - gcc
+      fail-fast: true
     runs-on: ubuntu-24.04
     env:
       CMAKE_VERSION: "3.22.0"
+      GRASS_SANCOV_CMAKE_ARG: ${{ matrix.sancov == 'gcc' && '-DWITH_SANCOV_COVERAGE=ON' || '' }}
     steps:
       - name: Checkout GRASS
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -72,9 +89,12 @@ jobs:
           mkdir "${HOME}/install"
       - name: Configure
         run: |
+          # GRASS_SANCOV_CMAKE_ARG is a single flag or empty; unquoted so an
+          # empty value adds no argument, rather than an empty string one.
+          # shellcheck disable=SC2086
           cmake -S "${GITHUB_WORKSPACE}" -B "${GITHUB_WORKSPACE}/build" -G Ninja \
             -DCMAKE_INSTALL_PREFIX="${HOME}/install" -DWITH_READLINE=ON -DWITH_ODBC=ON \
-            -DWITH_NETCDF=ON -DWITH_BZLIB=ON
+            -DWITH_NETCDF=ON -DWITH_BZLIB=ON ${GRASS_SANCOV_CMAKE_ARG}
       - name: Print CMakeCache.txt
         shell: bash -el {0}
         run: |
@@ -93,12 +113,31 @@ jobs:
         run: .github/workflows/print_versions.sh
       - name: Test executing of the grass command
         run: .github/workflows/test_simple.sh
+      - name: Prepare SanitizerCoverage dump directory
+        if: matrix.sancov == 'gcc'
+        run: |
+          mkdir -p "${PWD}/sancov-raw"
+          echo "GRASS_SANCOV_DUMP_DIR=${PWD}/sancov-raw" >> "${GITHUB_ENV}"
       - name: Run tests
         run: .github/workflows/test_thorough.sh --config .gunittest.cfg --min-success 100
+      - name: Symbolize and summarize SanitizerCoverage
+        if: ${{ !cancelled() && matrix.sancov == 'gcc' }}
+        run: |
+          python3 utils/sancov_symbolize.py "${GRASS_SANCOV_DUMP_DIR}" \
+              --ignore-path-substring /usr/ \
+              > sancov-coverage-summary.txt
+          wc -l sancov-coverage-summary.txt
       - name: Make HTML test report available
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: testreport-CMake
+          name: testreport-CMake-${{ matrix.sancov }}
           path: testreport
           retention-days: 3
+      - name: Make SanitizerCoverage summary available
+        if: ${{ !cancelled() && matrix.sancov == 'gcc' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: sancov-coverage-summary-cmake
+          path: sancov-coverage-summary.txt
+          retention-days: 7

.github/workflows/macos.yml

@@ -23,6 +23,7 @@ jobs:
     runs-on: macos-26
     env:
       PYTHONWARNINGS: always
+      GRASS_LLVM_COVERAGE: "1"
     steps:
       - name: Info
         run: |
@@ -57,8 +58,11 @@ jobs:
       - name: Prepare Conda environment file
         env:
           test_dependencies: |
+            coverage
             ipython
+            llvm-tools
             pytest
+            pytest-cov
             pytest-github-actions-annotate-failures
             pytest-timeout
             pytest-xdist
@@ -92,6 +96,22 @@ jobs:
         shell: micromamba-shell {0}
         run: source ./.github/workflows/print_versions.sh
 
+      - name: Prepare coverage directories
+        shell: micromamba-shell {0}
+        run: |
+          mkdir -p "${PWD}/llvm-cov-raw"
+          # Wipe stray default.profraw the build step left behind (Makefiles
+          # run instrumented tools to dump interface XML) before the tests
+          # start writing controlled per-PID files.
+          find . -name 'default.profraw' -type f -delete
+          # %p (process id) separates every subprocess; %9m caps the per-binary
+          # pool at 512 files so long xdist runs don't fill the runner disk.
+          {
+            echo "LLVM_PROFILE_FILE=${PWD}/llvm-cov-raw/grass-%p-%9m.profraw"
+            echo "INITIAL_GISBASE=$(grass --config path)"
+            echo "INITIAL_PWD=${PWD}"
+          } >> "${GITHUB_ENV}"
+
       - name: Run pytest with multiple workers in parallel
         shell: micromamba-shell {0}
         run: |
@@ -101,6 +121,7 @@ jobs:
           export LD_LIBRARY_PATH
           pytest \
             @.github/workflows/pytest_args_ci.txt \
+            @.github/workflows/pytest_args_cov.txt \
             @.github/workflows/pytest_args_parallel.txt \
             --junitxml=pytest.xdist.junit.xml \
             -k 'not testsuite'
@@ -116,6 +137,7 @@ jobs:
           export LD_LIBRARY_PATH
           pytest \
             @.github/workflows/pytest_args_ci.txt \
+            @.github/workflows/pytest_args_cov.txt \
             @.github/workflows/pytest_args_not_parallel.txt \
             --junitxml=pytest.needs_solo_run.junit.xml \
             -k 'not testsuite'
@@ -127,6 +149,7 @@ jobs:
           export PYTHONPATH
           export LD_LIBRARY_PATH
           pytest \
+            @.github/workflows/pytest_args_cov.txt \
             @.github/workflows/pytest_args_gunittest.txt \
             --junitxml=pytest.gunittest.junit.xml
 
@@ -171,6 +194,64 @@ jobs:
           SAMPLE_DATA_URL: "file://${{ github.workspace }}/sample-data/\
             nc_spm_full_v2beta1.tar.gz"
 
+      - name: Coverage disk usage (before combine)
+        if: ${{ !cancelled() }}
+        shell: micromamba-shell {0}
+        run: utils/coverage_disk_usage.sh
+
+      - name: Combine Python coverage and export XML
+        if: ${{ !cancelled() }}
+        shell: micromamba-shell {0}
+        run: |
+          # coverage combine before coverage_mapper.py: parallel mode leaves
+          # only .coverage.<host>.<pid>.<rand> partials, no .coverage yet.
+          coverage combine || true
+          if [ -f .coverage ]; then
+              python utils/coverage_mapper.py
+              if [ -f .coverage.fixed_scripts ]; then
+                  mv .coverage.fixed_scripts .coverage
+              fi
+              coverage html
+              coverage xml -o coverage.xml
+          else
+              echo "No .coverage data collected; skipping Python coverage export."
+          fi
+
+      - name: Merge and export LLVM C/C++ coverage
+        if: ${{ !cancelled() }}
+        shell: micromamba-shell {0}
+        run: utils/llvm_coverage_export.sh
+
+      - name: Upload Python coverage to Codecov (macOS)
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        with:
+          verbose: true
+          files: coverage.xml
+          flags: macos-pytest-python
+          name: macos-pytest-python
+          token: ${{ secrets.CODECOV_TOKEN }}
+          skip_validation: true
+
+      - name: Upload LLVM C/C++ coverage to Codecov (macOS)
+        if: ${{ !cancelled() }}
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+        with:
+          verbose: true
+          files: coverage-llvm.lcov
+          flags: macos-llvm-c
+          name: macos-llvm-c
+          token: ${{ secrets.CODECOV_TOKEN }}
+          skip_validation: true
+
+      - name: Make python-only code coverage test report available
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: python-codecoverage-report-macOS
+          path: coverage_html_report
+          retention-days: 1
+
       - name: Make HTML test report available
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1