diff --git a/.github/workflows/build_ubuntu-24.04.sh b/.github/workflows/build_ubuntu-24.04.sh index 677c46d5f33..308049bbdcc 100755 --- a/.github/workflows/build_ubuntu-24.04.sh +++ b/.github/workflows/build_ubuntu-24.04.sh @@ -30,6 +30,49 @@ set -u export INSTALL_PREFIX=$1 +# Extra configure args added under coverage; keep declared under `set -u`. +OPENMP_ARGS=() + +# LLVM source-based coverage instrumentation, opt-in via env var. Requires +# clang (installed only when opted in, see pytest.yml). Kept out of the +# default O2 path so non-coverage runs are unaffected. +if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ]; then + export CC="${CC:-clang}" + export CXX="${CXX:-clang++}" + # LLVM source-based coverage generates its region mapping before the + # optimizer runs, so optimization does not affect coverage quality + # (clang doc: SourceBasedCodeCoverage, "Impact of llvm optimizations + # on coverage reports"). Keep the project's normal -O2 to hold the + # runtime impact small. -fprofile-update=atomic makes counter + # increments atomic; required for GRASS's OpenMP-parallel regions + # (e.g. r.univar) so threads racing on shared counters do not lose + # updates. + # + # Setting CFLAGS explicitly at configure time suppresses autoconf's + # default "-g -O2", so pass -O2 here. + COV_FLAGS="-fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic" + export CFLAGS="${CFLAGS:--O2} ${COV_FLAGS}" + export CXXFLAGS="${CXXFLAGS:--O2} ${COV_FLAGS}" + export LDFLAGS="${LDFLAGS:-} -fprofile-instr-generate -fcoverage-mapping" + # Clang ships omp.h in its resource-dir instead of /usr/include, so + # configure's plain omp.h check misses it. Point configure at both + # the include and (from apt libomp-dev) the runtime lib. + CLANG_RES_DIR="$("${CC}" -print-resource-dir 2>/dev/null || true)" + if [ -n "${CLANG_RES_DIR}" ] && [ -d "${CLANG_RES_DIR}/include" ]; then + OPENMP_ARGS+=("--with-openmp-includes=${CLANG_RES_DIR}/include") + # Debian's libomp-dev installs libomp.so under /usr/lib/llvm-/lib, + # not the linker's default search path, so configure's link probe for + # omp_get_num_threads fails and it falls back to -lgomp - which clang's + # OpenMP-instrumented code (using __kmpc_*) then can't link against. + # The resource-dir sits at /clang/, so its + # grandparent is , where libomp.so lives. + LLVM_LIB_DIR="$(cd "${CLANG_RES_DIR}/../.." && pwd)" + if [ -e "${LLVM_LIB_DIR}/libomp.so" ]; then + OPENMP_ARGS+=("--with-openmp-libs=${LLVM_LIB_DIR}") + fi + fi +fi + ./configure \ --enable-largefile \ --prefix="$INSTALL_PREFIX/" \ @@ -50,7 +93,8 @@ export INSTALL_PREFIX=$1 --with-readline \ --with-sqlite \ --with-tiff \ - --with-zstd + --with-zstd \ + ${OPENMP_ARGS[@]+"${OPENMP_ARGS[@]}"} eval $makecmd make install diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index bfd8d77b693..cbd1262ce97 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -32,6 +32,9 @@ jobs: FORCE_COLOR: 1 # for software including pip: https://force-color.org/ CLICOLOR_FORCE: 1 # for other software including ninja: https://bixense.com/clicolors/ PYTHONWARNINGS: always + # Only exercise LLVM C coverage on the latest Python; the older one + # runs the default gcc/no-instrumentation build for speed. + GRASS_LLVM_COVERAGE: ${{ matrix.python-version == '3.13' && '1' || '0' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -54,6 +57,12 @@ jobs: xargs -a <(awk '! /^ *(#|$)/' ".github/workflows/apt.txt") -r -- \ sudo apt-get install -y --no-install-recommends --no-install-suggests + - name: Install LLVM coverage toolchain + if: env.GRASS_LLVM_COVERAGE == '1' + run: | + sudo apt-get install -y --no-install-recommends --no-install-suggests \ + clang llvm libomp-dev + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - name: Install Python dependencies @@ -90,16 +99,29 @@ jobs: - name: Test executing of the grass command run: .github/workflows/test_simple.sh + - name: Prepare coverage directories + run: | + mkdir -p "${PWD}/llvm-cov-raw" + # Wipe stray default.profraw the build step left behind (Makefiles + # run instrumented tools to dump interface XML), so their build- + # time data can't contaminate the test-run coverage. + find . -name 'default.profraw' -type f -delete + # %p (process id) separates every subprocess; %9m caps the per-binary + # pool at 512 files so long suites don't create millions of tiny + # .profraw. Together they eliminate the write-collision that made + # earlier runs non-deterministic. + { + echo "LLVM_PROFILE_FILE=${PWD}/llvm-cov-raw/grass-%p-%9m.profraw" + echo "INITIAL_GISBASE=$(grass --config path)" + echo "INITIAL_PWD=${PWD}" + } >> "${GITHUB_ENV}" + - name: Run pytest with multiple workers in parallel run: | PYTHONPATH="$(grass --config python_path):${PYTHONPATH}" LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}" - INITIAL_GISBASE="$(grass --config path)" - INITIAL_PWD="${PWD}" export PYTHONPATH export LD_LIBRARY_PATH - export INITIAL_GISBASE - export INITIAL_PWD pytest \ @.github/workflows/pytest_args_ci.txt \ @.github/workflows/pytest_args_cov.txt \ @@ -113,12 +135,8 @@ jobs: run: | PYTHONPATH="$(grass --config python_path):${PYTHONPATH}" LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}" - INITIAL_GISBASE="$(grass --config path)" - INITIAL_PWD="${PWD}" export PYTHONPATH export LD_LIBRARY_PATH - export INITIAL_GISBASE - export INITIAL_PWD pytest \ @.github/workflows/pytest_args_ci.txt \ @.github/workflows/pytest_args_cov.txt \ @@ -129,29 +147,40 @@ jobs: run: | PYTHONPATH="$(grass --config python_path):${PYTHONPATH}" LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}" - INITIAL_GISBASE="$(grass --config path)" - INITIAL_PWD="${PWD}" export PYTHONPATH export LD_LIBRARY_PATH - export INITIAL_GISBASE - export INITIAL_PWD pytest \ @.github/workflows/pytest_args_cov.txt \ @.github/workflows/pytest_args_gunittest.txt \ --junitxml=pytest.gunittest.junit.xml - - name: Fix non-standard installed script paths in coverage data + + - name: Coverage disk usage (before combine) + if: ${{ !cancelled() }} + run: utils/coverage_disk_usage.sh + + - name: Combine Python coverage and export XML + if: ${{ !cancelled() }} run: | - PYTHONPATH="$(grass --config python_path):${PYTHONPATH}" - LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}" - INITIAL_GISBASE="$(grass --config path)" - INITIAL_PWD="${PWD}" - export PYTHONPATH - export LD_LIBRARY_PATH - export INITIAL_GISBASE - export INITIAL_PWD - python utils/coverage_mapper.py - coverage combine - coverage html + # coverage combine must run before coverage_mapper.py: parallel + # mode leaves .coverage... partials and no + # .coverage file for the mapper to read. Tolerate "no data" so + # a build failure earlier in the job does not mask itself as a + # later failure here. + coverage combine || true + if [ -f .coverage ]; then + python utils/coverage_mapper.py + if [ -f .coverage.fixed_scripts ]; then + mv .coverage.fixed_scripts .coverage + fi + coverage html + coverage xml -o coverage.xml + else + echo "No .coverage data collected; skipping Python coverage export." + fi + + - name: Merge and export LLVM C/C++ coverage + if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }} + run: utils/llvm_coverage_export.sh - name: Upload test results to Codecov if: ${{ !cancelled() }} @@ -162,13 +191,25 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} report_type: test_results - - name: Upload coverage reports to Codecov + - name: Upload Python coverage to Codecov uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: verbose: true + files: coverage.xml flags: pytest-python-${{ matrix.python-version }} name: pytest-python-${{ matrix.python-version }} token: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload LLVM C/C++ coverage to Codecov + if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }} + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + verbose: true + files: coverage-llvm.lcov + flags: ubuntu-llvm-c + name: ubuntu-llvm-c-${{ matrix.python-version }} + token: ${{ secrets.CODECOV_TOKEN }} + - name: Make python-only code coverage test report available if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.gitignore b/.gitignore index ac47be4c591..b69799a91ea 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,11 @@ lib/*/latex/ .coverage .coverage.* coverage.xml +*.profraw +*.profdata +coverage-llvm.lcov +llvm-cov-raw/ +coverage_html_report/ # Ignore project-local git worktree directories created by AI coding # agent workflows (e.g. Claude Code) diff --git a/utils/coverage_disk_usage.sh b/utils/coverage_disk_usage.sh new file mode 100755 index 00000000000..f12a5c59c85 --- /dev/null +++ b/utils/coverage_disk_usage.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Print disk usage of raw coverage artifacts before combining. Emitted to +# the CI log so we can watch it approach the runner storage budget. + +set -u + +: "${INITIAL_PWD:=$(pwd)}" +: "${RAW_DIR:=${INITIAL_PWD}/llvm-cov-raw}" + +echo "=== LLVM .profraw files ===" +if [ -d "${RAW_DIR}" ]; then + count=$(find "${RAW_DIR}" -name '*.profraw' -type f | wc -l | tr -d ' ') + echo "count: ${count}" + du -sh "${RAW_DIR}" 2>/dev/null || true +else + echo "no ${RAW_DIR} (build was not instrumented, or no tests ran)" +fi + +echo "=== Python coverage.py partial files ===" +py_count=$(find "${INITIAL_PWD}" -maxdepth 2 -name '.coverage.*' -type f 2>/dev/null \ + | wc -l | tr -d ' ') +echo "count: ${py_count}" +if [ "${py_count}" -gt 0 ]; then + du -sch "${INITIAL_PWD}"/.coverage.* 2>/dev/null | tail -1 +fi + +echo "=== Filesystem ===" +df -h "${INITIAL_PWD}" 2>/dev/null || df -h diff --git a/utils/llvm_coverage_export.sh b/utils/llvm_coverage_export.sh new file mode 100755 index 00000000000..c2a9d860fc7 --- /dev/null +++ b/utils/llvm_coverage_export.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# Merge LLVM source-based coverage .profraw files into a single .profdata, +# then export lcov for Codecov. Run from CI after all test steps, but +# generic enough to invoke by hand too: build with GRASS_LLVM_COVERAGE=1 +# (see build_ubuntu-24.04.sh), run tests with LLVM_PROFILE_FILE pointed at +# a raw dir (see pytest.yml's "Prepare coverage directories" step), then +# run this script from the repository root with a `grass` matching that +# build on PATH. +# +# Inputs (env), all optional: +# INITIAL_PWD - repository working directory (default: pwd) +# RAW_DIR - directory holding *.profraw (default: $INITIAL_PWD/llvm-cov-raw) +# OUT_PROFDATA - merged profdata output (default: $INITIAL_PWD/grass.profdata) +# OUT_LCOV - lcov output for Codecov (default: $INITIAL_PWD/coverage-llvm.lcov) +# GISBASE - installed GRASS prefix; falls back to `grass --config path` +# +# Requires llvm-profdata and llvm-cov on PATH matching the clang used to build. + +set -euo pipefail + +: "${INITIAL_PWD:=$(pwd)}" +: "${RAW_DIR:=${INITIAL_PWD}/llvm-cov-raw}" +: "${OUT_PROFDATA:=${INITIAL_PWD}/grass.profdata}" +: "${OUT_LCOV:=${INITIAL_PWD}/coverage-llvm.lcov}" +: "${GISBASE:=$(grass --config path)}" + +if ! compgen -G "${RAW_DIR}/*.profraw" > /dev/null; then + echo "llvm_coverage_export: no .profraw files under ${RAW_DIR}; nothing to export." + : > "${OUT_LCOV}" + exit 0 +fi + +# -sparse drops zero-count regions so merges are smaller and deterministic +# for a given input set. The per-process "%p-%9m" naming can produce +# hundreds of thousands of .profraw files in a full test run, so pass them +# via -f (a newline-separated list) rather than shell-globbing them onto +# argv, which overflows the OS argument-list limit ("Argument list too +# long"). +PROFRAW_LIST="$(mktemp)" +trap 'rm -f "${PROFRAW_LIST}"' EXIT +find "${RAW_DIR}" -name '*.profraw' -print > "${PROFRAW_LIST}" +llvm-profdata merge -sparse -f "${PROFRAW_LIST}" -o "${OUT_PROFDATA}" + +# Enumerate every instrumented ELF binary and shared library the install +# produced. find -perm -u+x also sweeps in executable Python and shell +# scripts, so filter to the ELF magic bytes before passing to llvm-cov. +OBJ_ARGS=() +while IFS= read -r -d '' obj; do + magic=$(head -c4 "${obj}" 2>/dev/null | od -An -tx1 | tr -d ' \n') + if [ "${magic}" = "7f454c46" ]; then + OBJ_ARGS+=(-object "${obj}") + fi +done < <( + find "${GISBASE}" \ + \( -type f -perm -u+x -o -name '*.so' -o -name '*.so.*' \) \ + -print0 +) + +if [ "${#OBJ_ARGS[@]}" -eq 0 ]; then + echo "llvm_coverage_export: no instrumented objects found under ${GISBASE}." >&2 + exit 1 +fi + +# Drop system, conda, and build-tree paths so Codecov only sees repo files. +llvm-cov export \ + -format=lcov \ + -instr-profile="${OUT_PROFDATA}" \ + -ignore-filename-regex='(^/opt/|^/usr/|/\.cache/|/bin\.[^/]+/|/dist\.[^/]+/|/OBJ\.[^/]+/)' \ + "${OBJ_ARGS[@]}" \ + > "${OUT_LCOV}" + +echo "llvm_coverage_export: wrote ${OUT_LCOV}" +du -sh "${OUT_PROFDATA}" "${OUT_LCOV}"