Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 45 additions & 1 deletion .github/workflows/build_ubuntu-24.04.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,49 @@ set -u

export INSTALL_PREFIX=$1

# Extra configure args added under coverage; keep declared under `set -u`.
OPENMP_ARGS=()

# LLVM source-based coverage instrumentation, opt-in via env var. Requires
# clang (installed via apt.txt). Kept out of the default O2 path so
# non-coverage runs are unaffected.
if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ]; then
export CC="${CC:-clang}"
export CXX="${CXX:-clang++}"
# LLVM source-based coverage generates its region mapping before the
# optimizer runs, so optimization does not affect coverage quality
# (clang doc: SourceBasedCodeCoverage, "Impact of llvm optimizations
# on coverage reports"). Keep the project's normal -O2 to hold the
# runtime impact small. -fprofile-update=atomic makes counter
# increments atomic; required for GRASS's OpenMP-parallel regions
# (e.g. r.univar) so threads racing on shared counters do not lose
# updates.
#
# Setting CFLAGS explicitly at configure time suppresses autoconf's
# default "-g -O2", so pass -O2 here.
COV_FLAGS="-fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic"
export CFLAGS="${CFLAGS:--O2} ${COV_FLAGS}"
export CXXFLAGS="${CXXFLAGS:--O2} ${COV_FLAGS}"
export LDFLAGS="${LDFLAGS:-} -fprofile-instr-generate -fcoverage-mapping"
# Clang ships omp.h in its resource-dir instead of /usr/include, so
# configure's plain omp.h check misses it. Point configure at both
# the include and (from apt libomp-dev) the runtime lib.
CLANG_RES_DIR="$("${CC}" -print-resource-dir 2>/dev/null || true)"
if [ -n "${CLANG_RES_DIR}" ] && [ -d "${CLANG_RES_DIR}/include" ]; then
OPENMP_ARGS+=("--with-openmp-includes=${CLANG_RES_DIR}/include")
# Debian's libomp-dev installs libomp.so under /usr/lib/llvm-<N>/lib,
# not the linker's default search path, so configure's link probe for
# omp_get_num_threads fails and it falls back to -lgomp - which clang's
# OpenMP-instrumented code (using __kmpc_*) then can't link against.
# The resource-dir sits at <llvm-libdir>/clang/<ver>, so its
# grandparent is <llvm-libdir>, where libomp.so lives.
LLVM_LIB_DIR="$(cd "${CLANG_RES_DIR}/../.." && pwd)"
if [ -e "${LLVM_LIB_DIR}/libomp.so" ]; then
OPENMP_ARGS+=("--with-openmp-libs=${LLVM_LIB_DIR}")
fi
fi
fi

./configure \
--enable-largefile \
--prefix="$INSTALL_PREFIX/" \
Expand All @@ -50,7 +93,8 @@ export INSTALL_PREFIX=$1
--with-readline \
--with-sqlite \
--with-tiff \
--with-zstd
--with-zstd \
${OPENMP_ARGS[@]+"${OPENMP_ARGS[@]}"}

eval $makecmd
make install
81 changes: 81 additions & 0 deletions .github/workflows/macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ jobs:
runs-on: macos-26
env:
PYTHONWARNINGS: always
GRASS_LLVM_COVERAGE: "1"
steps:
- name: Info
run: |
Expand Down Expand Up @@ -57,8 +58,11 @@ jobs:
- name: Prepare Conda environment file
env:
test_dependencies: |
coverage
ipython
llvm-tools
pytest
pytest-cov
pytest-github-actions-annotate-failures
pytest-timeout
pytest-xdist
Expand Down Expand Up @@ -92,6 +96,22 @@ jobs:
shell: micromamba-shell {0}
run: source ./.github/workflows/print_versions.sh

- name: Prepare coverage directories
shell: micromamba-shell {0}
run: |
mkdir -p "${PWD}/llvm-cov-raw"
# Wipe stray default.profraw the build step left behind (Makefiles
# run instrumented tools to dump interface XML) before the tests
# start writing controlled per-PID files.
find . -name 'default.profraw' -type f -delete
# %p (process id) separates every subprocess; %9m caps the per-binary
# pool at 512 files so long xdist runs don't fill the runner disk.
{
echo "LLVM_PROFILE_FILE=${PWD}/llvm-cov-raw/grass-%p-%9m.profraw"
echo "INITIAL_GISBASE=$(grass --config path)"
echo "INITIAL_PWD=${PWD}"
} >> "${GITHUB_ENV}"

- name: Run pytest with multiple workers in parallel
shell: micromamba-shell {0}
run: |
Expand All @@ -101,6 +121,7 @@ jobs:
export LD_LIBRARY_PATH
pytest \
@.github/workflows/pytest_args_ci.txt \
@.github/workflows/pytest_args_cov.txt \
@.github/workflows/pytest_args_parallel.txt \
--junitxml=pytest.xdist.junit.xml \
-k 'not testsuite'
Expand All @@ -116,6 +137,7 @@ jobs:
export LD_LIBRARY_PATH
pytest \
@.github/workflows/pytest_args_ci.txt \
@.github/workflows/pytest_args_cov.txt \
@.github/workflows/pytest_args_not_parallel.txt \
--junitxml=pytest.needs_solo_run.junit.xml \
-k 'not testsuite'
Expand All @@ -127,6 +149,7 @@ jobs:
export PYTHONPATH
export LD_LIBRARY_PATH
pytest \
@.github/workflows/pytest_args_cov.txt \
@.github/workflows/pytest_args_gunittest.txt \
--junitxml=pytest.gunittest.junit.xml

Expand Down Expand Up @@ -171,6 +194,64 @@ jobs:
SAMPLE_DATA_URL: "file://${{ github.workspace }}/sample-data/\
nc_spm_full_v2beta1.tar.gz"

- name: Coverage disk usage (before combine)
if: ${{ !cancelled() }}
shell: micromamba-shell {0}
run: utils/coverage_disk_usage.sh

- name: Combine Python coverage and export XML
if: ${{ !cancelled() }}
shell: micromamba-shell {0}
run: |
# coverage combine before coverage_mapper.py: parallel mode leaves
# only .coverage.<host>.<pid>.<rand> partials, no .coverage yet.
coverage combine || true
if [ -f .coverage ]; then
python utils/coverage_mapper.py
if [ -f .coverage.fixed_scripts ]; then
mv .coverage.fixed_scripts .coverage
fi
coverage html
coverage xml -o coverage.xml
else
echo "No .coverage data collected; skipping Python coverage export."
fi

- name: Merge and export LLVM C/C++ coverage
if: ${{ !cancelled() }}
shell: micromamba-shell {0}
run: utils/llvm_coverage_export.sh

- name: Upload Python coverage to Codecov (macOS)
if: ${{ !cancelled() }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
verbose: true
files: coverage.xml
flags: macos-pytest-python
name: macos-pytest-python
token: ${{ secrets.CODECOV_TOKEN }}
skip_validation: true

- name: Upload LLVM C/C++ coverage to Codecov (macOS)
if: ${{ !cancelled() }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
verbose: true
files: coverage-llvm.lcov
flags: macos-llvm-c
name: macos-llvm-c
token: ${{ secrets.CODECOV_TOKEN }}
skip_validation: true

- name: Make python-only code coverage test report available
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-codecoverage-report-macOS
path: coverage_html_report
retention-days: 1

- name: Make HTML test report available
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/macos_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,21 @@ export CFLAGS="-O2 -pipe -ffp-contract=off -arch ${CONDA_ARCH} -DGL_SILENCE_DEPR
export CXXFLAGS="-O2 -pipe -ffp-contract=off -stdlib=libc++ -arch ${CONDA_ARCH} -Wall -Wextra -Wpedantic"
export CPPFLAGS="-isystem${CONDA_PREFIX}/include"

# LLVM source-based coverage instrumentation, opt-in via env var. The
# coverage-mapping section is generated before the LLVM optimizer runs, so
# optimization does not affect coverage quality (per clang's
# SourceBasedCodeCoverage docs) - keep the project's -O2. -fprofile-update=
# atomic makes counter increments atomic; without it, OpenMP parallel
# regions (e.g. r.univar's process_raster) lose counter updates across
# threads and the merged .profdata is non-deterministic.
if [ "${GRASS_LLVM_COVERAGE:-0}" = "1" ]; then
COV_FLAGS="-fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic"
CFLAGS="${CFLAGS} ${COV_FLAGS}"
CXXFLAGS="${CXXFLAGS} ${COV_FLAGS}"
LDFLAGS="${LDFLAGS:-} -fprofile-instr-generate -fcoverage-mapping"
export CFLAGS CXXFLAGS LDFLAGS
fi

./configure $CONFIGURE_FLAGS

EXEMPT=""
Expand Down
90 changes: 65 additions & 25 deletions .github/workflows/pytest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ jobs:
FORCE_COLOR: 1 # for software including pip: https://force-color.org/
CLICOLOR_FORCE: 1 # for other software including ninja: https://bixense.com/clicolors/
PYTHONWARNINGS: always
# Only exercise LLVM C coverage on the latest Python; the older one
# runs the default gcc/no-instrumentation build for speed.
GRASS_LLVM_COVERAGE: ${{ matrix.python-version == '3.13' && '1' || '0' }}

steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -54,6 +57,12 @@ jobs:
xargs -a <(awk '! /^ *(#|$)/' ".github/workflows/apt.txt") -r -- \
sudo apt-get install -y --no-install-recommends --no-install-suggests

- name: Install LLVM coverage toolchain
if: env.GRASS_LLVM_COVERAGE == '1'
run: |
sudo apt-get install -y --no-install-recommends --no-install-suggests \
clang llvm libomp-dev

- uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1

- name: Install Python dependencies
Expand Down Expand Up @@ -90,16 +99,29 @@ jobs:
- name: Test executing of the grass command
run: .github/workflows/test_simple.sh

- name: Prepare coverage directories
run: |
mkdir -p "${PWD}/llvm-cov-raw"
# Wipe stray default.profraw the build step left behind (Makefiles
# run instrumented tools to dump interface XML), so their build-
# time data can't contaminate the test-run coverage.
find . -name 'default.profraw' -type f -delete
# %p (process id) separates every subprocess; %9m caps the per-binary
# pool at 512 files so long suites don't create millions of tiny
# .profraw. Together they eliminate the write-collision that made
# earlier runs non-deterministic.
{
echo "LLVM_PROFILE_FILE=${PWD}/llvm-cov-raw/grass-%p-%9m.profraw"
echo "INITIAL_GISBASE=$(grass --config path)"
echo "INITIAL_PWD=${PWD}"
} >> "${GITHUB_ENV}"

- name: Run pytest with multiple workers in parallel
run: |
PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
INITIAL_GISBASE="$(grass --config path)"
INITIAL_PWD="${PWD}"
export PYTHONPATH
export LD_LIBRARY_PATH
export INITIAL_GISBASE
export INITIAL_PWD
pytest \
@.github/workflows/pytest_args_ci.txt \
@.github/workflows/pytest_args_cov.txt \
Expand All @@ -113,12 +135,8 @@ jobs:
run: |
PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
INITIAL_GISBASE="$(grass --config path)"
INITIAL_PWD="${PWD}"
export PYTHONPATH
export LD_LIBRARY_PATH
export INITIAL_GISBASE
export INITIAL_PWD
pytest \
@.github/workflows/pytest_args_ci.txt \
@.github/workflows/pytest_args_cov.txt \
Expand All @@ -129,29 +147,40 @@ jobs:
run: |
PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
INITIAL_GISBASE="$(grass --config path)"
INITIAL_PWD="${PWD}"
export PYTHONPATH
export LD_LIBRARY_PATH
export INITIAL_GISBASE
export INITIAL_PWD
pytest \
@.github/workflows/pytest_args_cov.txt \
@.github/workflows/pytest_args_gunittest.txt \
--junitxml=pytest.gunittest.junit.xml
- name: Fix non-standard installed script paths in coverage data

- name: Coverage disk usage (before combine)
if: ${{ !cancelled() }}
run: utils/coverage_disk_usage.sh

- name: Combine Python coverage and export XML
if: ${{ !cancelled() }}
run: |
PYTHONPATH="$(grass --config python_path):${PYTHONPATH}"
LD_LIBRARY_PATH="$(grass --config path)/lib:${LD_LIBRARY_PATH}"
INITIAL_GISBASE="$(grass --config path)"
INITIAL_PWD="${PWD}"
export PYTHONPATH
export LD_LIBRARY_PATH
export INITIAL_GISBASE
export INITIAL_PWD
python utils/coverage_mapper.py
coverage combine
coverage html
# coverage combine must run before coverage_mapper.py: parallel
# mode leaves .coverage.<host>.<pid>.<rand> partials and no
# .coverage file for the mapper to read. Tolerate "no data" so
# a build failure earlier in the job does not mask itself as a
# later failure here.
coverage combine || true
if [ -f .coverage ]; then
python utils/coverage_mapper.py
if [ -f .coverage.fixed_scripts ]; then
mv .coverage.fixed_scripts .coverage
fi
coverage html
coverage xml -o coverage.xml
else
echo "No .coverage data collected; skipping Python coverage export."
fi

- name: Merge and export LLVM C/C++ coverage
if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }}
run: utils/llvm_coverage_export.sh

- name: Upload test results to Codecov
if: ${{ !cancelled() }}
Expand All @@ -162,13 +191,24 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
report_type: test_results

- name: Upload coverage reports to Codecov
- name: Upload Python coverage to Codecov
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
verbose: true
files: coverage.xml
flags: pytest-python-${{ matrix.python-version }}
name: pytest-python-${{ matrix.python-version }}
token: ${{ secrets.CODECOV_TOKEN }}

- name: Upload LLVM C/C++ coverage to Codecov
if: ${{ !cancelled() && env.GRASS_LLVM_COVERAGE == '1' }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
verbose: true
files: coverage-llvm.lcov
flags: ubuntu-llvm-c
name: ubuntu-llvm-c-${{ matrix.python-version }}
token: ${{ secrets.CODECOV_TOKEN }}
- name: Make python-only code coverage test report available
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
Expand Down
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ lib/*/latex/
.coverage
.coverage.*
coverage.xml
*.profraw
*.profdata
coverage-llvm.lcov
llvm-cov-raw/
coverage_html_report/

# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos
# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos
Expand Down
Loading
Loading