#60: Add Mull mutation testing workflow

.github/workflows/check_bazel_tests.yml

@@ -147,7 +147,7 @@ jobs:
           OPENSSL_LIBRARY_PREFIX: "/usr/lib/x86_64-linux-gnu"
           OPENSSL_INCLUDE_PREFIX: "/usr/include/openssl"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         if: failure()
         with:
           name: "${{ matrix.name }}"

.github/workflows/check_mull.yml

@@ -0,0 +1,74 @@
+name: Check Mull Mutation Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+env:
+  USE_BAZEL_VERSION: 8.3.1
+
+jobs:
+  discover_mull_targets:
+    runs-on: ubuntu-24.04
+    outputs:
+      mull_matrix: ${{ steps.discover.outputs.mull_matrix }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: bazel-contrib/setup-bazel@8cb04a772ab4c1eb984e9c1b493a182e96c5e425 # 0.19.0
+        with:
+          bazelisk-cache: true
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v9
+        with:
+          python-version: "3.10"
+          poetry-version: "2.3.0"
+
+      - name: Install Poetry dependencies
+        run: poetry install --with dev
+
+      - name: Discover Mull targets
+        id: discover
+        run: poetry run -- nox --sessions=mull-targets -- --github-output-var mull_matrix
+
+  mull:
+    needs: discover_mull_targets
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.discover_mull_targets.outputs.mull_matrix) }}
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: bazel-contrib/setup-bazel@8cb04a772ab4c1eb984e9c1b493a182e96c5e425 # 0.19.0
+        with:
+          bazelisk-cache: true
+
+      - name: Install LLVM and Mull
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang-20
+          curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
+          sudo apt-get install -y mull-20=0.34.1
+          mull-runner-20 --version
+
+      - name: Setup Python & Poetry Environment
+        uses: exasol/python-toolbox/.github/actions/python-environment@v9
+        with:
+          python-version: "3.10"
+          poetry-version: "2.3.0"
+
+      - name: Install Poetry dependencies
+        run: poetry install --with dev
+
+      - name: Run Mull mutation tests
+        run: poetry run -- nox --sessions=mull -- --target "${{ matrix.target }}"
+
+      - name: Upload Mull reports
+        if: always()
+        uses: actions/upload-artifact@v6
+        with:
+          name: mull-reports-${{ matrix.target }}
+          path: .build_output/mull/
+          if-no-files-found: warn

doc/changes/unreleased.md

@@ -19,6 +19,11 @@ n/a
 
 ## Internal
 
+* #60: Added Mull mutation testing workflow and report-viewing documentation
+  for v2; targets without generated mutants now produce warnings instead of
+  failing the workflow, and separated Linux EventFd code and factory-based
+  Linux queue construction from the generic waitable-queue mutation target;
+  updated artifact uploads to the Node.js 24-compatible action version
 * #64: Added GoogleTest, GoogleMock, and Google Benchmark support for v2 tests
 * #57: Defined and enforced public v2 C++ coding style
 * #51: Added agent and contributor guidance for v1/v2 development, SLC workflows, CI testing, and PR conventions

doc/developer_guide/slc.md

@@ -20,8 +20,9 @@ export DOCKER_HOST="$(limactl list docker-udf-client \
 ```
 
 After setting `DOCKER_HOST`, run normal `exaslct` commands from the host. The
-Lima template provides Bazel, Protobuf, ZeroMQ, SWIG, Python, Poetry, and the
-native dependency environment used by the v1 build.
+Lima template provides Bazel, Protobuf, ZeroMQ, SWIG, Python, Poetry, the LLVM
+20/Mull mutation-testing toolchain, and the native dependency environment used
+by the v1 build.
 
 Export the checked-in v1 flavor to a local archive with:
 

doc/developer_guide/v2/v2.md

@@ -3,7 +3,7 @@
 The v2 developer documentation is split into focused guides:
 
 - [Build and test](v2_build_and_test.md) — Bazel setup and test execution.
-- [Code quality](v2_code_quality.md) — clang-tidy and clang-format checks.
+- [Code quality](v2_code_quality.md) — clang-tidy, clang-format, and Mull mutation checks.
 - [Coverage](v2_coverage.md) — local LCOV and HTML coverage reports.
 - [Coding style](v2_coding_style.md) — public C++ conventions for v2 code.
 - [Fuzzing](v2_fuzzing.md) — libFuzzer targets, Nox campaigns, and regression

doc/developer_guide/v2/v2_code_quality.md

@@ -81,3 +81,66 @@ alias(
     tags = ["noclangtidy"],
 )
 ```
+
+## Mutation testing with Mull
+
+Mutation testing for the functional v2 C++ tests uses [Mull](https://mull-project.com/)
+with the pinned Mull 0.34.1 release and matching LLVM 20 toolchain. The
+`docker-udf-client` Lima template provisions these tools automatically; on
+other environments install the LLVM 20 compiler and `mull-20`, then verify
+that `mull-runner-20` and `/usr/lib/mull-ir-frontend-20` are available.
+
+Run the mutation session from the repository root:
+
+```bash
+poetry run -- nox --sessions=mull
+```
+
+If Bazel reports stale or incompatible Mull output from a previous VM or host
+build, clean the Mull-specific output roots and reports before retrying:
+
+```bash
+poetry run -- nox --sessions=mull-clean
+```
+
+If the Bazel executable is named `bazelisk`, run:
+`BAZEL=bazelisk poetry run -- nox --sessions=mull`.
+
+The session discovers Bazel `cc_test` targets and runs each eligible target with
+Mull instrumentation. It writes reports to `.build_output/mull/` and enforces
+an 80% mutation-score threshold for every target that produces at least one
+mutant. Targets for which Mull produces no mutants emit a warning and succeed;
+the warning is shown in local Nox output and as a GitHub Actions annotation.
+Build failures, test failures, invalid reports, and mutation scores below 80%
+remain errors. The LLVM major version can be changed with `MULL_LLVM_VERSION`;
+custom tool paths can be supplied with `MULL_CXX`, `MULL_RUNNER`, and
+`MULL_IR_FRONTEND`. The C compiler used by Bazel can be overridden with
+`MULL_CC`.
+
+### Viewing Mull HTML reports
+
+Mull writes an HTML page and its matching JSON data file for each target to
+`.build_output/mull/`. The HTML page loads the JSON file dynamically, so serve
+the directory over HTTP instead of opening the page directly with `file://`:
+
+```bash
+cd .build_output/mull
+python3 -m http.server 8000
+```
+
+Then open `http://localhost:8000/<target>.html`, for example
+`http://localhost:8000/event_fd_test.html`. The page also loads the Mutation
+Testing Elements JavaScript from its configured CDN and therefore needs network
+access.
+
+The Mull workflow uploads these files as the
+`mull-reports-<target>` artifact. Download and extract the artifact, start the
+same HTTP server in the extracted directory, and open the target's HTML page in
+your browser.
+
+Mutation testing is not reliable for C++ template implementations or tests
+that only exercise third-party dependencies. Keep those tests in normal Bazel
+test coverage and exclude them from Mull with the `no-mull` tag. Production
+implementation units with Mull-compatible non-template code should have a
+dedicated test target that remains in the mutation matrix.
+

ext/lima_vm_templates/docker-udf-client.yaml

@@ -24,7 +24,7 @@
 minimumLimaVersion: 1.1.0
 
 base:
-- template://_images/ubuntu-lts
+- template://_images/ubuntu-24.04
 - template://_default/mounts
 
 # containerd is managed by Docker, not by Lima, so the values are set to false here.
@@ -58,17 +58,28 @@ provision:
     curl -fsSL https://get.docker.com | sh
 - mode: system
   script: |
+   #!/bin/bash
+   set -eux -o pipefail
    export DEBIAN_FRONTEND=noninteractive
+   curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash
    apt update && apt install -y protobuf-compiler libzmq3-dev openjdk-17-jdk build-essential git python3.12-dev python3-pip libpcre3-dev clang-tidy-20 lcov
+   apt install -y clang-20 mull-20=0.34.1
+   mull-runner-20 --version
+   test -x /usr/lib/mull-ir-frontend-20
    curl -L  https://github.com/bazelbuild/bazelisk/releases/download/v1.27.0/bazelisk-linux-amd64 -o /usr/bin/bazel
    chmod +x /usr/bin/bazel
    pip install --break-system-packages numpy
    curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && tar zxf swig-2.0.4.tar.gz && (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && rm -rf swig-2.0.4 swig-2.0.4.tar.gz
-   curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local/bin/poetry python3 -
+   poetry_home=/opt/poetry
+   if [ ! -x "$poetry_home/bin/poetry" ]; then
+     curl -sSL https://install.python-poetry.org | POETRY_HOME="$poetry_home" python3 - --version 2.3.0
+   fi
+   ln -sfn "$poetry_home/bin/poetry" /usr/local/bin/poetry
+   /usr/local/bin/poetry --version
 - mode: system
   script: |
    #!/bin/sh
-   echo 'export PATH="/usr/local/bin/poetry/bin:$PATH"' > /etc/profile.d/poetry.sh
+   echo 'export PATH="/opt/poetry/bin:$PATH"' > /etc/profile.d/poetry.sh
    chmod +x /etc/profile.d/poetry.sh
 probes:
 - script: |

mull.yml

@@ -0,0 +1,11 @@
+# Static Mull configuration. The Nox session prepends Bazel-generated
+# includePaths before passing this file to Mull.
+excludePaths:
+  - "(^|.*/).*_test\\.(cc|cpp)"
+  - "(^|.*/).*_benchmark\\.cc"
+  - "(^|.*/)third_party/.*"
+  - ".*bazel-out/.*"
+
+parallelization:
+  workers: 2
+  executionWorkers: 2