diff --git a/.github/workflows/flake-hunt.yml b/.github/workflows/flake-hunt.yml new file mode 100644 index 0000000..a211de0 --- /dev/null +++ b/.github/workflows/flake-hunt.yml @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# +# THE FLAKE HUNT: the suite, run many times a week on the gate's own four pairs and images, to +# find a test that fails only sometimes before a contributor does. A flaky test shows on one run +# in tens; the gate runs each leg once per push, so the three found so far were each one red leg +# that a rerun turned green, and a contributor reported others on their own machine that no run +# here reproduced (100 runs on two commits). This runs tools/flake_hunt.sh on each leg twice: +# with the runner's schedulers, and with two (`+S 2`), where a test that waits on time instead of +# on its condition is likeliest to lose the race. Each run is a fresh VM with a seed of its own; +# a red run names its tests and the command that replays it, and its output is kept with the run. +# +# NOT A REQUIRED CHECK, BY DESIGN: it runs on a schedule and by hand, never on a push or a pull +# request, so a flake it finds is a finding to fix, not a merge it blocks. A red hunt is a +# failed scheduled run, which GitHub reports to the maintainer. +# +# The matrix is the gate's (governance_test.exs holds the two equal), so a hunt measures the +# pairs the gate does, on the images the gate names. +name: flake-hunt + +on: + schedule: + - cron: "17 5 * * 1" + workflow_dispatch: + inputs: + runs: + description: "runs per scheduler setting on each leg" + required: false + default: "10" + +permissions: + contents: read + +jobs: + hunt: + name: Flake hunt (OTP ${{ matrix.otp }}, Elixir ${{ matrix.elixir }}, ${{ matrix.leg }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - { leg: floor, otp: "27", elixir: "1.17", os: ubuntu-24.04, version-type: loose } + - { leg: pinned, otp: "28", elixir: "1.18", os: ubuntu-24.04, version-type: loose } + - { leg: head, otp: "29", elixir: "1.20", os: ubuntu-24.04, version-type: loose } + - { leg: box, otp: "28.5.0.8", elixir: "1.20.4", os: ubuntu-26.04-arm, version-type: strict } + env: + # Through the environment, never inline in a script: the input is the dispatcher's text, + # and tools/flake_hunt.sh refuses anything but a positive integer. + RUNS: ${{ inputs.runs || '10' }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + otp-version: ${{ matrix.otp }} + elixir-version: ${{ matrix.elixir }} + version-type: ${{ matrix.version-type }} + - name: the pair setup-beam installed + run: | + erl -noshell -eval 'io:format("OTP ~s~n", [erlang:system_info(otp_release)]), halt().' + erl -noshell -eval 'ok = application:load(crypto), [{_, _, V}] = crypto:info_lib(), io:format("crypto linked to ~s~n", [V]), halt().' + elixir --version + - run: mix deps.get + - run: MIX_ENV=test mix compile + - name: the hunt's own probe (offline) + run: bash tools/probe_flake_hunt.sh + - name: the suite, RUNS times, the runner's schedulers + run: ./tools/flake_hunt.sh "$RUNS" "$RUNNER_TEMP/hunt/default" + - name: the suite, RUNS times, two schedulers + if: ${{ !cancelled() }} + env: + ERL_FLAGS: "+S 2" + run: ./tools/flake_hunt.sh "$RUNS" "$RUNNER_TEMP/hunt/s2" + - name: the red runs' output, kept with the run + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: flake-hunt-${{ matrix.leg }} + path: ${{ runner.temp }}/hunt + if-no-files-found: ignore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a2c17d..e8f0582 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,14 @@ before the fix (below). A change to the wire or to the host contract is also mea mutation, as `CONVENTIONS.md`'s tier rule says. A pull request without its tests is sent back, whoever wrote it. +**A test that fails only sometimes is a bug, and worth an issue.** If `mix test` fails on a +`main` you have not changed, and the test passes alone, please open an issue with the test's +name and the seed: `tools/flake_hunt.sh 20` runs the suite twenty times, each with a seed of +its own, and prints each red run's failed tests with the command that replays it +(`ERL_FLAGS="+S 2"` runs it with two schedulers, where a test that waits on time rather than +on its condition loses most often). CI runs the same hunt weekly on the gate's four pairs +(`.github/workflows/flake-hunt.yml`). + ## Code review **How it is done.** Every change reaches `main` through a pull request, the maintainer's diff --git a/test/beam_mcp/governance_test.exs b/test/beam_mcp/governance_test.exs index ce0f2ee..eff8720 100644 --- a/test/beam_mcp/governance_test.exs +++ b/test/beam_mcp/governance_test.exs @@ -103,6 +103,27 @@ defmodule BeamMCP.GovernanceTest do "gate.yml's toolchain step does not print crypto:info_lib()" end + # The hunt measures what the gate measures: a pair added to the gate and not to the hunt would + # be hunted nowhere, and a pair only in the hunt would be one the gate does not support. + test "the flake hunt runs the gate's four pairs on the gate's images, and never on a push or a pull request" do + pairs = fn wf -> + File.read!(wf) + |> String.split("\n") + |> Enum.filter(&(&1 =~ ~r/^\s+- \{ leg:/)) + |> Enum.map(&(&1 |> String.trim() |> String.replace(~r/\s+/, " "))) + |> Enum.sort() + end + + gate = pairs.(".github/workflows/gate.yml") + assert length(gate) == 4 + assert pairs.(".github/workflows/flake-hunt.yml") == gate + + [triggers, _] = + String.split(File.read!(".github/workflows/flake-hunt.yml"), ~r/^permissions:/m, parts: 2) + + refute triggers =~ ~r/^\s+(push|pull_request)/m + end + test "every workflow declares top-level permissions, and write permissions live on the job that needs them" do for wf <- @workflows do text = File.read!(wf) diff --git a/tools/flake_hunt.sh b/tools/flake_hunt.sh new file mode 100755 index 0000000..f7ccd8c --- /dev/null +++ b/tools/flake_hunt.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# +# The suite, run many times to find a test that fails only sometimes. A flaky test shows on one +# run in tens: the three found so far each failed one CI leg once, passed on that leg's rerun, +# and passed alone, so a single green run says nothing about them. This runs `mix test` +# times, each in a fresh VM (not `--repeat-until-failure`, which reruns in one VM, where state +# one run leaves can fail the next for a reason that is not the test's), each with a seed of +# its own: ExUnit does not print the seed it chose, and a failure that cannot be replayed +# cannot be fixed. A red run prints its failed tests and the command that replays it, and its +# whole output is kept; a green run's is dropped. ERL_FLAGS passes through to every run, so +# `ERL_FLAGS="+S 2"` hunts with two schedulers. +# +# tools/flake_hunt.sh [] +# env: FLAKE_HUNT_SEED_BASE (default: the clock); run i uses seed base + i. +# exit 0: every run green; 1: a run red, or a run that printed no summary; 2: usage. +set -uo pipefail +runs=${1:-} +case $runs in '' | *[!0-9]* | 0) echo "usage: tools/flake_hunt.sh [] (runs: a positive integer, got '$runs')" >&2; exit 2 ;; esac +out=${2:-${TMPDIR:-/tmp}/beam_mcp-flake-hunt} +mkdir -p "$out" +base=${FLAKE_HUNT_SEED_BASE:-$(($(date +%s) % 1000000))} +failed=0 +for i in $(seq 1 "$runs"); do + seed=$((base + i)) + log="$out/run-$i-seed-$seed.txt" + mix test --seed "$seed" > "$log" 2>&1 + rc=$? + summary=$(grep -E '[0-9]+ tests?, [0-9]+ failures?' "$log" | tail -1) + echo "run $i: seed $seed; exit $rc; ${summary:-no summary}" + # A green exit with no summary measured nothing (the suite did not report), so it is not a pass. + if [ "$rc" -ne 0 ] || [ -z "$summary" ]; then + failed=$((failed + 1)) + grep -E '^ +[0-9]+\) (test|property|doctest) ' "$log" | sed -E 's/^ +/ /' + echo " replay: ERL_FLAGS='${ERL_FLAGS:-}' mix test --seed $seed (output: $log)" + else + rm -f "$log" + fi +done +echo "hunt: $runs run(s), $failed failed; ERL_FLAGS='${ERL_FLAGS:-}'" +[ "$failed" -eq 0 ] diff --git a/tools/probe_flake_hunt.sh b/tools/probe_flake_hunt.sh new file mode 100755 index 0000000..951c940 --- /dev/null +++ b/tools/probe_flake_hunt.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Probe: does tools/flake_hunt.sh give every run a seed of its own and pass it to `mix test`, +# fail the hunt on one red run among greens and name that run's tests with the command that +# replays it, count a run that printed no summary as failed (a green exit with nothing +# measured is not a pass), and refuse a run count that is not a positive integer -- with a +# fake `mix` on PATH that answers a scripted sequence of runs? The suite is never run; the +# probe is offline and takes a second. +# +# tools/probe_flake_hunt.sh +set -uo pipefail +here=$(cd "$(dirname "$0")/.." && pwd) +work=$(mktemp -d "${TMPDIR:-/tmp}/beam_mcp-probe-hunt.XXXXXX"); trap 'rm -rf "$work"' EXIT +mkdir -p "$work/bin" +# The fake mix: records its arguments, then plays the outcome named on the counter's line of +# $PROBE_SCRIPT (pass, fail or silent), holding the last line. +cat > "$work/bin/mix" <<'MIX' +#!/usr/bin/env bash +n=$(cat "$PROBE_DIR/n"); echo $((n + 1)) > "$PROBE_DIR/n" +echo "$*" >> "$PROBE_DIR/argv" +outcome=$(sed -n "$((n + 1))p" "$PROBE_SCRIPT"); [ -n "$outcome" ] || outcome=$(tail -1 "$PROBE_SCRIPT") +case $outcome in + pass) printf 'Finished in 1.0 seconds\n11 properties, 786 tests, 0 failures\n'; exit 0 ;; + fail) printf ' 1) test the restarted collector counts from no rows (BeamMCP.ObservedTest)\n test/beam_mcp/connectome/observed_test.exs:150\n\nFinished in 1.0 seconds\n11 properties, 786 tests, 1 failure\n'; exit 2 ;; + silent) exit 0 ;; +esac +MIX +chmod +x "$work/bin/mix" +bad=0 +run() { # run [...] + local name=$1 want=$2 outcomes=$3 runs=$4; shift 4 + d=$work/$(echo "$name" | cut -d' ' -f1); mkdir -p "$d"; echo 0 > "$d/n"; : > "$d/argv" + printf '%s\n' $outcomes > "$d/script" + out=$(PATH="$work/bin:$PATH" PROBE_DIR=$d PROBE_SCRIPT=$d/script FLAKE_HUNT_SEED_BASE=1000 \ + bash "$here/tools/flake_hunt.sh" "$runs" "$d/out" 2>&1); rc=$? + ok=1; [ "$rc" -eq "$want" ] || ok=0 + for pattern in "$@"; do printf '%s\n' "$out" | cat - "$d/argv" | grep -qE -- "$pattern" || { ok=0; echo " missing: $pattern"; }; done + printf ' %-58s exit %s (want %s)%s\n' "$name" "$rc" "$want" "$([ $ok -eq 1 ] || echo ' <-- WRONG')" + [ $ok -eq 1 ] || bad=1 +} +run "H1 three green runs" 0 "pass" 3 "hunt: 3 run\(s\), 0 failed" +run "H2 each run its own seed, passed to mix test" 0 "pass" 3 "^test --seed 1001$" "^test --seed 1002$" "^test --seed 1003$" +run "H3 one red run among greens" 1 "pass fail pass" 3 "run 2: seed 1002; exit 2" "the restarted collector counts from no rows" "mix test --seed 1002" "hunt: 3 run\(s\), 1 failed" +run "H4 a green exit with no summary is not a pass" 1 "silent" 1 "no summary" "hunt: 1 run\(s\), 1 failed" +run "H5 a run count that is not a positive integer" 2 "pass" 0 +run "H6 the red run's whole output is kept" 1 "fail" 1 +ls "$work/H6/out"/run-1-seed-1001.txt >/dev/null 2>&1 || { echo " missing: H6's kept output"; bad=1; } +if [ "$bad" -eq 0 ]; then echo "PROBE OK: every run has its own seed; a red or silent run fails the hunt, named with its replay"; else echo "PROBE FAILED"; exit 1; fi