HackTuah · Oct 20, 2026 · Oct 20, 2026
@@ -0,0 +1,80 @@
+# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+#
+# THE FLAKE HUNT: the suite, run many times a week on the gate's own four pairs and images, to
+# find a test that fails only sometimes before a contributor does. A flaky test shows on one run
+# in tens; the gate runs each leg once per push, so the three found so far were each one red leg
+# that a rerun turned green, and a contributor reported others on their own machine that no run
+# here reproduced (100 runs on two commits). This runs tools/flake_hunt.sh on each leg twice:
+# with the runner's schedulers, and with two (`+S 2`), where a test that waits on time instead of
+# on its condition is likeliest to lose the race. Each run is a fresh VM with a seed of its own;
+# a red run names its tests and the command that replays it, and its output is kept with the run.
+#
+# NOT A REQUIRED CHECK, BY DESIGN: it runs on a schedule and by hand, never on a push or a pull
+# request, so a flake it finds is a finding to fix, not a merge it blocks. A red hunt is a
+# failed scheduled run, which GitHub reports to the maintainer.
+#
+# The matrix is the gate's (governance_test.exs holds the two equal), so a hunt measures the
+# pairs the gate does, on the images the gate names.
+name: flake-hunt
+
+on:
+  schedule:
+    - cron: "17 5 * * 1"
+  workflow_dispatch:
+    inputs:
+      runs:
+        description: "runs per scheduler setting on each leg"
+        required: false
+        default: "10"
+
+permissions:
+  contents: read
+
+jobs:
+  hunt:
+    name: Flake hunt (OTP ${{ matrix.otp }}, Elixir ${{ matrix.elixir }}, ${{ matrix.leg }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { leg: floor,  otp: "27", elixir: "1.17", os: ubuntu-24.04, version-type: loose }
+          - { leg: pinned, otp: "28", elixir: "1.18", os: ubuntu-24.04, version-type: loose }
+          - { leg: head,   otp: "29", elixir: "1.20", os: ubuntu-24.04, version-type: loose }
+          - { leg: box,    otp: "28.5.0.8", elixir: "1.20.4", os: ubuntu-26.04-arm, version-type: strict }
+    env:
+      # Through the environment, never inline in a script: the input is the dispatcher's text,
+      # and tools/flake_hunt.sh refuses anything but a positive integer.
+      RUNS: ${{ inputs.runs || '10' }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          otp-version: ${{ matrix.otp }}
+          elixir-version: ${{ matrix.elixir }}
+          version-type: ${{ matrix.version-type }}
+      - name: the pair setup-beam installed
+        run: |
+          erl -noshell -eval 'io:format("OTP ~s~n", [erlang:system_info(otp_release)]), halt().'
+          erl -noshell -eval 'ok = application:load(crypto), [{_, _, V}] = crypto:info_lib(), io:format("crypto linked to ~s~n", [V]), halt().'
+          elixir --version
+      - run: mix deps.get
+      - run: MIX_ENV=test mix compile
+      - name: the hunt's own probe (offline)
+        run: bash tools/probe_flake_hunt.sh
+      - name: the suite, RUNS times, the runner's schedulers
+        run: ./tools/flake_hunt.sh "$RUNS" "$RUNNER_TEMP/hunt/default"
+      - name: the suite, RUNS times, two schedulers
+        if: ${{ !cancelled() }}
+        env:
+          ERL_FLAGS: "+S 2"
+        run: ./tools/flake_hunt.sh "$RUNS" "$RUNNER_TEMP/hunt/s2"
+      - name: the red runs' output, kept with the run
+        if: failure()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: flake-hunt-${{ matrix.leg }}
+          path: ${{ runner.temp }}/hunt
+          if-no-files-found: ignore
@@ -63,6 +63,14 @@ before the fix (below). A change to the wire or to the host contract is also mea
 mutation, as `CONVENTIONS.md`'s tier rule says. A pull request without its tests is sent
 back, whoever wrote it.
 
+**A test that fails only sometimes is a bug, and worth an issue.** If `mix test` fails on a
+`main` you have not changed, and the test passes alone, please open an issue with the test's
+name and the seed: `tools/flake_hunt.sh 20` runs the suite twenty times, each with a seed of
+its own, and prints each red run's failed tests with the command that replays it
+(`ERL_FLAGS="+S 2"` runs it with two schedulers, where a test that waits on time rather than
+on its condition loses most often). CI runs the same hunt weekly on the gate's four pairs
+(`.github/workflows/flake-hunt.yml`).
+
 ## Code review
 
 **How it is done.** Every change reaches `main` through a pull request, the maintainer's

@@ -103,6 +103,27 @@ defmodule BeamMCP.GovernanceTest do
            "gate.yml's toolchain step does not print crypto:info_lib()"
   end
 
+  # The hunt measures what the gate measures: a pair added to the gate and not to the hunt would
+  # be hunted nowhere, and a pair only in the hunt would be one the gate does not support.
+  test "the flake hunt runs the gate's four pairs on the gate's images, and never on a push or a pull request" do
+    pairs = fn wf ->
+      File.read!(wf)
+      |> String.split("\n")
+      |> Enum.filter(&(&1 =~ ~r/^\s+- \{ leg:/))
+      |> Enum.map(&(&1 |> String.trim() |> String.replace(~r/\s+/, " ")))
+      |> Enum.sort()
+    end
+
+    gate = pairs.(".github/workflows/gate.yml")
+    assert length(gate) == 4
+    assert pairs.(".github/workflows/flake-hunt.yml") == gate
+
+    [triggers, _] =
+      String.split(File.read!(".github/workflows/flake-hunt.yml"), ~r/^permissions:/m, parts: 2)
+
+    refute triggers =~ ~r/^\s+(push|pull_request)/m
+  end
+
   test "every workflow declares top-level permissions, and write permissions live on the job that needs them" do
     for wf <- @workflows do
       text = File.read!(wf)

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+#
+# The suite, run many times to find a test that fails only sometimes. A flaky test shows on one
+# run in tens: the three found so far each failed one CI leg once, passed on that leg's rerun,
+# and passed alone, so a single green run says nothing about them. This runs `mix test` <runs>
+# times, each in a fresh VM (not `--repeat-until-failure`, which reruns in one VM, where state
+# one run leaves can fail the next for a reason that is not the test's), each with a seed of
+# its own: ExUnit does not print the seed it chose, and a failure that cannot be replayed
+# cannot be fixed. A red run prints its failed tests and the command that replays it, and its
+# whole output is kept; a green run's is dropped. ERL_FLAGS passes through to every run, so
+# `ERL_FLAGS="+S 2"` hunts with two schedulers.
+#
+#     tools/flake_hunt.sh <runs> [<out-dir>]
+# env: FLAKE_HUNT_SEED_BASE (default: the clock); run i uses seed base + i.
+# exit 0: every run green; 1: a run red, or a run that printed no summary; 2: usage.
+set -uo pipefail
+runs=${1:-}
+case $runs in '' | *[!0-9]* | 0) echo "usage: tools/flake_hunt.sh <runs> [<out-dir>] (runs: a positive integer, got '$runs')" >&2; exit 2 ;; esac
+out=${2:-${TMPDIR:-/tmp}/beam_mcp-flake-hunt}
+mkdir -p "$out"
+base=${FLAKE_HUNT_SEED_BASE:-$(($(date +%s) % 1000000))}
+failed=0
+for i in $(seq 1 "$runs"); do
+  seed=$((base + i))
+  log="$out/run-$i-seed-$seed.txt"
+  mix test --seed "$seed" > "$log" 2>&1
+  rc=$?
+  summary=$(grep -E '[0-9]+ tests?, [0-9]+ failures?' "$log" | tail -1)
+  echo "run $i: seed $seed; exit $rc; ${summary:-no summary}"
+  # A green exit with no summary measured nothing (the suite did not report), so it is not a pass.
+  if [ "$rc" -ne 0 ] || [ -z "$summary" ]; then
+    failed=$((failed + 1))
+    grep -E '^ +[0-9]+\) (test|property|doctest) ' "$log" | sed -E 's/^ +/    /'
+    echo "    replay: ERL_FLAGS='${ERL_FLAGS:-}' mix test --seed $seed   (output: $log)"
+  else
+    rm -f "$log"
+  fi
+done
+echo "hunt: $runs run(s), $failed failed; ERL_FLAGS='${ERL_FLAGS:-}'"
+[ "$failed" -eq 0 ]
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 Sudo Apt Holdings LLC
+# SPDX-License-Identifier: Apache-2.0
+#
+# Probe: does tools/flake_hunt.sh give every run a seed of its own and pass it to `mix test`,
+# fail the hunt on one red run among greens and name that run's tests with the command that
+# replays it, count a run that printed no summary as failed (a green exit with nothing
+# measured is not a pass), and refuse a run count that is not a positive integer -- with a
+# fake `mix` on PATH that answers a scripted sequence of runs? The suite is never run; the
+# probe is offline and takes a second.
+#
+#     tools/probe_flake_hunt.sh
+set -uo pipefail
+here=$(cd "$(dirname "$0")/.." && pwd)
+work=$(mktemp -d "${TMPDIR:-/tmp}/beam_mcp-probe-hunt.XXXXXX"); trap 'rm -rf "$work"' EXIT
+mkdir -p "$work/bin"
+# The fake mix: records its arguments, then plays the outcome named on the counter's line of
+# $PROBE_SCRIPT (pass, fail or silent), holding the last line.
+cat > "$work/bin/mix" <<'MIX'
+#!/usr/bin/env bash
+n=$(cat "$PROBE_DIR/n"); echo $((n + 1)) > "$PROBE_DIR/n"
+echo "$*" >> "$PROBE_DIR/argv"
+outcome=$(sed -n "$((n + 1))p" "$PROBE_SCRIPT"); [ -n "$outcome" ] || outcome=$(tail -1 "$PROBE_SCRIPT")
+case $outcome in
+  pass) printf 'Finished in 1.0 seconds\n11 properties, 786 tests, 0 failures\n'; exit 0 ;;
+  fail) printf '  1) test the restarted collector counts from no rows (BeamMCP.ObservedTest)\n     test/beam_mcp/connectome/observed_test.exs:150\n\nFinished in 1.0 seconds\n11 properties, 786 tests, 1 failure\n'; exit 2 ;;
+  silent) exit 0 ;;
+esac
+MIX
+chmod +x "$work/bin/mix"
+bad=0
+run() { # run <case> <want exit> <outcomes, space separated> <runs> [<grep the output must match>...]
+  local name=$1 want=$2 outcomes=$3 runs=$4; shift 4
+  d=$work/$(echo "$name" | cut -d' ' -f1); mkdir -p "$d"; echo 0 > "$d/n"; : > "$d/argv"
+  printf '%s\n' $outcomes > "$d/script"
+  out=$(PATH="$work/bin:$PATH" PROBE_DIR=$d PROBE_SCRIPT=$d/script FLAKE_HUNT_SEED_BASE=1000 \
+    bash "$here/tools/flake_hunt.sh" "$runs" "$d/out" 2>&1); rc=$?
+  ok=1; [ "$rc" -eq "$want" ] || ok=0
+  for pattern in "$@"; do printf '%s\n' "$out" | cat - "$d/argv" | grep -qE -- "$pattern" || { ok=0; echo "    missing: $pattern"; }; done
+  printf '  %-58s exit %s (want %s)%s\n' "$name" "$rc" "$want" "$([ $ok -eq 1 ] || echo '  <-- WRONG')"
+  [ $ok -eq 1 ] || bad=1
+}
+run "H1 three green runs" 0 "pass" 3 "hunt: 3 run\(s\), 0 failed"
+run "H2 each run its own seed, passed to mix test" 0 "pass" 3 "^test --seed 1001$" "^test --seed 1002$" "^test --seed 1003$"
+run "H3 one red run among greens" 1 "pass fail pass" 3 "run 2: seed 1002; exit 2" "the restarted collector counts from no rows" "mix test --seed 1002" "hunt: 3 run\(s\), 1 failed"
+run "H4 a green exit with no summary is not a pass" 1 "silent" 1 "no summary" "hunt: 1 run\(s\), 1 failed"
+run "H5 a run count that is not a positive integer" 2 "pass" 0
+run "H6 the red run's whole output is kept" 1 "fail" 1
+ls "$work/H6/out"/run-1-seed-1001.txt >/dev/null 2>&1 || { echo "    missing: H6's kept output"; bad=1; }
+if [ "$bad" -eq 0 ]; then echo "PROBE OK: every run has its own seed; a red or silent run fails the hunt, named with its replay"; else echo "PROBE FAILED"; exit 1; fi