From 0c05325c0ae67a8839348f300b00ac42509f50c8 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 20 Oct 2026 14:25:45 -0400 Subject: [PATCH] Read Elixir 1.20's test summary in the flake hunt The first weekly hunt measured the floor and pinned legs (40 runs, 0 failures) but counted every run on the head and box legs as failed: Elixir 1.20 prints its summary as "Result: 798 passed (...)", and "Result: 797/798 passed" when a test fails, where 1.17 and 1.18 print "786 tests, 0 failures". The script found no summary in that form, and a run with no summary is counted as failed, so 40 green runs read red. - tools/flake_hunt.sh: the summary read accepts both forms. The verdict is still the exit code; the summary only shows the suite reported. - tools/probe_flake_hunt.sh: the fake mix speaks 1.20's form, green and red. Measured locally: the probe's two new cases fail before the change and pass after. On Elixir 1.20.4 the real suite with a planted failing test is named with its replay (exit 1), and two runs on a clean tree exit 0. Signed-off-by: Ayla Croft --- tools/flake_hunt.sh | 4 +++- tools/probe_flake_hunt.sh | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/flake_hunt.sh b/tools/flake_hunt.sh index f7ccd8c..15e045d 100755 --- a/tools/flake_hunt.sh +++ b/tools/flake_hunt.sh @@ -27,7 +27,9 @@ for i in $(seq 1 "$runs"); do log="$out/run-$i-seed-$seed.txt" mix test --seed "$seed" > "$log" 2>&1 rc=$? - summary=$(grep -E '[0-9]+ tests?, [0-9]+ failures?' "$log" | tail -1) + # The summary line in either of ExUnit's forms: "786 tests, 0 failures" (to Elixir 1.19) or + # "Result: 798 passed (...)", "Result: 797/798 passed" (1.20 on). + summary=$(grep -E '[0-9]+ tests?, [0-9]+ failures?|^Result: [0-9]+(/[0-9]+)? passed' "$log" | tail -1) echo "run $i: seed $seed; exit $rc; ${summary:-no summary}" # A green exit with no summary measured nothing (the suite did not report), so it is not a pass. if [ "$rc" -ne 0 ] || [ -z "$summary" ]; then diff --git a/tools/probe_flake_hunt.sh b/tools/probe_flake_hunt.sh index 951c940..2b4364c 100755 --- a/tools/probe_flake_hunt.sh +++ b/tools/probe_flake_hunt.sh @@ -25,6 +25,9 @@ case $outcome in pass) printf 'Finished in 1.0 seconds\n11 properties, 786 tests, 0 failures\n'; exit 0 ;; fail) printf ' 1) test the restarted collector counts from no rows (BeamMCP.ObservedTest)\n test/beam_mcp/connectome/observed_test.exs:150\n\nFinished in 1.0 seconds\n11 properties, 786 tests, 1 failure\n'; exit 2 ;; silent) exit 0 ;; + # Elixir 1.20's summary (measured on 1.20.4): "Result: N passed (...)" green, "Result: P/T passed" red. + pass120) printf 'Finished in 1.0 seconds (0.5s async, 0.5s sync)\n\nResult: 798 passed (11 properties, 787 tests)\n'; exit 0 ;; + fail120) printf ' 1) test the restarted collector counts from no rows (BeamMCP.ObservedTest)\n test/beam_mcp/connectome/observed_test.exs:150\n\nFinished in 1.0 seconds\n\nResult: 797/798 passed\n'; exit 2 ;; esac MIX chmod +x "$work/bin/mix" @@ -46,5 +49,7 @@ run "H3 one red run among greens" 1 "pass fail pass" 3 "run 2: seed 1002; exit 2 run "H4 a green exit with no summary is not a pass" 1 "silent" 1 "no summary" "hunt: 1 run\(s\), 1 failed" run "H5 a run count that is not a positive integer" 2 "pass" 0 run "H6 the red run's whole output is kept" 1 "fail" 1 +run "H7 Elixir 1.20's green summary is a summary" 0 "pass120" 2 "run 1: seed 1001; exit 0; Result: 798 passed" "hunt: 2 run\(s\), 0 failed" +run "H8 Elixir 1.20's red run is named with its replay" 1 "pass120 fail120" 2 "run 2: seed 1002; exit 2; Result: 797/798 passed" "the restarted collector" "mix test --seed 1002" ls "$work/H6/out"/run-1-seed-1001.txt >/dev/null 2>&1 || { echo " missing: H6's kept output"; bad=1; } if [ "$bad" -eq 0 ]; then echo "PROBE OK: every run has its own seed; a red or silent run fails the hunt, named with its replay"; else echo "PROBE FAILED"; exit 1; fi