From aecaad043c3de928cbed8a56b7d1472527de3d1a Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 12:06:56 +0000 Subject: [PATCH 1/9] Diagnose the Read the Docs timeout [skip ci] Not for merging. Stops the build before the time limit so that its log survives, with elapsed seconds on each line and a probe of load and memory. Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cc7adf1..c998254 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,4 +19,14 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html + # Diagnostic, not for merging: stop before Read the Docs' 15-minute limit, so that + # the log survives, with elapsed seconds on each line and a probe of load and memory. + - nproc; free -m; cat /proc/loadavg + - | + set -o pipefail + (while sleep 60; do echo "[probe] $(cat /proc/loadavg) $(free -m | sed -n 2p)"; done) & + probe=$! + PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH timeout -k 10 840 uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html 2>&1 | perl -pe 'BEGIN{$|=1;$t=time} printf "%5d ", time-$t' + status=$? + kill $probe + exit $status From 32cdb4bc9fa1cbf6df5b65c205f7687c4a300442 Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 12:09:12 +0000 Subject: [PATCH 2/9] Run the diagnostic under sh, which has no pipefail [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c998254..b5ccbec 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -23,10 +23,7 @@ build: # the log survives, with elapsed seconds on each line and a probe of load and memory. - nproc; free -m; cat /proc/loadavg - | - set -o pipefail (while sleep 60; do echo "[probe] $(cat /proc/loadavg) $(free -m | sed -n 2p)"; done) & probe=$! PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH timeout -k 10 840 uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html 2>&1 | perl -pe 'BEGIN{$|=1;$t=time} printf "%5d ", time-$t' - status=$? kill $probe - exit $status From 1e1016053357e115a061f3d2e55adae527bf9848 Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 12:48:38 +0000 Subject: [PATCH 3/9] Keep the diagnostic's log whatever the build does [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b5ccbec..15986cd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,11 +19,21 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - # Diagnostic, not for merging: stop before Read the Docs' 15-minute limit, so that - # the log survives, with elapsed seconds on each line and a probe of load and memory. + # Diagnostic, not for merging: the build writes to a file while the foreground loop + # reports progress, so the log survives; at 780 s it dumps the Python stacks and stops. - nproc; free -m; cat /proc/loadavg - | - (while sleep 60; do echo "[probe] $(cat /proc/loadavg) $(free -m | sed -n 2p)"; done) & - probe=$! - PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH timeout -k 10 840 uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html 2>&1 | perl -pe 'BEGIN{$|=1;$t=time} printf "%5d ", time-$t' - kill $probe + log=/tmp/sphinx.log + : > $log + (PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > $log 2>&1; echo "sphinx exit $?" >> $log) & + n=0 + while [ $n -lt 780 ] && ! grep -q '^sphinx exit ' $log; do + sleep 20 + n=$((n+20)) + echo "[$n] load $(cut -d' ' -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s ' ' | cut -d' ' -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 90)" + done + ps -eo pid,ppid,stat,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-170 + pkill -ABRT -f '[b]in/sphinx-build' && sleep 3 + echo "=== log" + cat $log + exit 1 From 88562069b7284bdbb5502895625a1869e991775d Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 12:50:38 +0000 Subject: [PATCH 4/9] Hold no single quotes in the diagnostic, as Read the Docs quotes it [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 15986cd..c14507c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -21,19 +21,20 @@ build: html: # Diagnostic, not for merging: the build writes to a file while the foreground loop # reports progress, so the log survives; at 780 s it dumps the Python stacks and stops. + # Read the Docs wraps a command in single quotes, so it must hold none. - nproc; free -m; cat /proc/loadavg - | log=/tmp/sphinx.log : > $log (PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > $log 2>&1; echo "sphinx exit $?" >> $log) & n=0 - while [ $n -lt 780 ] && ! grep -q '^sphinx exit ' $log; do + while [ $n -lt 780 ] && ! grep -q "^sphinx exit " $log; do sleep 20 n=$((n+20)) - echo "[$n] load $(cut -d' ' -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s ' ' | cut -d' ' -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 90)" + echo "[$n] load $(cut -d" " -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s " " | cut -d" " -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 90)" done ps -eo pid,ppid,stat,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-170 - pkill -ABRT -f '[b]in/sphinx-build' && sleep 3 + pkill -ABRT -f "[b]in/sphinx-build" && sleep 3 echo "=== log" cat $log exit 1 From 982c88128be9d579965f55f9ce5fb32c002a1fbe Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 13:02:14 +0000 Subject: [PATCH 5/9] Test the build with its output sent to a file [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c14507c..182e376 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,22 +19,6 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - # Diagnostic, not for merging: the build writes to a file while the foreground loop - # reports progress, so the log survives; at 780 s it dumps the Python stacks and stops. - # Read the Docs wraps a command in single quotes, so it must hold none. - - nproc; free -m; cat /proc/loadavg - - | - log=/tmp/sphinx.log - : > $log - (PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > $log 2>&1; echo "sphinx exit $?" >> $log) & - n=0 - while [ $n -lt 780 ] && ! grep -q "^sphinx exit " $log; do - sleep 20 - n=$((n+20)) - echo "[$n] load $(cut -d" " -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s " " | cut -d" " -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 90)" - done - ps -eo pid,ppid,stat,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-170 - pkill -ABRT -f "[b]in/sphinx-build" && sleep 3 - echo "=== log" - cat $log - exit 1 + # The build's output goes to a file and is printed at the end. Sent straight to Read + # the Docs since late October, the same build runs past its time limit (see #40). + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > /tmp/sphinx.log 2>&1; status=$?; cat /tmp/sphinx.log; exit $status From 2c47c262f5b8afc221696d11f409c3ab701386cb Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 13:37:50 +0000 Subject: [PATCH 6/9] Run CI on the Read the Docs fix The change is the one line in .readthedocs.yaml; earlier commits here were diagnostics, and their head commits skipped CI. Co-Authored-By: Claude Opus 5.5 From c37400067b315c26764b98fa6b03300550c34316 Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 14:20:23 +0000 Subject: [PATCH 7/9] Watch a build whose output goes to Read the Docs [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 182e376..94f7f5c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,6 +19,24 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - # The build's output goes to a file and is printed at the end. Sent straight to Read - # the Docs since late October, the same build runs past its time limit (see #40). - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > /tmp/sphinx.log 2>&1; status=$?; cat /tmp/sphinx.log; exit $status + # Diagnostic, not for merging: the output goes to Read the Docs as in the failing + # config and is copied to a file, with a monitor, and everything stops at 780 s. + # Read the Docs wraps a command in single quotes, so it must hold none. + - nproc; free -m; cat /proc/loadavg + - | + log=/tmp/sphinx.log + probe=/tmp/probe.log + : > $log + : > $probe + (n=0; while [ $n -lt 900 ]; do sleep 20; n=$((n+20)); echo "[$n] load $(cut -d" " -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s " " | cut -d" " -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 80) | $(pgrep -f "[b]in/sphinx-build" | head -1 | xargs -r -I P sh -c "cut -d\" \" -f3 /proc/P/stat; cat /proc/P/wchan" | tr "\n" " ")" >> $probe; done) > /dev/null 2>&1 & + sampler=$! + (sleep 780; ps -eo pid,ppid,stat,wchan:20,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-170 >> $probe; pkill -ABRT -f "[b]in/sphinx-build"; sleep 3; pkill -f "[b]in/sphinx-build"; sleep 3; pkill tee) > /dev/null 2>&1 & + killer=$! + PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html 2>&1 | tee $log + echo "=== finished" + kill $sampler $killer + echo "=== probe" + cat $probe + echo "=== tail of the copy" + tail -n 60 $log + exit 1 From 4bb99a50a182f495b4d87a947ba93c2da94f2f53 Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 14:42:02 +0000 Subject: [PATCH 8/9] Send only the end of the log to Read the Docs [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 94f7f5c..06d60d3 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,24 +19,6 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - # Diagnostic, not for merging: the output goes to Read the Docs as in the failing - # config and is copied to a file, with a monitor, and everything stops at 780 s. - # Read the Docs wraps a command in single quotes, so it must hold none. - - nproc; free -m; cat /proc/loadavg - - | - log=/tmp/sphinx.log - probe=/tmp/probe.log - : > $log - : > $probe - (n=0; while [ $n -lt 900 ]; do sleep 20; n=$((n+20)); echo "[$n] load $(cut -d" " -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s " " | cut -d" " -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 80) | $(pgrep -f "[b]in/sphinx-build" | head -1 | xargs -r -I P sh -c "cut -d\" \" -f3 /proc/P/stat; cat /proc/P/wchan" | tr "\n" " ")" >> $probe; done) > /dev/null 2>&1 & - sampler=$! - (sleep 780; ps -eo pid,ppid,stat,wchan:20,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-170 >> $probe; pkill -ABRT -f "[b]in/sphinx-build"; sleep 3; pkill -f "[b]in/sphinx-build"; sleep 3; pkill tee) > /dev/null 2>&1 & - killer=$! - PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html 2>&1 | tee $log - echo "=== finished" - kill $sampler $killer - echo "=== probe" - cat $probe - echo "=== tail of the copy" - tail -n 60 $log - exit 1 + # Under test (#40): the full log stays in a file, and only its end goes to Read the + # Docs. Read the Docs wraps a command in single quotes, so it must hold none. + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > /tmp/sphinx.log 2>&1; status=$?; if [ $status -eq 0 ]; then tail -n 20 /tmp/sphinx.log; else tail -n 300 /tmp/sphinx.log; fi; exit $status From 11ec1363c647188ad2fb679b11081d6f70a52ebc Mon Sep 17 00:00:00 2001 From: John Travers Date: Wed, 4 Nov 2026 14:58:13 +0000 Subject: [PATCH 9/9] Trace builds until one fails to finish in time [skip ci] Co-Authored-By: Claude Opus 5.5 --- .readthedocs.yaml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 06d60d3..e9f6a06 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,6 +19,22 @@ build: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --frozen --all-extras build: html: - # Under test (#40): the full log stays in a file, and only its end goes to Read the - # Docs. Read the Docs wraps a command in single quotes, so it must hold none. - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > /tmp/sphinx.log 2>&1; status=$?; if [ $status -eq 0 ]; then tail -n 20 /tmp/sphinx.log; else tail -n 300 /tmp/sphinx.log; fi; exit $status + # Diagnostic, not for merging: the build writes to a file while the foreground loop + # reports progress, so the log survives; at 780 s it dumps the Python stacks and stops. + # Read the Docs wraps a command in single quotes, so it must hold none. + - nproc; free -m; cat /proc/loadavg + - | + log=/tmp/sphinx.log + : > $log + (PYTHONFAULTHANDLER=1 PYTHONUNBUFFERED=1 UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv run --frozen sphinx-build -b html -W -n --keep-going docs $READTHEDOCS_OUTPUT/html > $log 2>&1; echo "sphinx exit $?" >> $log) & + n=0 + while [ $n -lt 780 ] && ! grep -q "^sphinx exit " $log; do + sleep 20 + n=$((n+20)) + echo "[$n] load $(cut -d" " -f1-3 /proc/loadavg) mem $(free -m | sed -n 2p | tr -s " " | cut -d" " -f3,7) lines $(wc -l < $log) | $(tail -n 1 $log | tail -c 90)" + done + ps -eo pid,ppid,stat,wchan:20,etimes,time,pcpu,rss,args --sort=-pcpu | head -12 | cut -c1-190 + pkill -ABRT -f "[b]in/sphinx-build" && sleep 3 + echo "=== log" + cat $log + exit 1