From 9cf4dd8e8c850e11bea522cc45ba2a29931e97ac Mon Sep 17 00:00:00 2001 From: Wanjun Gu Date: Mon, 19 Oct 2026 21:51:12 -0700 Subject: [PATCH] ci: keep main's Rust caches resident, and size the test job for a cold one Two pushes to main on 2026-10-18 (39156748556, 39158205693) were cancelled at 40.0 and 40.2 minutes in test (ubuntu-latest) with every step green. The cache key was unchanged (v0-rust-ubuntu-latest-test-Linux-x64-edf29d0d- c299e55f, hit at 17:39 by the release PR), yet both runs logged "No cache found". GitHub had evicted the entry: at 18:16-18:18 the release branch's GUI package workflows saved two cross caches (1.2 + 1.4 GB, readable by that branch alone and never read again), the repository passed its 10 GB cache limit, and the least recently used entry was the ubuntu test cache, last read at 17:39:11. A cold ubuntu run takes 30.5-40.2 minutes (n=10). A second, recurring cause: rust-cache hashes every installed toolchain into its key, including the runner image's own stable that nothing here builds with. Image 20261002 moved it 1.98.1 -> 1.99.0 and every ubuntu run built cold from 2026-10-10 until main next saved on 10-12. - test: timeout 40 -> 60, with the measured warm/cold table per OS. - test, serve, frontend api-contract, apps-smoke: remove every toolchain but the pinned one before rust-cache, so image updates stop changing keys. - serve, frontend api-contract, apps-smoke: save-if main, as test already did. During the 10-11/12 eviction every PR's serve job wrote its own 1.9 GB copy. - linux/windows GUI packages: no persisted cross cache. Branch saves were never reused, and main saves would not fit beside main's ~8.3 GB of test caches, because rust.yml's cross-check reads the same key prefix on every run. Cold builds take 30.8-39.1 minutes against a 100 minute limit. --- .github/workflows/apps-smoke.yml | 13 ++++ .github/workflows/frontend.yml | 15 ++++ .github/workflows/linux-gui-packages.yml | 33 +++++---- .github/workflows/rust.yml | 79 ++++++++++++++++++++-- .github/workflows/windows-gui-packages.yml | 11 ++- 5 files changed, 127 insertions(+), 24 deletions(-) diff --git a/.github/workflows/apps-smoke.yml b/.github/workflows/apps-smoke.yml index e02395f6c..7c934b601 100644 --- a/.github/workflows/apps-smoke.yml +++ b/.github/workflows/apps-smoke.yml @@ -35,9 +35,22 @@ jobs: - uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # The image's own toolchain must not reach the cache key, and only main + # saves. rust.yml's `test` job has the measurements behind both. + - name: Keep only the pinned Rust toolchain (stable cache key) + shell: bash + run: | + set -euo pipefail + pinned=$(rustup show active-toolchain | tr -d '\r' | cut -d' ' -f1) + [ -n "$pinned" ] + for t in $(rustup toolchain list | tr -d '\r' | cut -d' ' -f1); do + if [ "$t" != "$pinned" ]; then rustup toolchain uninstall "$t"; fi + done + rustup toolchain list - uses: Swatinem/rust-cache@v2 with: key: apps-smoke + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Rust native build dependencies run: sudo apt-get update && sudo apt-get install --yes libdbus-1-dev libxcb1-dev pkg-config - name: Install desktop test runtime diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 339676b29..96e19cdf9 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -75,9 +75,24 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} + # The image's own toolchain must not reach the cache key, and only main + # saves. rust.yml's `test` job has the measurements behind both. + - name: Keep only the pinned Rust toolchain (stable cache key) + working-directory: ${{ github.workspace }} + shell: bash + run: | + set -euo pipefail + pinned=$(rustup show active-toolchain | tr -d '\r' | cut -d' ' -f1) + [ -n "$pinned" ] + for t in $(rustup toolchain list | tr -d '\r' | cut -d' ' -f1); do + if [ "$t" != "$pinned" ]; then rustup toolchain uninstall "$t"; fi + done + rustup toolchain list + - uses: Swatinem/rust-cache@v2 with: key: openapi-contract + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Rust native build dependencies working-directory: ${{ github.workspace }} diff --git a/.github/workflows/linux-gui-packages.yml b/.github/workflows/linux-gui-packages.yml index c915a3669..f5f720bba 100644 --- a/.github/workflows/linux-gui-packages.yml +++ b/.github/workflows/linux-gui-packages.yml @@ -42,19 +42,26 @@ jobs: timeout-minutes: 100 steps: - uses: actions/checkout@v4 - # ⚠ `release.sh` `run_cross_release` deliberately destroys its target - # volume before every release build, because cached objects keep stale - # glibc symbol versions. This job keeps a cache instead, and that is only - # safe because `check-glibc-floor.sh` runs afterwards on what was actually - # produced — and again in the packaging job on what is about to be - # packaged. Remove either assertion and this cache becomes the hole. - - uses: actions/cache@v4 - with: - path: | - .cross-cache/registry - .cross-cache/target/x86_64-unknown-linux-gnu - key: cross-x86_64-unknown-linux-gnu-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml', 'scripts/cross-env.sh') }} - restore-keys: cross-x86_64-unknown-linux-gnu- + # ⚠ No persisted cache: the cross target dir starts empty on every run, + # as `release.sh` `run_cross_release` makes its own (it destroys its + # target volume before every release build, because cached objects keep + # stale glibc symbol versions). `check-glibc-floor.sh` still runs here on + # what was produced and again in the packaging job, whatever the cache. + # + # This job used to restore and save `cross--` + # through actions/cache, and that cache cost more than it saved. Measured + # 2026-09-24..10-18, this job takes 34.7-37.5 min cold and 19.3-23.6 + # warm (n=12), against a 100 minute limit. A save from a release + # branch is readable by that branch alone, and none was read again + # (release/1.92.0, 1.93.1 and 1.93.2 each dispatched this once). Each + # save is 1.2 GB here and 1.4 GB in windows-gui-packages.yml, and on + # 2026-10-18 that pair pushed the repository past GitHub's 10 GB cache + # limit, which evicted rust.yml's ubuntu test cache and cancelled the + # next two pushes to main (see the note at that job's rust-cache step). + # Saving from main only would not fit either: main's own caches come to + # about 8.3 GB, and rust.yml's cross-check restores this same key prefix + # on every run, so the pair would stay resident and push one of the test + # caches out instead. Do not add a cache here without finding the room. # This is the ONLY supported way to produce shipped Linux binaries: it # pins the bullseye image and then asserts the floor it just built against. diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3babae220..2b2622de4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -67,9 +67,10 @@ jobs: # These budgets are deliberately well ABOVE the measured maxima (warm # cache, 2026-08-27: guards 0.1m, cross-check 8.2m, test 15.0m, serve # 9.5m) so a merely slow run is never failed -- they exist to make a HANG - # cheap and legible, not to police duration. A deadlock now costs 40 - # minutes of runner time instead of six hours, on every branch that - # pushes. + # cheap and legible, not to police duration. A deadlock now costs at most + # an hour of runner time instead of six, on every branch that pushes. + # (The `test` job's budget was 40 until 2026-10-18, when its cold runs + # reached it; that job's comment has the numbers.) timeout-minutes: 10 if: github.event_name != 'schedule' runs-on: ubuntu-latest @@ -159,7 +160,24 @@ jobs: # NOTE: windows-latest is MSVC; we SHIP mingw/gnu. Job 2 covers the gnu # ABI, this job covers the Windows behaviour. Both are load-bearing. test: - timeout-minutes: 40 + # ⚠ Sized for a COLD cache, because main's first run after any change to + # the cache key (a dependency in Cargo.lock, the toolchain, a CARGO_* env) + # is cold by design, and an evicted cache makes it cold without one. 40 + # was set when this job took 15 minutes warm; the integration-binary step + # and the socket gate have since doubled that, and the budget stayed. So a + # cold ubuntu run was a red X with every step green. Measured over the + # completed runs of 2026-10-10..18, by rust-cache's own restore line: + # + # ubuntu warm 21.3-31.0m (n=52) cold 30.5-40.2m (n=10, eight >37; + # 39156748556 and 39158205693 were cancelled at 40.0 and 40.2 + # with every step, "Complete job" included, green) + # macos warm 14.3-38.2m (n=60) cold 20.1-37.6m (n=6) + # windows warm 15.5-29.7m (n=54) cold 21.3-31.6m (n=8), and one cold + # run still in `cargo test` when 40 hit (39057096529, main) + # + # 60 keeps the margin this file promises above those maxima (see `guards`) + # and still makes a hang cost an hour rather than six. + timeout-minutes: 60 if: github.event_name != 'schedule' strategy: fail-fast: false @@ -198,6 +216,30 @@ jobs: shell: bash run: echo 'CARGO_PROFILE_TEST_DEBUG=0' >> "$GITHUB_ENV" + # ⚠ rust-cache folds EVERY installed toolchain into its key: it runs + # `rustup toolchain list` and hashes `rustc -vV` of each one. The runner + # image ships its own `stable`, which nothing here builds with + # (rust-toolchain.toml pins 1.92), so each image that moves that stable + # changes every Rust cache key in the repository at once. Measured: image + # 20260901.588 -> 20261002.596 moved it from 1.98.1 to 1.99.0, the ubuntu + # key went from `...-15238ee1-c299e55f` to `...-edf29d0d-c299e55f`, and + # every ubuntu run built cold from 2026-10-10 until main next saved, on + # 10-12 (37.0-39.2 minutes each). While an image rolls out, runners on + # the old and new images compute different keys, and main ends up saving + # both. Removing every toolchain but the pinned one keeps the key on what + # this build actually compiles with. `serve`, frontend.yml and + # apps-smoke.yml do the same before their rust-cache steps. + - name: Keep only the pinned Rust toolchain (stable cache key) + shell: bash + run: | + set -euo pipefail + pinned=$(rustup show active-toolchain | tr -d '\r' | cut -d' ' -f1) + [ -n "$pinned" ] + for t in $(rustup toolchain list | tr -d '\r' | cut -d' ' -f1); do + if [ "$t" != "$pinned" ]; then rustup toolchain uninstall "$t"; fi + done + rustup toolchain list + # ⚠ `cache-on-failure` is what breaks a self-perpetuating deadlock, not a # convenience. rust-cache's post step is skipped whenever the job does not # succeed, so a job that cannot finish never saves a cache, is cold on the @@ -230,6 +272,18 @@ jobs: # A PR that changes Cargo.lock still builds cold and cannot save; that is # the accepted cost, and it is much smaller than three OSes' worth of # write-only caches evicting the shared ones on every push. + # + # ⚠ The rule only works if EVERY cache writer follows it, and until + # 2026-10-18 this was the only one that did. On 2026-10-18 the release + # branch's package workflows saved two cross caches (2.6 GB, readable by + # that branch alone and never read again) at 18:16-18:18, the repository + # went past 10 GB, and GitHub evicted the least recently used entry: this + # job's ubuntu cache, last read at 17:39:11, one second before `serve`'s. + # The next two pushes to main (39156748556, 39158205693) found "No cache + # found" for an unchanged key and were cancelled at 40 minutes. `serve`, + # frontend.yml and apps-smoke.yml now save from main only as well, and + # the two GUI package workflows no longer persist a cache at all (their + # files say why). Main's own caches come to about 8.3 GB of the 10. - uses: Swatinem/rust-cache@v2 with: key: ${{ matrix.os }} @@ -578,9 +632,26 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: "1.92" + # Same as the `test` job: the image's own toolchain must not reach the + # cache key. + - name: Keep only the pinned Rust toolchain (stable cache key) + shell: bash + run: | + set -euo pipefail + pinned=$(rustup show active-toolchain | tr -d '\r' | cut -d' ' -f1) + [ -n "$pinned" ] + for t in $(rustup toolchain list | tr -d '\r' | cut -d' ' -f1); do + if [ "$t" != "$pinned" ]; then rustup toolchain uninstall "$t"; fi + done + rustup toolchain list + # Saves from main only, like `test` and for the reason given there. When + # main's copy was evicted (2026-10-11/12), every PR's first run here + # wrote its own 1.9 GB copy that no other branch could read, which + # pushed the repository further past the ceiling. - uses: Swatinem/rust-cache@v2 with: key: serve + save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-node@v4 with: node-version: 24 diff --git a/.github/workflows/windows-gui-packages.yml b/.github/workflows/windows-gui-packages.yml index ba1dcef97..9b3b593d7 100644 --- a/.github/workflows/windows-gui-packages.yml +++ b/.github/workflows/windows-gui-packages.yml @@ -44,13 +44,10 @@ jobs: timeout-minutes: 100 steps: - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: | - .cross-cache/registry - .cross-cache/target/x86_64-pc-windows-gnu - key: cross-x86_64-pc-windows-gnu-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml', 'scripts/cross-env.sh') }} - restore-keys: cross-x86_64-pc-windows-gnu- + # No persisted cache, for the reasons linux-gui-packages.yml gives at its + # build step. Measured 2026-09-29..10-18, this job takes 30.8-39.1 min + # cold and 19.3-21.0 warm (n=8), against a 100 minute limit, and a + # release branch's 1.4 GB save here was never read again. # Also stages the mingw runtime DLLs beside the exes ($WIN_DLL_STAGE). - name: Cross-build the Windows backend