From 59b9019b1140a8239d9595ae54799754b4145541 Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 09:54:11 +0200 Subject: [PATCH 1/6] ci: bound every job and harden the unixODBC install The workflow declared no timeout anywhere, so a job that hangs pins the run for GitHub's 6 hour default. A hung `apt-get` on the two Snowflake lint legs did exactly that: `test` needs `lint`, and `coverage` needs `test`, so one stuck leg of the matrix froze the whole pipeline and no test ever ran. Lint, test and coverage now carry a timeout, and the apt step that hangs carries its own, tighter one, plus retries on the mirror and a noninteractive frontend. A failing leg is now visible in minutes and can be re-run. Deploy is left unbounded on purpose: cutting semantic-release off in the middle of pushing thirteen gems is worse than waiting for it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0888f9851..18eeb86d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ jobs: lint: name: Lint runs-on: ubuntu-latest + timeout-minutes: 15 env: BUNDLE_PATH: /tmp/bundle strategy: @@ -39,9 +40,16 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Bounded on its own: an unreachable apt mirror blocks with no timeout of + # its own, and this step gates every later one in the job. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} - run: sudo apt-get update && sudo apt-get install -y unixodbc-dev + timeout-minutes: 5 + env: + DEBIAN_FRONTEND: noninteractive + run: | + sudo apt-get update -o Acquire::Retries=3 + sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 @@ -60,6 +68,7 @@ jobs: test: name: Test runs-on: ubuntu-latest + timeout-minutes: 20 needs: [lint] env: BUNDLE_PATH: /tmp/bundle @@ -88,9 +97,16 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # Bounded on its own: an unreachable apt mirror blocks with no timeout of + # its own, and this step gates every later one in the job. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} - run: sudo apt-get update && sudo apt-get install -y unixodbc-dev + timeout-minutes: 5 + env: + DEBIAN_FRONTEND: noninteractive + run: | + sudo apt-get update -o Acquire::Retries=3 + sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 @@ -128,6 +144,7 @@ jobs: coverage: name: Send Coverage runs-on: ubuntu-latest + timeout-minutes: 10 needs: [test] strategy: matrix: From 603dad1f372476430a7688be70c6468ad9d2ccb3 Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 10:08:51 +0200 Subject: [PATCH 2/6] ci: fall back to archive.ubuntu.com when azure stalls The timeout added in the previous commit bounded the damage but not the cause, and this workflow proved it on its own run: the 4.0 Snowflake leg died on the timeout while the 3.4 leg passed on the same commit. The log shows `apt-get update` looping on `Ign: azure.archive.ubuntu.com`, 72 seconds before the first one, while the Microsoft and Google https repos answer in milliseconds in the same job. Each acquisition is now bounded at 15s, and a failed attempt is retried against the canonical archive instead of being given up on. The step timeout goes to 8 minutes so the fallback has room to run. The package cannot simply be dropped, as the previous commit wondered: the leg that passed reports `4 newly installed` and unpacks unixodbc-dev, so it is genuinely absent from the runner image. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 38 +++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18eeb86d0..423081c37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,16 +40,25 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # Bounded on its own: an unreachable apt mirror blocks with no timeout of - # its own, and this step gates every later one in the job. + # `unixodbc-dev` is not on the runner image, and the Azure mirror the image + # points at is intermittently unreachable: bounded, then retried against the + # canonical archive rather than stalling every step it gates. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} - timeout-minutes: 5 + timeout-minutes: 8 env: DEBIAN_FRONTEND: noninteractive run: | - sudo apt-get update -o Acquire::Retries=3 - sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 unixodbc-dev + apt_install() { + sudo apt-get update -o Acquire::Retries=2 -o Acquire::http::Timeout=15 && + sudo apt-get install -y --no-install-recommends unixodbc-dev + } + apt_install && exit 0 + + echo 'Azure mirror unreachable; retrying against archive.ubuntu.com' + sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ + /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true + apt_install - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 @@ -97,16 +106,25 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # Bounded on its own: an unreachable apt mirror blocks with no timeout of - # its own, and this step gates every later one in the job. + # `unixodbc-dev` is not on the runner image, and the Azure mirror the image + # points at is intermittently unreachable: bounded, then retried against the + # canonical archive rather than stalling every step it gates. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} - timeout-minutes: 5 + timeout-minutes: 8 env: DEBIAN_FRONTEND: noninteractive run: | - sudo apt-get update -o Acquire::Retries=3 - sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 unixodbc-dev + apt_install() { + sudo apt-get update -o Acquire::Retries=2 -o Acquire::http::Timeout=15 && + sudo apt-get install -y --no-install-recommends unixodbc-dev + } + apt_install && exit 0 + + echo 'Azure mirror unreachable; retrying against archive.ubuntu.com' + sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ + /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true + apt_install - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 From 238a9d2c5b366f9552daebd18977cbb4494bffdb Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 10:46:44 +0200 Subject: [PATCH 3/6] ci: bound apt with timeout, not with its own knobs The previous attempt trusted `Acquire::http::Timeout` to bound the step. It does not: it caps a single connection, not the walk over two dozen index files, and with retries on top the first `apt-get update` spent the whole 8 minute budget looping on `Ign: azure.archive.ubuntu.com`. The step timed out inside that first attempt, so the mirror fallback was never reached -- its message appears nowhere in the log. The bound now comes from `timeout 90` around each update, which returns whatever apt does. Refreshing the lists is also demoted to best effort, since the image already carries usable ones and only the install has to succeed; a failed install is what triggers the fallback to the canonical archive. Checked under `bash -e`, the shell the runner uses, over the four paths: healthy mirror, update timing out with the image lists sufficing, fallback succeeding, and everything failing -- which exits non-zero with apt's own status. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 42 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 423081c37..7e0f2f9e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,24 +41,29 @@ jobs: uses: actions/checkout@v4 # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable: bounded, then retried against the - # canonical archive rather than stalling every step it gates. + # points at is intermittently unreachable. apt's own timeouts bound a single + # connection, not the walk over two dozen index files, so the bound comes + # from `timeout` -- and refreshing the lists is best effort, since only the + # install has to succeed. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 env: DEBIAN_FRONTEND: noninteractive run: | - apt_install() { - sudo apt-get update -o Acquire::Retries=2 -o Acquire::http::Timeout=15 && - sudo apt-get install -y --no-install-recommends unixodbc-dev + apt_update() { + sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + echo 'apt-get update incomplete; continuing with the lists on the image' } - apt_install && exit 0 - echo 'Azure mirror unreachable; retrying against archive.ubuntu.com' + apt_update + sudo apt-get install -y --no-install-recommends unixodbc-dev && exit 0 + + echo 'Install failed; falling back to archive.ubuntu.com' sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true - apt_install + apt_update + sudo apt-get install -y --no-install-recommends unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 @@ -107,24 +112,29 @@ jobs: uses: actions/checkout@v4 # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable: bounded, then retried against the - # canonical archive rather than stalling every step it gates. + # points at is intermittently unreachable. apt's own timeouts bound a single + # connection, not the walk over two dozen index files, so the bound comes + # from `timeout` -- and refreshing the lists is best effort, since only the + # install has to succeed. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 env: DEBIAN_FRONTEND: noninteractive run: | - apt_install() { - sudo apt-get update -o Acquire::Retries=2 -o Acquire::http::Timeout=15 && - sudo apt-get install -y --no-install-recommends unixodbc-dev + apt_update() { + sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + echo 'apt-get update incomplete; continuing with the lists on the image' } - apt_install && exit 0 - echo 'Azure mirror unreachable; retrying against archive.ubuntu.com' + apt_update + sudo apt-get install -y --no-install-recommends unixodbc-dev && exit 0 + + echo 'Install failed; falling back to archive.ubuntu.com' sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true - apt_install + apt_update + sudo apt-get install -y --no-install-recommends unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 From d5ed2f2f4a4589db82a2d8147a8496f26cffa2a6 Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 12:03:42 +0200 Subject: [PATCH 4/6] ci: bound the install too, and drop the mirror fallback Three gaps from review, all visible in this PR's own green run. The install was unbounded: `timeout 90` wrapped the index refresh only, while the four .deb files come from the same mirror. The log stalls exactly there, `Ign:` lines at 08:49:05 sitting after `Need to get 306 kB`, with ~6.5 min of the step budget still to burn. A harder stall would have been killed inside that first install, which is the shape the previous commit claimed to have fixed. The mirror fallback duplicated apt's own: the image mirrorlist already carries archive.ubuntu.com, and the same log shows apt switching to it unaided at 08:47:13. Rewriting the host also desynchronises the indices under /var/lib/apt/lists, so the retry could die on `Unable to locate package` and name the wrong cause. It is gone; apt does this better. DEBIAN_FRONTEND never reached apt, sudo's env_reset dropping it on the way. It is now set across that boundary. Both commands are bounded, worst case 5.5 min under the 8 minute cap, and the step is down to two lines with no control flow to reason about. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 56 ++++++++++++------------------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e0f2f9e8..dbeae876d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,29 +41,19 @@ jobs: uses: actions/checkout@v4 # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable. apt's own timeouts bound a single - # connection, not the walk over two dozen index files, so the bound comes - # from `timeout` -- and refreshing the lists is best effort, since only the - # install has to succeed. + # points at is intermittently unreachable -- during the index refresh and + # during the package fetch alike. apt's own timeouts bound a single + # connection rather than a whole fetch, so both commands are bounded from the + # outside. Refreshing the lists is best effort: the image carries usable + # ones, and only the install has to succeed. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 - env: - DEBIAN_FRONTEND: noninteractive run: | - apt_update() { - sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || - echo 'apt-get update incomplete; continuing with the lists on the image' - } - - apt_update - sudo apt-get install -y --no-install-recommends unixodbc-dev && exit 0 - - echo 'Install failed; falling back to archive.ubuntu.com' - sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ - /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true - apt_update - sudo apt-get install -y --no-install-recommends unixodbc-dev + sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + echo 'apt-get update incomplete; continuing with the lists on the image' + sudo DEBIAN_FRONTEND=noninteractive timeout 240 apt-get install -y \ + --no-install-recommends -o Acquire::Retries=1 -o Acquire::http::Timeout=10 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 @@ -112,29 +102,19 @@ jobs: uses: actions/checkout@v4 # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable. apt's own timeouts bound a single - # connection, not the walk over two dozen index files, so the bound comes - # from `timeout` -- and refreshing the lists is best effort, since only the - # install has to succeed. + # points at is intermittently unreachable -- during the index refresh and + # during the package fetch alike. apt's own timeouts bound a single + # connection rather than a whole fetch, so both commands are bounded from the + # outside. Refreshing the lists is best effort: the image carries usable + # ones, and only the install has to succeed. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 - env: - DEBIAN_FRONTEND: noninteractive run: | - apt_update() { - sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || - echo 'apt-get update incomplete; continuing with the lists on the image' - } - - apt_update - sudo apt-get install -y --no-install-recommends unixodbc-dev && exit 0 - - echo 'Install failed; falling back to archive.ubuntu.com' - sudo sed -i 's|azure.archive.ubuntu.com|archive.ubuntu.com|g' \ - /etc/apt/apt-mirrors.txt /etc/apt/sources.list.d/ubuntu.sources || true - apt_update - sudo apt-get install -y --no-install-recommends unixodbc-dev + sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + echo 'apt-get update incomplete; continuing with the lists on the image' + sudo DEBIAN_FRONTEND=noninteractive timeout 240 apt-get install -y \ + --no-install-recommends -o Acquire::Retries=1 -o Acquire::http::Timeout=10 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 From e0c035cbac81e43a78e80fece0f6eaafb98bd6b2 Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 12:20:45 +0200 Subject: [PATCH 5/6] ci: make the apt bounds effective with --kill-after `timeout` sends SIGTERM, which apt and dpkg may defer during a transaction. Without a kill-after the two bounds are decorative and only the 8 minute step cap stops anything. Worst case is now 6.5 min, still under it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dbeae876d..bfe4bd1f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,9 +50,9 @@ jobs: if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 run: | - sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + sudo timeout -k 30 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || echo 'apt-get update incomplete; continuing with the lists on the image' - sudo DEBIAN_FRONTEND=noninteractive timeout 240 apt-get install -y \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 30 240 apt-get install -y \ --no-install-recommends -o Acquire::Retries=1 -o Acquire::http::Timeout=10 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} @@ -111,9 +111,9 @@ jobs: if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 run: | - sudo timeout 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || + sudo timeout -k 30 90 apt-get update -o Acquire::Retries=1 -o Acquire::http::Timeout=10 || echo 'apt-get update incomplete; continuing with the lists on the image' - sudo DEBIAN_FRONTEND=noninteractive timeout 240 apt-get install -y \ + sudo DEBIAN_FRONTEND=noninteractive timeout -k 30 240 apt-get install -y \ --no-install-recommends -o Acquire::Retries=1 -o Acquire::http::Timeout=10 unixodbc-dev - name: Set up Ruby ${{ matrix.ruby-version }} From c2d83633234bf2f183b31d8ad6596ef069fa715e Mon Sep 17 00:00:00 2001 From: Brun Christophe Date: Wed, 19 Aug 2026 14:10:30 +0200 Subject: [PATCH 6/6] ci: trim the unixODBC comment to what the code cannot say The block restated the step name and the echo string sitting two lines below it. Two facts are left, neither readable from the code: the mirror the image points at stalls, and apt's Acquire timeouts bound a single connection rather than a whole fetch. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfe4bd1f1..c3d365917 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,12 +40,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable -- during the index refresh and - # during the package fetch alike. apt's own timeouts bound a single - # connection rather than a whole fetch, so both commands are bounded from the - # outside. Refreshing the lists is best effort: the image carries usable - # ones, and only the install has to succeed. + # The Azure mirror the runner image points at stalls intermittently, and apt's + # own Acquire timeouts bound a single connection, not a whole fetch. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8 @@ -101,12 +97,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # `unixodbc-dev` is not on the runner image, and the Azure mirror the image - # points at is intermittently unreachable -- during the index refresh and - # during the package fetch alike. apt's own timeouts bound a single - # connection rather than a whole fetch, so both commands are bounded from the - # outside. Refreshing the lists is best effort: the image carries usable - # ones, and only the install has to succeed. + # The Azure mirror the runner image points at stalls intermittently, and apt's + # own Acquire timeouts bound a single connection, not a whole fetch. - name: Install unixODBC headers for ruby-odbc if: ${{ matrix.packages == 'forest_admin_datasource_snowflake' }} timeout-minutes: 8