From 105604678ed272199c62c2d1fdfc9b8cc89083be Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 07:52:01 -0700 Subject: [PATCH 1/4] CI: run every job on the self-hosted runner on dev-01 Hosted minutes ran out. Every job runs on [self-hosted, linux, dev-01], a shared host whose own Compose stack holds :8080, :8000 and 127.0.0.1:5432, so no job maps a fixed host port or installs host packages: - api runs in python:3.12 (setup-python has no builds for Debian 13) with postgres, mongo and the MLflow server as services reached by name. - stack runs its own Compose project on ephemeral loopback ports, Playwright in its image, and removes the stack, volumes and images after. - lint runs ruff from Astral's image of the pinned version; images tags its builds under a CI project name and removes them after. - Container jobs hand the workspace back to the runner's user; host jobs reclaim it first. - Pull requests only: no badge reads main's runs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- .github/workflows/ci.yml | 146 +++++++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1384cdcb..f0b234fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,11 @@ name: CI +# Every job runs on the self-hosted runner on dev-01, a shared host: its own Compose stack holds :8080, :8000 and +# 127.0.0.1:5432, and other sessions run containers there. So no job maps a fixed host port or installs host +# packages (no sudo): service containers are reached by name from a job container, the CI's Compose stack gets its +# own project name and ephemeral loopback ports, and tools that need Python or browsers run in their images. +# Pull requests only: no badge reads main's runs, and a merge to main has already passed as a pull request. on: - push: - branches: [main] pull_request: concurrency: @@ -15,60 +18,73 @@ permissions: jobs: api: name: API tests - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 45 + # In python:3.12 (the image api/Dockerfile.test builds on): setup-python has no builds for dev-01's Debian, and + # from a job container the services are reached by name on the job's network, with no host ports. + container: + image: python:3.12 + volumes: + - rf-ci-pip-cache:/root/.cache/pip services: postgres: image: postgres:16 env: POSTGRES_PASSWORD: test - ports: ["5432:5432"] options: >- --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 mongo: image: mongo:7 - ports: ["27017:27017"] options: >- --health-cmd "mongosh --quiet --eval \"db.adminCommand('ping')\"" --health-interval 5s --health-timeout 5s --health-retries 10 + # The tracking server the MLflow tests log to. --allowed-hosts: it is reached by name, not localhost. + mlflow: + image: ghcr.io/mlflow/mlflow:v3.17.0 + command: >- + mlflow server --host 0.0.0.0 --port 5000 --backend-store-uri sqlite:////tmp/mlflow.db + --serve-artifacts --artifacts-destination /tmp/mlartifacts --static-prefix /mlflow + --allowed-hosts mlflow,mlflow:5000 env: - RAINFOREST_TEST_DATABASE_URL: postgresql://postgres:test@localhost:5432/postgres - RAINFOREST_TEST_MONGO_URL: mongodb://localhost:27017 + RAINFOREST_TEST_DATABASE_URL: postgresql://postgres:test@postgres:5432/postgres + RAINFOREST_TEST_MONGO_URL: mongodb://mongo:27017 + PIP_ROOT_USER_ACTION: ignore + PIP_DISABLE_PIP_VERSION_CHECK: "1" defaults: run: working-directory: api steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - cache: pip - cache-dependency-path: api/requirements*.txt - - run: sudo apt-get install -y --no-install-recommends libgomp1 + # libgomp1: LightGBM's OpenMP runtime; procps: ps and pkill for the worker tests. + - run: apt-get update && apt-get install -y --no-install-recommends libgomp1 procps - run: pip install -r requirements-dev.txt - - name: Start an MLflow tracking server + - name: Wait for the MLflow tracking server run: | - docker run -d --name mlflow -p 5000:5000 ghcr.io/mlflow/mlflow:v3.17.0 mlflow server \ - --host 0.0.0.0 --port 5000 --backend-store-uri sqlite:////tmp/mlflow.db \ - --serve-artifacts --artifacts-destination /tmp/mlartifacts --static-prefix /mlflow - for i in $(seq 60); do curl -fs http://localhost:5000/mlflow/health && exit 0; sleep 2; done - docker logs mlflow; exit 1 + for i in $(seq 60); do curl -fs http://mlflow:5000/mlflow/health && exit 0; sleep 2; done + exit 1 - run: pytest -q -rs env: - MLFLOW_TEST_URI: http://localhost:5000/mlflow + MLFLOW_TEST_URI: http://mlflow:5000/mlflow - run: pytest -q client/tests - name: MLflow sidecar tests run: pytest -q working-directory: mlflow + - name: Hand the workspace back to the runner's user + if: always() + working-directory: / + run: chown -R "$(stat -c %u:%g "$GITHUB_WORKSPACE/..")" "$GITHUB_WORKSPACE" "$RUNNER_TEMP" ui: name: UI typecheck, tests and build - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 10 defaults: run: working-directory: ui steps: + - name: Reclaim the workspace from files a container job left as root + working-directory: / + run: docker run --rm -v "$GITHUB_WORKSPACE:/w" -v "$RUNNER_TEMP:/t" alpine:3.20 chown -R "$(id -u):$(id -g)" /w /t - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: @@ -79,16 +95,15 @@ jobs: - run: npm run typecheck - run: npm test - run: npm run build + # envsubst (gettext-base) is on dev-01; without it the script re-runs itself in the nginx image. - name: nginx config rendering - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends gettext-base - bash nginx/test.sh + run: bash nginx/test.sh e2e: name: UI end-to-end (fixture and gallery) - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 20 + # The job container also keeps the fixture servers' ports (5173, 8765, 61000) off the host. container: image: mcr.microsoft.com/playwright:v1.65.0-noble defaults: @@ -107,34 +122,43 @@ jobs: name: playwright-report path: ui/playwright-report retention-days: 7 + - name: Hand the workspace back to the runner's user + if: always() + working-directory: / + run: chown -R "$(stat -c %u:%g "$GITHUB_WORKSPACE/..")" "$GITHUB_WORKSPACE" "$RUNNER_TEMP" stack: name: UI end-to-end (Compose stack) - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 30 steps: + - name: Reclaim the workspace from files a container job left as root + run: docker run --rm -v "$GITHUB_WORKSPACE:/w" -v "$RUNNER_TEMP:/t" alpine:3.20 chown -R "$(id -u):$(id -g)" /w /t - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: npm - cache-dependency-path: ui/package-lock.json + # Its own Compose project (containers, volumes, images) beside dev-01's `rainforest` one, and ephemeral + # loopback ports in place of the stack's :8080, :8000 and :5432, which dev-01's stack holds. + - name: Keep the CI's stack apart from dev-01's + run: | + echo "COMPOSE_PROJECT_NAME=rf-ci-stack-${RUNNER_NAME,,}" >> "$GITHUB_ENV" + { echo UI_PORT=127.0.0.1:0; echo API_PORT=0; echo DB_PORT=0; } >> "$GITHUB_ENV" + - name: Remove a stack an earlier run left + run: docker compose down -v --remove-orphans - name: Start the stack run: | docker compose up -d --build + url=http://$(docker compose port ui 80) + echo "E2E_BASE_URL=$url" >> "$GITHUB_ENV" for i in $(seq 1 60); do - curl -fsS http://localhost:8080/api/auth/users > /dev/null && break + curl -fsS "$url/api/auth/users" > /dev/null && break sleep 5 done - curl -fsS http://localhost:8080/api/auth/users > /dev/null - - run: npm ci - working-directory: ui - - run: npx playwright install --with-deps chromium - working-directory: ui - - run: npm run e2e:stack - working-directory: ui - env: - E2E_BASE_URL: http://localhost:8080 + curl -fsS "$url/api/auth/users" > /dev/null + # In the Playwright image (browsers and their system libraries), as the runner's user, on the host network. + - name: npm ci and npm run e2e:stack + run: | + docker run --rm --network host --user "$(id -u):$(id -g)" -e HOME=/tmp/home -e CI=true -e E2E_BASE_URL \ + -v "$GITHUB_WORKSPACE/ui:/ui" -w /ui mcr.microsoft.com/playwright:v1.65.0-noble \ + sh -c 'npm ci && npm run e2e:stack' - name: Stack logs if: failure() run: docker compose logs --no-color --tail=200 api worker serve @@ -144,24 +168,27 @@ jobs: name: playwright-report-stack path: ui/playwright-report retention-days: 7 + - name: Remove the stack, its volumes and images + if: always() + run: docker compose down -v --remove-orphans --rmi local lint: name: Lint - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 10 steps: + - name: Reclaim the workspace from files a container job left as root + run: docker run --rm -v "$GITHUB_WORKSPACE:/w" -v "$RUNNER_TEMP:/t" alpine:3.20 chown -R "$(id -u):$(id -g)" /w /t - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Install ruff (the version in requirements-dev.txt) - run: pip install "$(grep '^ruff==' api/requirements-dev.txt)" + # Astral's image of the version in requirements-dev.txt (dev-01 has no Python for setup-python or pip). + - name: Ruff image (the version in requirements-dev.txt) + run: echo "RUFF_IMAGE=ghcr.io/astral-sh/ruff:$(sed -n 's/^ruff==//p' api/requirements-dev.txt)" >> "$GITHUB_ENV" - name: Ruff working-directory: api - run: ruff check --output-format github . + run: docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/io" -w /io "$RUFF_IMAGE" check --output-format github . - name: Ruff format working-directory: api - run: ruff format --check . + run: docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/io" -w /io "$RUFF_IMAGE" format --check . - uses: actions/setup-node@v4 with: node-version: "20" @@ -178,9 +205,11 @@ jobs: chart: name: Helm chart - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 10 steps: + - name: Reclaim the workspace from files a container job left as root + run: docker run --rm -v "$GITHUB_WORKSPACE:/w" -v "$RUNNER_TEMP:/t" alpine:3.20 chown -R "$(id -u):$(id -g)" /w /t - uses: actions/checkout@v4 - uses: azure/setup-helm@v4 with: @@ -197,10 +226,21 @@ jobs: images: name: Docker images build - runs-on: ubuntu-24.04 + runs-on: [self-hosted, linux, dev-01] timeout-minutes: 20 steps: + - name: Reclaim the workspace from files a container job left as root + run: docker run --rm -v "$GITHUB_WORKSPACE:/w" -v "$RUNNER_TEMP:/t" alpine:3.20 chown -R "$(id -u):$(id -g)" /w /t - uses: actions/checkout@v4 + # Tagged under the CI's own project name, so the build never retags the images dev-01's stack runs. + - name: Name the CI's images apart from dev-01's + run: echo "COMPOSE_PROJECT_NAME=rf-ci-images-${RUNNER_NAME,,}" >> "$GITHUB_ENV" - run: docker compose --profile keycloak build # Not a compose service: Compose runs no notebooks (Kubernetes only), but CI publishes the image. - - run: docker build -f notebook/Dockerfile -t rainforest-notebook . + - run: docker build -f notebook/Dockerfile -t "$COMPOSE_PROJECT_NAME-notebook" . + # The layers stay in the build cache for the next run; the tags go. + - name: Remove the images it built + if: always() + run: | + docker compose --profile keycloak config --images | grep "^$COMPOSE_PROJECT_NAME-" | xargs -r docker image rm -f + docker image rm -f "$COMPOSE_PROJECT_NAME-notebook" || true From 4dfe6a38b08e454f2dc8c312fb4bb635462556ed Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 08:09:30 -0700 Subject: [PATCH 2/4] CI: raise the Mongo service's open-file limit on dev-01 dev-01's Docker gives containers a soft nofile of 1024; the API suite's collections exhausted it and WiredTiger panicked. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0b234fc..2d8bc0d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,12 @@ jobs: POSTGRES_PASSWORD: test options: >- --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + # --ulimit: dev-01's Docker gives containers a soft limit of 1024 open files, which the suite's collections + # exhaust (WiredTiger panics on "Too many open files"); GitHub's hosted runners allow far more. mongo: image: mongo:7 options: >- + --ulimit nofile=64000:64000 --health-cmd "mongosh --quiet --eval \"db.adminCommand('ping')\"" --health-interval 5s --health-timeout 5s --health-retries 10 # The tracking server the MLflow tests log to. --allowed-hosts: it is reached by name, not localhost. From 3df4531fdffdb31e2f72e17ce04aa361d53ef81e Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 09:30:30 -0700 Subject: [PATCH 3/4] CI: name the Mongo service testmongo The URI-redaction tests assert the store's host name never appears in an error; with the host named mongo, the connection's own name (mongo-) matched it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d8bc0d0..c8adff14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,11 @@ jobs: POSTGRES_PASSWORD: test options: >- --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + # Named testmongo, not mongo: the tests check that errors never carry the server's host name, and a + # connection's name (mongo-) holds "mongo". # --ulimit: dev-01's Docker gives containers a soft limit of 1024 open files, which the suite's collections # exhaust (WiredTiger panics on "Too many open files"); GitHub's hosted runners allow far more. - mongo: + testmongo: image: mongo:7 options: >- --ulimit nofile=64000:64000 @@ -50,7 +52,7 @@ jobs: --allowed-hosts mlflow,mlflow:5000 env: RAINFOREST_TEST_DATABASE_URL: postgresql://postgres:test@postgres:5432/postgres - RAINFOREST_TEST_MONGO_URL: mongodb://mongo:27017 + RAINFOREST_TEST_MONGO_URL: mongodb://testmongo:27017 PIP_ROOT_USER_ACTION: ignore PIP_DISABLE_PIP_VERSION_CHECK: "1" defaults: From f4f2fd56ffcefa36a637098a9534953eb5d7c099 Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 09:53:40 -0700 Subject: [PATCH 4/4] CI: give the API job container dev-01's upstream DNS systemd-resolved on dev-01 takes 12-15 s to fail a single-label name over LLMNR; the own-address guard's lookups of the Compose names give up after 2 s, so the guard tests depended on timing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8adff14..b27218c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,11 @@ jobs: timeout-minutes: 45 # In python:3.12 (the image api/Dockerfile.test builds on): setup-python has no builds for dev-01's Debian, and # from a job container the services are reached by name on the job's network, with no host ports. + # --dns: dev-01's upstream resolvers, bypassing its systemd-resolved, whose LLMNR takes 12-15 s to fail a + # single-label name. The own-address guard looks up the Compose names (api, db, ...) and gives up after 2 s. container: image: python:3.12 + options: --dns 10.2.1.108 --dns 1.1.1.1 volumes: - rf-ci-pip-cache:/root/.cache/pip services: