From 2ee4e2c54388dc995076685952b26746217e44ac Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 10:27:09 -0700 Subject: [PATCH 1/5] Test pass: stale tests after phases 9b-11 (its-team copy, member Settings sections, pick mode no longer the default), MLFLOW_LOGGING in Compose, config.py's settings read allowed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The 9b final review (M9) chose "first in its team" on purpose; the unit, e2e and user-guide copy follow it. - Settings › Application is admin-only since phase 10; the member section list drops it. - helm: pick mode's case sets auth.mode=pick and devPick; a new case pins the password default. - CI stack job and smoke.sh seed mode: the stack specs sign in by picking, so the stack runs pick with the dev flag. - docker-compose.yml passes MLFLOW_LOGGING (test_config_wiring). - test_access_lint: config.py reads meta.app_settings with its own connection (db.py's pool is built from get_settings(), which calls it), listed in DRIVER_HOMES with that reason. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- .env.example | 2 ++ .github/workflows/ci.yml | 2 ++ api/tests/test_access_lint.py | 3 +++ deploy/chart/rainforest/tests/api_test.yaml | 16 ++++++++++++++++ deploy/smoke.sh | 5 ++++- docker-compose.yml | 1 + docs/user-guide.md | 2 +- ui/e2e/fixture/gpuQueue.spec.ts | 4 ++-- ui/src/gpuQueue.test.ts | 4 ++-- ui/src/kit/kit.test.tsx | 2 +- ui/src/pages/notebooks/notebooksPage.test.tsx | 2 +- ui/src/theme.test.ts | 7 ++++--- 12 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 27c4428b..b9d0e860 100644 --- a/.env.example +++ b/.env.example @@ -127,6 +127,8 @@ RAINFOREST_SECRET_KEY= # MLflow tracking (docs/mlflow.md). Unset: the compose mlflow service at http://mlflow:5000/mlflow. # Set it empty to turn logging off. # MLFLOW_TRACKING_URI= +# false stops logging runs while the address stays set; admins also change it on Settings › Application. +# MLFLOW_LOGGING=true # The address browsers open MLflow at, when it isn't the compose server behind /mlflow/ (an existing MLflow server). # MLFLOW_BROWSE_URL= # 0 removes the UI's /mlflow/ proxy (1 by default). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b27218c8..9ec2eb36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,8 @@ jobs: run: | echo "COMPOSE_PROJECT_NAME=rf-ci-stack-${RUNNER_NAME,,}" >> "$GITHUB_ENV" { echo UI_PORT=127.0.0.1:0; echo API_PORT=0; echo DB_PORT=0; } >> "$GITHUB_ENV" + # The stack specs sign in by picking a user; password sign-in is the default, so turn the dev mode on. + { echo RAINFOREST_AUTH_MODE=pick; echo RAINFOREST_DEV_PICK=true; } >> "$GITHUB_ENV" - name: Remove a stack an earlier run left run: docker compose down -v --remove-orphans - name: Start the stack diff --git a/api/tests/test_access_lint.py b/api/tests/test_access_lint.py index 7b033ee4..2a702ce2 100644 --- a/api/tests/test_access_lint.py +++ b/api/tests/test_access_lint.py @@ -331,6 +331,9 @@ def test_secrets_table_only_in_store(): "sources/postgres.py", "sources/mongodb.py", "sources/s3.py", "sample_mongo.py", "sample_s3.py", "connections/netguard.py", "db.py", "worker.py", "jobevents.py", # Rainforest's own database: psycopg.connect + # The in-app settings (meta.app_settings, phase 10): read from the environment's database_url before the merged + # settings exist; db.py's pool is built from get_settings(), which calls this, so going through db.py would recurse. + "config.py", } # fmt: skip diff --git a/deploy/chart/rainforest/tests/api_test.yaml b/deploy/chart/rainforest/tests/api_test.yaml index 16e61948..c18004c2 100644 --- a/deploy/chart/rainforest/tests/api_test.yaml +++ b/deploy/chart/rainforest/tests/api_test.yaml @@ -329,14 +329,30 @@ tests: path: data.RAINFOREST_OIDC_DISCOVERY_URL value: http://keycloak:8080/.well-known/openid-configuration + # Phase 11: built-in accounts are the default, and pick mode never turns on without devPick. + - it: defaults to password mode with the development pick flag off + template: configmap.yaml + asserts: + - equal: + path: data.RAINFOREST_AUTH_MODE + value: password + - equal: + path: data.RAINFOREST_DEV_PICK + value: "false" + - it: carries pick mode's user list template: configmap.yaml set: + auth.mode: pick + auth.devPick: true auth.users: "tanner:Tanner:admin" asserts: - equal: path: data.RAINFOREST_AUTH_MODE value: pick + - equal: + path: data.RAINFOREST_DEV_PICK + value: "true" - equal: path: data.RAINFOREST_USERS value: "tanner:Tanner:admin" diff --git a/deploy/smoke.sh b/deploy/smoke.sh index a9c56185..d4192227 100755 --- a/deploy/smoke.sh +++ b/deploy/smoke.sh @@ -21,6 +21,8 @@ # # Environment: # BASE seed mode: the URL of a running stack's UI (pick sign-in, user "Admin"); nothing is built or deleted. +# Password sign-in is the default, so start that stack with RAINFOREST_AUTH_MODE=pick and +# RAINFOREST_DEV_PICK=true (development only; the chart path sets both from ci/dev-values.yaml). # REGISTRY image repository prefix: /{api,ui,mlflow,notebook}: (default: rainforest) # IMAGE_TAG tag of the four images (default: smoke) # BUILD 1: build the images from api/, ui/, mlflow/ and notebook/ (default). 0: use images already present @@ -299,7 +301,8 @@ check_reachable() { [ "$SECONDS" -ge "$deadline" ] && fail "$BASE/api/auth/config does not answer after ${FORWARD_TIMEOUT}s" sleep 1 done - jq -e '.mode == "pick"' "$WORK/config" >/dev/null || fail "unexpected /api/auth/config: $(cat "$WORK/config")" + jq -e '.mode == "pick"' "$WORK/config" >/dev/null \ + || fail "the stack must run RAINFOREST_AUTH_MODE=pick with RAINFOREST_DEV_PICK=true; /api/auth/config: $(cat "$WORK/config")" log "seeding $BASE: $(jq -c . "$WORK/config")" } diff --git a/docker-compose.yml b/docker-compose.yml index 2d0c4906..0c38991a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ x-api-env: &api-env RAINFOREST_CONNECT_DENY: ${RAINFOREST_CONNECT_DENY:-127.0.0.0/8,::1/128,0.0.0.0/8,169.254.0.0/16,fe80::/10,fd00:ec2::254/128,100.100.100.200/32,168.63.129.16/32,224.0.0.0/4,255.255.255.255/32,ff00::/8,fc00::/7} # Empty turns MLflow logging off. The server serves everything, its REST API too, under /mlflow. MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI-http://mlflow:5000/mlflow} + MLFLOW_LOGGING: ${MLFLOW_LOGGING:-true} MLFLOW_BROWSE_URL: ${MLFLOW_BROWSE_URL:-} MLFLOW_EXPERIMENT_PREFIX: ${MLFLOW_EXPERIMENT_PREFIX:-rainforest} # MLflow's own pages cannot apply project access: closed unless true (docs/mlflow.md). diff --git a/docs/user-guide.md b/docs/user-guide.md index bf64b1ea..752ed511 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -786,7 +786,7 @@ text and on the job's page), on the run, and on the Lab's card: - *Waiting for the GPU · about 3 ahead*: three jobs are in front of it. *About* means the count is Kueue's and moves as teams use the card; the built-in queue gives an exact count and drops *about*. -- *first in your team, Fraud* or *2 ahead in your team, Fraud*: its place within its team's line. Work from projects +- *first in its team, Fraud* or *2 ahead in its team, Fraud*: its place within its team's line. Work from projects without a team waits in the shared queue (*first in the shared queue*). - *in use by Ada's Lab (Risk), at most 1 h 40 min left*: who holds the card. *At most* is an upper bound, because a Lab or study can end sooner; a holder past its bound reads *finishing*. Work from a project you can't see is named diff --git a/ui/e2e/fixture/gpuQueue.spec.ts b/ui/e2e/fixture/gpuQueue.spec.ts index e353dddf..d5c576f7 100644 --- a/ui/e2e/fixture/gpuQueue.spec.ts +++ b/ui/e2e/fixture/gpuQueue.spec.ts @@ -28,7 +28,7 @@ test("the Jobs page shows a waiting line with its team line and who orders it", const row = page.locator('[data-testid="job-row"][data-job-id="9101"]'); await expect(row).toBeVisible(); await expect(row.getByTestId("job-waiting")).toHaveText("Waiting for the GPU · about 2 ahead"); - await expect(row.getByTestId("job-waiting")).toHaveAttribute("title", /first in your team, Fraud/); + await expect(row.getByTestId("job-waiting")).toHaveAttribute("title", /first in its team, Fraud/); await expect(row.getByTestId("job-waiting")).toHaveAttribute( "title", /in use by Tuned · XGBoost study \(Risk\), at most/, @@ -48,7 +48,7 @@ test("a GPU Lab waits in line and starts on the CPU instead", async ({ page }) = await page.goto("/notebooks"); const card = page.getByTestId("lab-card"); await expect(page.getByTestId("lab-status")).toHaveText("Queued"); - await expect(page.getByTestId("lab-waiting-detail")).toContainText("first in your team, Fraud"); + await expect(page.getByTestId("lab-waiting-detail")).toContainText("first in its team, Fraud"); await card.getByTestId("lab-to-cpu").click(); await expect(page.getByRole("dialog")).toContainText("Start your Lab on the CPU instead of waiting for the GPU?"); await page.getByRole("dialog").getByRole("button", { name: "Start on CPU instead" }).click(); diff --git a/ui/src/gpuQueue.test.ts b/ui/src/gpuQueue.test.ts index 580f80e4..3aa220b9 100644 --- a/ui/src/gpuQueue.test.ts +++ b/ui/src/gpuQueue.test.ts @@ -32,10 +32,10 @@ describe("waitingDetail", () => { it("waiting: the team's place and the holder with the time it has left", () => { const holder = { label: "Tuned · XGBoost study", team: "Risk", since: at(8, 0), ends_by: at(11, 40) }; expect(detail({ in_use_by: holder })).toBe( - "(first in your team, Fraud) — in use by Tuned · XGBoost study (Risk), at most 1 h 40 min left", + "(first in its team, Fraud) — in use by Tuned · XGBoost study (Risk), at most 1 h 40 min left", ); expect(detail({ ahead_in_team: 3, in_use_by: { ...holder, ends_by: null, team: null } })).toBe( - "(3 ahead in your team, Fraud) — in use by Tuned · XGBoost study", + "(3 ahead in its team, Fraud) — in use by Tuned · XGBoost study", ); }); it("next: who holds it, since when", () => { diff --git a/ui/src/kit/kit.test.tsx b/ui/src/kit/kit.test.tsx index d4762728..8e2a6a11 100644 --- a/ui/src/kit/kit.test.tsx +++ b/ui/src/kit/kit.test.tsx @@ -91,7 +91,7 @@ describe("JobStatus waiting", () => { const el = render(); const line = el.querySelector('[data-testid="job-waiting"]')!; expect(line.textContent).toBe("Waiting for the GPU · about 2 ahead"); - expect(line.getAttribute("title")).toBe("(1 ahead in your team, Fraud)"); + expect(line.getAttribute("title")).toBe("(1 ahead in its team, Fraud)"); }); it("shows nothing for a running job", () => { diff --git a/ui/src/pages/notebooks/notebooksPage.test.tsx b/ui/src/pages/notebooks/notebooksPage.test.tsx index 2e46d984..9464b116 100644 --- a/ui/src/pages/notebooks/notebooksPage.test.tsx +++ b/ui/src/pages/notebooks/notebooksPage.test.tsx @@ -289,7 +289,7 @@ describe("GPU Labs", () => { await mount("/notebooks", ); expect(byId("lab-waiting")?.textContent).toBe("Waiting for the GPU · about 2 ahead"); expect(text()).not.toContain("Waiting for a slot"); - expect(byId("lab-waiting-detail")?.textContent).toBe("(1 ahead in your team, Fraud) — in use by Ann's Lab (Risk)"); + expect(byId("lab-waiting-detail")?.textContent).toBe("(1 ahead in its team, Fraud) — in use by Ann's Lab (Risk)"); expect(byId("stop-lab")?.textContent).toBe("Cancel"); await act(async () => byId("lab-to-cpu")!.click()); await settle(); diff --git a/ui/src/theme.test.ts b/ui/src/theme.test.ts index bf6baadb..aa75a486 100644 --- a/ui/src/theme.test.ts +++ b/ui/src/theme.test.ts @@ -38,8 +38,10 @@ describe("theme preference", () => { }); describe("settings sections", () => { - it("shows Tags to admins and project owners and Storage to everyone", () => { - expect(visibleSections({ role: "admin" }).map((s) => s.key)).toEqual(expect.arrayContaining(["tags", "storage"])); + it("shows Tags to admins and project owners, Storage to everyone, and Application only to admins", () => { + expect(visibleSections({ role: "admin" }).map((s) => s.key)).toEqual( + expect.arrayContaining(["tags", "storage", "gpu-queue", "application"]), + ); for (const role of ["member"] as const) { expect(visibleSections({ role }).map((s) => s.key)).toEqual([ "profile", @@ -48,7 +50,6 @@ describe("settings sections", () => { "notebooks", "storage", "integrations", - "application", ]); } }); From da19f26c9559af42094b0f796f213dce26ce2106 Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 10:33:03 -0700 Subject: [PATCH 2/5] Testing: Phase 11 password accounts (hashing, lockout, no enumeration, one-time links, must-change, pick needs the dev flag) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- api/tests/test_passwords.py | 512 ++++++++++++++++++++++++++++++++++++ 1 file changed, 512 insertions(+) create mode 100644 api/tests/test_passwords.py diff --git a/api/tests/test_passwords.py b/api/tests/test_passwords.py new file mode 100644 index 00000000..e4d198ba --- /dev/null +++ b/api/tests/test_passwords.py @@ -0,0 +1,512 @@ +"""Phase 11: password accounts. Hashing and the password rules, the sign-in throttle (account and address), one answer +for every refusal, the one-time reset / invite / setup links, must-change-on-first-sign-in, and pick mode needing the +dev flag. Needs the test database; each test signs in from an address of its own so throttles never meet.""" + +import asyncio +import hashlib +import uuid + +import pytest +from fastapi.testclient import TestClient + +from app import auth, db, passwords, users +from app.config import get_settings +from app.main import app + +from .team_helpers import DB_URL, app_ # noqa: F401 + +pytestmark = pytest.mark.skipif(not DB_URL, reason="set RAINFOREST_TEST_DATABASE_URL to run the API tests") + +PW = "Lilac-Harbour-Tangent-77" +NEW_PW = "Quartz-Meadow-Lantern-42" +BAD = {"detail": passwords.BAD_CREDENTIALS} + + +@pytest.fixture(autouse=True) +def password_mode(app_, monkeypatch): + """Password sign-in (the default), pick off, one proxy hop.""" + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "password") + monkeypatch.delenv("RAINFOREST_DEV_PICK", raising=False) + monkeypatch.delenv("RAINFOREST_CLIENT_IP_HOPS", raising=False) + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _tag() -> str: + return uuid.uuid4().hex[:10] + + +def ip() -> str: + return f"ip-{_tag()}" + + +def make(*, role="member", password: str | None = PW, must_change=False, active=True, email="") -> dict: + """A password account (no password when `password` is None); returns the user with its email.""" + email = email or f"pw-{_tag()}@example.com" + user = users.create(f"Pw {_tag()}", email, role) + with db.connection(system=True) as conn: + if password is not None: + passwords.set_password(conn, user["id"], password, must_change=must_change) + if not active: + conn.execute("UPDATE meta.users SET active = false WHERE id = %s", (user["id"],)) + return {**user, "email": email} + + +def sql(query: str, params=()) -> list[dict]: + with db.connection(system=True) as conn: + return conn.execute(query, params).fetchall() + + +def login(client, email, password, address=None, **headers): + return client.post( + "/api/auth/login", + json={"email": email, "password": password}, + headers={"X-Forwarded-For": address or ip(), **headers}, + ) + + +def throttle(kind: str, key: str) -> dict | None: + rows = sql("SELECT * FROM meta.sign_in_throttle WHERE kind = %s AND key = %s", (kind, key)) + return rows[0] if rows else None + + +def account_key(email: str) -> str: + return hashlib.sha256(email.strip().lower().encode()).hexdigest() + + +def link_for(user_id: int, purpose="reset", created_by=None) -> str: + with db.connection(system=True) as conn: + return passwords.issue_link(conn, user_id, purpose, created_by)["url"].split("token=", 1)[1] + + +@pytest.fixture +def client(app_): + with TestClient(app_) as c: + yield c + + +def admin_client(app_) -> TestClient: + admin = make(role="admin") + c = TestClient(app_) + r = login(c, admin["email"], PW) + assert r.status_code == 200, r.text + return c + + +# ------------------------------------------------------------------------------------------- hashing +def test_hash_is_argon2id_and_never_the_plaintext(): + user = make() + (row,) = sql("SELECT password_hash FROM meta.user_credentials WHERE user_id = %s", (user["id"],)) + assert row["password_hash"].startswith("$argon2id$") + assert PW not in row["password_hash"] + assert passwords.hash_password(PW) != passwords.hash_password(PW) # salted + + +def test_verify_right_wrong_and_missing(): + stored = passwords.hash_password(PW) + assert passwords.verify(stored, PW) + assert not passwords.verify(stored, PW + "x") + assert not passwords.verify(None, PW) + assert not passwords.verify("not a hash", PW) + + +def test_login_with_the_right_password_signs_in(client): + user = make() + r = login(client, user["email"].upper(), PW) # an email is matched without regard to case + assert r.status_code == 200, r.text + assert r.json()["id"] == user["id"] and r.json()["must_change_password"] is False + assert client.get("/api/auth/me").json()["id"] == user["id"] + + +@pytest.mark.parametrize( + "password,needle", + [("short1!", "at least"), ("password1234", "too common"), ("aaaaaaaaaaaaaa", "too common"), ("x" * 300, "at most")], +) +def test_refused_passwords(password, needle): + assert needle in passwords.problem(password) + + +def test_a_password_made_of_the_email_or_name_is_refused(): + assert "email or name" in passwords.problem("jane.doe@example.com", email="jane.doe@example.com") + assert "email or name" in passwords.problem("BartholomewSmith1!", email="a@b.com", name="Bartholomew Smith") + assert passwords.problem(PW, email="a@b.com", name="Jane") is None + + +def test_weak_passwords_are_refused_where_they_are_set(app_): + admin = admin_client(app_) + r = admin.post("/api/admin/people", json={"name": "Weak", "email": f"w-{_tag()}@example.com", "password": "short"}) + assert r.status_code == 422 + r = admin.post( + "/api/admin/people", json={"name": "Weak", "email": f"w-{_tag()}@example.com", "password": "password1234"} + ) + assert r.status_code == 422 and "common" in r.json()["detail"] + user = make() + token = link_for(user["id"]) + c = TestClient(app_) + r = c.post("/api/auth/password-token", json={"token": token, "password": "password1234"}) + assert r.status_code == 422 + assert c.post("/api/auth/password-token/check", json={"token": token}).status_code == 200 # not used up + + +# ------------------------------------------------------------------------------------------ lockout +def test_five_wrong_passwords_lock_the_account_and_even_the_right_one_is_refused(client): + user = make() + address = ip() + for _ in range(5): + assert login(client, user["email"], "Wrong-Password-123", address).status_code == 401 + assert throttle("account", account_key(user["email"]))["locked_until"] is not None + r = login(client, user["email"], PW, address) # the right password, still locked + assert r.status_code == 429 and "Try again in 15 minutes" in r.json()["detail"] + r = login(client, user["email"], PW, ip()) # a different address, same account + assert r.status_code == 429 + + +def test_the_lock_ends(client): + user = make() + for _ in range(5): + login(client, user["email"], "Wrong-Password-123") + sql( + "UPDATE meta.sign_in_throttle SET locked_until = now() - interval '1 second' WHERE kind = 'account' AND key = %s " + "RETURNING key", + (account_key(user["email"]),), + ) + assert login(client, user["email"], PW).status_code == 200 + assert throttle("account", account_key(user["email"])) is None # a success clears the count + + +def test_a_success_resets_the_failure_count(client): + user = make() + for _ in range(4): + login(client, user["email"], "Wrong-Password-123") + assert login(client, user["email"], PW).status_code == 200 + for _ in range(4): + assert login(client, user["email"], "Wrong-Password-123").status_code == 401 + assert login(client, user["email"], PW).status_code == 200 + + +def test_the_lock_is_stored_before_the_refusal(client): + """The sixth attempt is refused with 429, and the lock it sets must survive the refusal (a43613e: raising inside the + transaction rolled it back).""" + user = make() + key = account_key(user["email"]) + sql("INSERT INTO meta.sign_in_throttle (kind, key, failures) VALUES ('account', %s, 5) RETURNING key", (key,)) + r = login(client, user["email"], PW) + assert r.status_code == 429 + row = throttle("account", key) + assert row is not None and row["locked_until"] is not None + assert login(client, user["email"], PW).status_code == 429 + + +def test_an_address_that_fails_twenty_times_is_locked(client): + address = ip() + for i in range(passwords.IP_LIMIT): + assert login(client, f"nobody-{i}-{_tag()}@example.com", "Wrong-Password-123", address).status_code == 401 + assert throttle("ip", address)["locked_until"] is not None + user = make() + assert login(client, user["email"], PW, address).status_code == 429 + assert login(client, user["email"], PW, ip()).status_code == 200 + + +def test_the_client_address_is_the_forwarded_entry_hops_from_the_right(client, monkeypatch): + user = make() + mine, theirs = ip(), ip() + # One hop (the default): the last entry; what a client puts before it is ignored. + assert login(client, user["email"], "Wrong-Password-123", f"{theirs}, {mine}").status_code == 401 + assert throttle("ip", mine) and throttle("ip", theirs) is None + # Two hops: the second from the right. + monkeypatch.setenv("RAINFOREST_CLIENT_IP_HOPS", "2") + get_settings.cache_clear() + mine2, proxy = ip(), ip() + login(client, user["email"], "Wrong-Password-123", f"{mine2}, {proxy}") + assert throttle("ip", mine2) and throttle("ip", proxy) is None + # Zero hops: the peer, whatever the header says. + monkeypatch.setenv("RAINFOREST_CLIENT_IP_HOPS", "0") + get_settings.cache_clear() + spoof = ip() + login(client, user["email"], "Wrong-Password-123", spoof) + assert throttle("ip", spoof) is None + + +def test_a_spoofed_leftmost_entry_does_not_dodge_the_address_lock(client): + real = ip() + for i in range(passwords.IP_LIMIT): + login(client, f"nobody-{i}-{_tag()}@example.com", "Wrong-Password-123", f"spoof-{i}, {real}") + user = make() + assert login(client, user["email"], PW, f"another, {real}").status_code == 429 + + +# --------------------------------------------------------------------------------------- enumeration +def test_every_refusal_looks_the_same(client): + known = make() + inactive = make(active=False) + no_password = make(password=None) + cases = [ + login(client, f"ghost-{_tag()}@example.com", PW), + login(client, known["email"], "Wrong-Password-123"), + login(client, inactive["email"], PW), + login(client, no_password["email"], PW), + ] + assert {r.status_code for r in cases} == {401} + assert all(r.json() == BAD for r in cases) + assert not any("set-cookie" in r.headers for r in cases) + + +def test_an_unknown_email_locks_like_a_real_one(client): + ghost = f"ghost-{_tag()}@example.com" + real = make() + ghost_codes = [login(client, ghost, "Wrong-Password-123").status_code for _ in range(7)] + real_codes = [login(client, real["email"], "Wrong-Password-123").status_code for _ in range(7)] + assert ghost_codes == real_codes == [401] * 5 + [429] * 2 + bodies = { + login(client, ghost, "x").json()["detail"], + login(client, real["email"], "x").json()["detail"], + } + assert len(bodies) == 1 + + +def test_the_failures_are_audited_with_a_reason(client): + known = make() + login(client, known["email"], "Wrong-Password-123") + rows = sql( + "SELECT detail->>'reason' AS reason FROM meta.audit WHERE action = 'auth.sign_in_failed' AND subject_user = %s", + (known["id"],), + ) + assert [r["reason"] for r in rows] == ["wrong"] + + +# -------------------------------------------------------------------------------------- one-time links +def test_a_link_is_stored_hashed_and_works_once(client, app_): + user = make() + token = link_for(user["id"]) + stored = sql("SELECT hash, purpose FROM meta.password_tokens WHERE user_id = %s", (user["id"],)) + assert [r["hash"] for r in stored] == [hashlib.sha256(token.encode()).hexdigest()] and token not in stored[0][ + "hash" + ] + info = client.post("/api/auth/password-token/check", json={"token": token}) + assert info.status_code == 200 + assert info.json() == { + "name": user["name"], + "email": user["email"], + "purpose": "reset", + "needs_email": False, + } + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 200 and r.json()["id"] == user["id"] + assert client.get("/api/auth/me").json()["id"] == user["id"] # signed in by it + # Used: both the check and a second redemption are refused, with the same message. + again = TestClient(app_) + for r in ( + again.post("/api/auth/password-token/check", json={"token": token}), + again.post("/api/auth/password-token", json={"token": token, "password": "Another-Fine-Pass-99"}), + ): + assert r.status_code == 404 and r.json()["detail"] == passwords.BAD_LINK + assert login(again, user["email"], NEW_PW).status_code == 200 + assert login(again, user["email"], PW).status_code == 401 + + +def test_an_expired_link_is_refused(client): + user = make() + token = link_for(user["id"]) + sql( + "UPDATE meta.password_tokens SET expires_at = now() - interval '1 second' WHERE user_id = %s RETURNING id", + (user["id"],), + ) + assert client.post("/api/auth/password-token/check", json={"token": token}).status_code == 404 + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 404 and r.json()["detail"] == passwords.BAD_LINK + assert login(client, user["email"], PW).status_code == 200 # the old password is untouched + + +def test_lifetimes_by_purpose(): + user = make() + with db.connection(system=True) as conn: + for purpose, hours in passwords.TOKEN_HOURS.items(): + link = passwords.issue_link(conn, user["id"], purpose, None) + row = conn.execute( + "SELECT extract(epoch FROM expires_at - now()) / 3600 AS h FROM meta.password_tokens WHERE hash = %s", + (passwords._token_hash(link["url"].split("token=", 1)[1]),), + ).fetchone() + assert hours - 0.1 < row["h"] <= hours + assert (passwords.TOKEN_HOURS["reset"], passwords.TOKEN_HOURS["invite"]) == (24, 72) + + +def test_a_new_link_revokes_the_earlier_one(client): + user = make() + old = link_for(user["id"]) + new = link_for(user["id"], "invite") + assert client.post("/api/auth/password-token/check", json={"token": old}).status_code == 404 + assert client.post("/api/auth/password-token/check", json={"token": new}).json()["purpose"] == "invite" + + +def test_unknown_garbage_and_inactive_links_are_refused(client): + assert client.post("/api/auth/password-token/check", json={"token": "no-such-token"}).status_code == 404 + gone = make(active=False) + token = link_for(gone["id"]) + assert client.post("/api/auth/password-token/check", json={"token": token}).status_code == 404 + r = client.post("/api/auth/password-token", json={"token": "x" * 129, "password": NEW_PW}) + assert r.status_code == 422 + + +def test_redeeming_signs_the_person_out_elsewhere_and_clears_the_lock(app_): + user = make() + elsewhere = TestClient(app_) + assert login(elsewhere, user["email"], PW).status_code == 200 + for _ in range(5): + login(TestClient(app_), user["email"], "Wrong-Password-123") + assert throttle("account", account_key(user["email"]))["locked_until"] is not None + c = TestClient(app_) + token = link_for(user["id"]) + assert c.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}).status_code == 200 + assert elsewhere.get("/api/auth/me").status_code == 401 # session_version moved on + assert c.get("/api/auth/me").status_code == 200 + assert throttle("account", account_key(user["email"])) is None # the reset lifts a lock + + +def test_a_setup_link_for_an_account_without_an_email_asks_for_one(client, app_): + made = users.create(f"Setup {_tag()}", None, "admin") + token = link_for(made["id"], "setup") + info = client.post("/api/auth/password-token/check", json={"token": token}).json() + assert info["needs_email"] is True and info["purpose"] == "setup" + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 422 and "email" in r.json()["detail"] + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": "nope"}) + assert r.status_code == 422 + taken = make() + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": taken["email"]}) + assert r.status_code == 422 and "already" in r.json()["detail"] + email = f"setup-{_tag()}@example.com" + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": email}) + assert r.status_code == 200 and r.json()["email"] == email + assert login(TestClient(app_), email, NEW_PW).status_code == 200 + + +# ------------------------------------------------------------------ admin: accounts and links, must change +def test_an_admin_creates_a_person_with_an_initial_password_that_must_change(app_): + admin = admin_client(app_) + email = f"init-{_tag()}@example.com" + r = admin.post("/api/admin/people", json={"name": "Init", "email": email, "password": PW}) + assert r.status_code == 201, r.text + assert r.json()["setup_url"] is None + person = TestClient(app_) + r = login(person, email, PW) + assert r.status_code == 200 and r.json()["must_change_password"] is True + assert person.get("/api/auth/me").json()["must_change_password"] is True + blocked = person.get("/api/projects") + assert blocked.status_code == 403 and blocked.json()["detail"]["code"] == "must_change_password" + # Same password again is refused; a new one clears the flag and the rest of the app opens. + r = person.post("/api/auth/password", json={"current": PW, "new": PW}) + assert r.status_code == 422 + r = person.post("/api/auth/password", json={"current": "Wrong-Password-123", "new": NEW_PW}) + assert r.status_code == 400 and r.json()["detail"] == passwords.BAD_CURRENT + r = person.post("/api/auth/password", json={"current": PW, "new": NEW_PW}) + assert r.status_code == 200 and r.json()["must_change_password"] is False + assert person.get("/api/projects").status_code == 200 + assert login(TestClient(app_), email, NEW_PW).json()["must_change_password"] is False + + +def test_wrong_current_passwords_are_throttled_like_sign_in(app_): + person = make() + c = TestClient(app_) + assert login(c, person["email"], PW).status_code == 200 + codes = [ + c.post("/api/auth/password", json={"current": "Wrong-Password-123", "new": NEW_PW}).status_code + for _ in range(6) + ] + assert codes == [400] * 5 + [429] + + +def test_an_invite_link_is_returned_once_and_sets_the_password(app_): + admin = admin_client(app_) + email = f"inv-{_tag()}@example.com" + r = admin.post("/api/admin/people", json={"name": "Invitee", "email": email}) + assert r.status_code == 201, r.text + token = r.json()["setup_url"].split("token=", 1)[1] + assert r.json()["setup_url"].startswith("/set-password#token=") and r.json()["setup_expires_at"] + stored = sql( + "SELECT hash, purpose FROM meta.password_tokens WHERE hash = %s", (hashlib.sha256(token.encode()).hexdigest(),) + ) + assert stored and stored[0]["purpose"] == "invite" + assert token not in str(sql("SELECT * FROM meta.password_tokens WHERE user_id = %s", (r.json()["id"],))) + person = TestClient(app_) + assert login(person, email, PW).status_code == 401 # no password yet + r = person.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 200 and r.json()["must_change_password"] is False + assert person.get("/api/projects").status_code == 200 + + +def test_people_need_an_email_and_a_unique_one(app_): + admin = admin_client(app_) + assert admin.post("/api/admin/people", json={"name": "NoMail"}).status_code == 422 + taken = make() + r = admin.post("/api/admin/people", json={"name": "Dup", "email": taken["email"].upper()}) + assert r.status_code == 409 + + +def test_a_reset_link_leaves_the_old_password_working_until_used(app_): + admin = admin_client(app_) + person = make() + r = admin.post(f"/api/admin/people/{person['id']}/reset-link") + assert r.status_code == 200, r.text + token = r.json()["url"].split("token=", 1)[1] + c = TestClient(app_) + assert login(c, person["email"], PW).status_code == 200 + assert c.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}).status_code == 200 + assert login(TestClient(app_), person["email"], PW).status_code == 401 + assert login(TestClient(app_), person["email"], NEW_PW).status_code == 200 + + +def test_reset_links_are_admin_only_and_refuse_unsuitable_people(app_): + admin = admin_client(app_) + member = make() + other = make() + c = TestClient(app_) + login(c, member["email"], PW) + assert c.post(f"/api/admin/people/{other['id']}/reset-link").status_code == 403 + assert TestClient(app_).post(f"/api/admin/people/{other['id']}/reset-link").status_code == 401 + assert admin.post("/api/admin/people/999999999/reset-link").status_code == 404 + assert admin.post(f"/api/admin/people/{make(active=False)['id']}/reset-link").status_code == 409 + no_email = users.create(f"NoMail {_tag()}", None, "member") + assert admin.post(f"/api/admin/people/{no_email['id']}/reset-link").status_code == 409 + + +# --------------------------------------------------------------------------------------------- pick mode +def test_pick_without_the_dev_flag_is_refused(client, monkeypatch): + user = make() + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + get_settings.cache_clear() + assert "RAINFOREST_DEV_PICK" in auth.config_problem() + assert client.post("/api/auth/pick", json={"user_id": user["id"]}).status_code in (404, 503) + assert client.get("/api/auth/users").status_code in (404, 503) + assert client.get("/api/auth/me").status_code == 503 # every other route says why + assert not client.cookies + assert client.get("/api/auth/config").json()["problem"] + + +def test_the_server_does_not_start_in_pick_mode_without_the_flag(monkeypatch): + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + get_settings.cache_clear() + + async def start(): + async with app.router.lifespan_context(app): + pass + + with pytest.raises(SystemExit, match="RAINFOREST_DEV_PICK"): + asyncio.run(start()) + + +def test_pick_works_with_the_flag(client, monkeypatch): + user = make() + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + monkeypatch.setenv("RAINFOREST_DEV_PICK", "true") + get_settings.cache_clear() + assert any(u["id"] == user["id"] for u in client.get("/api/auth/users").json()) + r = client.post("/api/auth/pick", json={"user_id": user["id"]}) + assert r.status_code == 200 and r.json()["mode"] == "pick" + assert client.get("/api/auth/me").json()["id"] == user["id"] + + +def test_pick_and_login_are_off_in_the_wrong_mode(client): + user = make() + assert client.post("/api/auth/pick", json={"user_id": user["id"]}).status_code == 404 + assert client.get("/api/auth/users").status_code == 404 From a8daba22cd269d1f2eb13a67273ce4f43ec9522a Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 10:37:58 -0700 Subject: [PATCH 3/5] Test pass: the API suite green after phases 9b-11, and tests for phase 10 and 9b D1 Stale tests (the code changed on purpose): - test_auth: pick mode needs RAINFOREST_DEV_PICK; the in-memory store stubs the sign-in audit and credentials; /me's new password fields; the pick is audited. - test_oidc: /api/auth/config names which sign-ins are on (password, oidc). - test_mlflow: emails are unique since 046, so the trainer Ana is made once. - test_gpu_waiting: a study's ends_by is policy.study_max_exec_s (its hours plus the last trial's margin). - test_gpu_supervise: a cluster blip on create requeues without burning a try (counts=False). - test_rls_app: the replay tripwire moves to 046 (045 and 046 replay cleanly). - test_route_matrix: the vault table keeps a route's own gate (reset-link is a 404 in pick mode). New: - test_app_settings: an override applies at once and a reset drops the row; a Lab choice above NOTEBOOK_CPU_LIMIT/NOTEBOOK_MEMORY_LIMIT is refused with nothing saved; every change is audited; bad values and members refused. - 9b D1: WaitingForQuota is in line (dispatch and kube.workload_state), the redaction filter keeps an untouched record's args (uvicorn's AccessFormatter), the sweep leaves a Job's own pod. Each fails with its fix reverted. - FakeKueue: a Job's pod carries the template's GPU and job-id labels and an ownerReferences to its Job, and objects() reports owner_kind as the real client does (the baa26aa class of bug); wait_for_quota(). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- api/tests/fakes/kueue.py | 34 ++++++-- api/tests/test_app_settings.py | 120 +++++++++++++++++++++++++++ api/tests/test_auth.py | 11 ++- api/tests/test_connection_secrets.py | 22 +++++ api/tests/test_gpu_dispatch.py | 50 +++++++++++ api/tests/test_gpu_supervise.py | 3 +- api/tests/test_gpu_waiting.py | 4 +- api/tests/test_mlflow.py | 6 +- api/tests/test_oidc.py | 2 + api/tests/test_rls_app.py | 2 +- api/tests/test_route_matrix.py | 5 +- 11 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 api/tests/test_app_settings.py diff --git a/api/tests/fakes/kueue.py b/api/tests/fakes/kueue.py index 763e63fb..b545519f 100644 --- a/api/tests/fakes/kueue.py +++ b/api/tests/fakes/kueue.py @@ -5,7 +5,10 @@ adds a pending Workload owned by it; a Job's pod appears when it is admitted (a Job without the label is not gated: its pod appears at once). Creating a name that exists, deleting or not, raises `Conflict`. -What a test drives: `admit`, `evict`, `inadmissible`, `finish` (by the owner's name), `set_pending(lq, items)`, +A Job's pod is made from its template, as Kubernetes makes it: the template's labels (the GPU and job-id labels +included, so `objects()` lists the pod beside its Job) and an `ownerReferences` entry naming the Job. + +What a test drives: `admit`, `evict`, `inadmissible`, `wait_for_quota`, `finish` (by the owner's name), `set_pending(lq, items)`, `set_pod`, `terminating`, `remove` (a terminating object finally goes), `set_queue`, `installed`, `down(True)`. What it checks: `created` (manifests), `deleted` ((kind, name)), `patched` ((kind, name, labels)), `ensured`. """ @@ -63,7 +66,7 @@ def _workload(self, owner: str) -> dict: def admit(self, owner_name: str) -> None: w = self._workload(owner_name) - w.update(admitted=True, evicted=None, quota_reserved=True, message=None) + w.update(admitted=True, evicted=None, quota_reserved=True, waiting_for_quota=False, message=None) if owner_name in self.jobs and not self.job_pods(owner_name): self._make_job_pod(owner_name) @@ -83,7 +86,13 @@ def reconcile(self) -> None: q.update(active=True, message=None, new=False) def inadmissible(self, owner_name: str, message: str) -> None: - self._workload(owner_name).update(admitted=False, quota_reserved=False, message=message) + self._workload(owner_name).update( + admitted=False, quota_reserved=False, waiting_for_quota=False, message=message + ) + + def wait_for_quota(self, owner_name: str, message: str = "couldn't assign flavors to pod set main") -> None: + """Live Kueue (v0.20): a Workload in line is QuotaReserved False with reason WaitingForQuota.""" + self._workload(owner_name).update(admitted=False, quota_reserved=False, waiting_for_quota=True, message=message) def finish(self, owner_name: str) -> None: self._workload(owner_name).update(finished=True) @@ -143,7 +152,14 @@ def _make_job_pod(self, job: str) -> None: labels = {**(tpl.get("metadata", {}).get("labels") or {}), "batch.kubernetes.io/job-name": job} self.core.create_namespaced_pod( self.ns, - {"metadata": {"name": f"{job}-{self._suffix(job)}", "labels": labels}, "spec": copy.deepcopy(tpl["spec"])}, + { + "metadata": { + "name": f"{job}-{self._suffix(job)}", + "labels": labels, + "ownerReferences": [{"apiVersion": "batch/v1", "kind": "Job", "name": job, "uid": f"uid-{job}"}], + }, + "spec": copy.deepcopy(tpl["spec"]), + }, ) def _drop(self, kind: str, name: str) -> None: @@ -199,7 +215,8 @@ def pending(self, local_queue: str) -> list[dict]: out = [] for i in self.pending_items.get(local_queue, []): w = self.wl.get(i.get("owner_name")) - if w is not None and (w["admitted"] or w["finished"] or w["quota_reserved"] is False): + inadmissible = w is not None and w["quota_reserved"] is False and not w.get("waiting_for_quota") + if w is not None and (w["admitted"] or w["finished"] or inadmissible): continue out.append(dict(i)) return out @@ -233,6 +250,7 @@ def create(self, manifest: dict) -> None: "evicted": None, "finished": False, "quota_reserved": None, + "waiting_for_quota": False, "message": None, } @@ -258,7 +276,7 @@ def objects(self) -> list[dict]: continue pods = self.job_pods(name) phase = pods[0].status.phase if pods else "Pending" - out.append(self._entry("Job", name, job["metadata"], phase, None, None, None)) + out.append(self._entry("Job", name, job["metadata"], phase, None, None, None, None)) return out + self.pod_objects() def pod_objects(self) -> list[dict]: @@ -281,11 +299,12 @@ def pod_objects(self) -> list[dict]: waiting, self.unsched.get(name), started, + next((r.kind for r in pod.metadata.owner_references or []), None), ) ) return out - def _entry(self, kind, name, meta, phase, waiting, unschedulable, started) -> dict: + def _entry(self, kind, name, meta, phase, waiting, unschedulable, started, owner_kind) -> dict: try: job_id = int((meta.get("labels") or {}).get(kube.JOB_ID_LABEL, "")) except ValueError: @@ -293,6 +312,7 @@ def _entry(self, kind, name, meta, phase, waiting, unschedulable, started) -> di return { "kind": kind, "name": name, + "owner_kind": owner_kind, "job_id": job_id, "queue": (meta.get("labels") or {}).get(kube.QUEUE_LABEL), "deleting": (kind, name) in self.gone, diff --git a/api/tests/test_app_settings.py b/api/tests/test_app_settings.py new file mode 100644 index 00000000..c5224bae --- /dev/null +++ b/api/tests/test_app_settings.py @@ -0,0 +1,120 @@ +"""Settings › Application (phase 10): an admin's change overrides the env value in every process (here, at once in the +saving one), a reset drops the row so the env default applies again, a Lab choice above the deployment's limit is +refused with nothing saved, and every change is audited.""" + +import pytest +from fastapi.testclient import TestClient + +from .team_helpers import app_ # noqa: F401 (fixture) + + +@pytest.fixture +def admin(app_, login_as): # noqa: F811 + from app import config, db + + def clean(): + db.execute("DELETE FROM meta.app_settings") + config.invalidate_overrides() + + clean() + yield login_as(TestClient(app_), "admin") + clean() + + +def _setting(body: dict, key: str) -> dict: + return next(s for s in body["settings"] if s["key"] == key) + + +def _audits(key: str) -> list[dict]: + from app import db + + return db.fetch_all( + "SELECT actor, detail FROM meta.audit WHERE action = 'settings.change' AND detail ->> 'key' = %s ORDER BY id", + (key,), + ) + + +def test_a_change_overrides_the_env_value_and_a_reset_brings_it_back(admin): + from app.config import env_settings, get_settings + + default = env_settings().job_log_keep_days + before = _setting(admin.get("/api/admin/settings").json(), "job_log_keep_days") + assert (before["value"], before["default"], before["overridden"]) == (default, default, False) + + r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": default + 7}}) + assert r.status_code == 200, r.text + row = _setting(r.json(), "job_log_keep_days") + assert (row["value"], row["default"], row["overridden"], row["updated_by"]) == (default + 7, default, True, "Admin") + assert get_settings().job_log_keep_days == default + 7 # the saving process applies it without waiting + assert env_settings().job_log_keep_days == default # the deployment's own value is untouched + + r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}}) + row = _setting(r.json(), "job_log_keep_days") + assert (row["value"], row["overridden"], row["updated_by"]) == (default, False, None) + assert get_settings().job_log_keep_days == default + from app import db + + assert db.fetch_one("SELECT 1 FROM meta.app_settings WHERE key = 'job_log_keep_days'") is None + + +def test_each_change_is_audited_with_its_old_and_new_value(admin): + from app.config import env_settings + + default = env_settings().job_log_keep_days + n = len(_audits("job_log_keep_days")) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}}) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}}) # no change, no row + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 120}}) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}}) + rows = _audits("job_log_keep_days")[n:] + assert [(r["detail"]["from"], r["detail"]["to"]) for r in rows] == [(None, 90), (90, 120), (120, None)] + assert all(r["actor"] == admin.user["id"] for r in rows) + assert all(r["detail"]["env"] == "JOB_LOG_KEEP_DAYS" and r["detail"]["default"] == default for r in rows) + + +def test_a_lab_choice_above_the_deployments_limit_is_refused_and_nothing_is_saved(admin): + from app.config import env_settings, get_settings + + env = env_settings() + cap = int(env.notebook_cpu_limit) + view = _setting(admin.get("/api/admin/settings").json(), "notebook_cpu_choices") + assert view["max"] <= cap and view["ceiling"] == f"NOTEBOOK_CPU_LIMIT = {env.notebook_cpu_limit}" + n = len(_audits("job_log_keep_days")) + + r = admin.patch( + "/api/admin/settings", json={"values": {"job_log_keep_days": 45, "notebook_cpu_choices": [1, cap + 1]}} + ) + assert r.status_code == 422 + assert "above the deployment's NOTEBOOK_CPU_LIMIT" in r.json()["detail"] + # all or nothing: the valid change beside it is not saved either + assert get_settings().job_log_keep_days == env.job_log_keep_days + assert get_settings().notebook_cpu_choices == env.notebook_cpu_choices + assert len(_audits("job_log_keep_days")) == n + + r = admin.patch("/api/admin/settings", json={"values": {"notebook_memory_choices_gb": [2, 4096]}}) + assert r.status_code == 422 and "NOTEBOOK_MEMORY_LIMIT" in r.json()["detail"] + + r = admin.patch("/api/admin/settings", json={"values": {"notebook_cpu_choices": [cap, 1, 1]}}) + assert r.status_code == 200, r.text + assert get_settings().notebook_cpu_choices == sorted({1, cap}) # sorted, duplicates dropped + + +@pytest.mark.parametrize( + "values, says", + [ + ({"database_url": "postgresql://x/y"}, "can't be changed in the app"), + ({"job_log_keep_days": 0}, "outside"), + ({"mlflow_logging": "yes"}, "not a valid value"), + ({"log_level": "loud"}, "one of"), + ({"notebook_cpu_choices": []}, "at least one"), + ], +) +def test_bad_values_are_refused_with_a_reason(admin, values, says): + r = admin.patch("/api/admin/settings", json={"values": values}) + assert r.status_code == 422 and says in r.json()["detail"], r.text + + +def test_members_can_neither_read_nor_change_them(app_, admin, login_as): # noqa: F811 + member = login_as(TestClient(app_), "owner") + assert member.get("/api/admin/settings").status_code == 403 + assert member.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 9}}).status_code == 403 diff --git a/api/tests/test_auth.py b/api/tests/test_auth.py index 5a5eb5f0..2b4260a4 100644 --- a/api/tests/test_auth.py +++ b/api/tests/test_auth.py @@ -49,6 +49,7 @@ def touch(self, user_id): def env(tmp_path, monkeypatch): monkeypatch.setenv("DATA_DIR", str(tmp_path)) monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + monkeypatch.setenv("RAINFOREST_DEV_PICK", "true") # phase 11: pick mode needs the development flag for name in ( "RAINFOREST_SESSION_SECRET", "RAINFOREST_ALLOWED_HOSTS", @@ -70,6 +71,10 @@ def store(monkeypatch): # No Postgres here: a user's scope is empty (these tests are about sessions, not about what a session may read). monkeypatch.setattr(access, "scope", lambda user, fresh=False: access.Scope(user["id"], user["role"] == "admin", 0)) monkeypatch.setattr(auth_module, "_my_projects", lambda user_id: []) + monkeypatch.setattr(auth_module.passwords, "credentials", lambda user_id: None) # pick-mode users have none + # Phase 11 audits every sign-in (meta.audit); here the rows are kept in memory. + fake.sign_ins = [] + monkeypatch.setattr(auth_module, "_audit_sign_in", lambda user_id, via, ip: fake.sign_ins.append((user_id, via))) return fake @@ -102,7 +107,7 @@ def test_protected_routes_need_a_session(client): assert r.status_code == 401 -def test_pick_me_logout(client): +def test_pick_me_logout(client, store): cfg = client.get("/api/auth/config").json() assert cfg["mode"] == "pick" and cfg["problem"] is None assert client.get("/api/auth/users").json() == [ @@ -112,6 +117,7 @@ def test_pick_me_logout(client): assert _pick(client, 99).status_code == 404 r = _pick(client, 2) assert r.status_code == 200 and r.json()["name"] == "Dana" + assert store.sign_ins == [(2, "pick")] # audited; the refused id 99 is not cookie = r.headers["set-cookie"] assert cookie.startswith(f"{auth.COOKIE}=") and "HttpOnly" in cookie and "SameSite=strict" in cookie assert "Max-Age=604800" in cookie and "Path=/" in cookie and "Secure" not in cookie @@ -124,6 +130,9 @@ def test_pick_me_logout(client): "avatar_url": None, "mode": "pick", "can_rename": True, + "has_password": False, + "must_change_password": False, + "can_change_password": False, "personal": None, "projects": [], "can_create_projects": True, diff --git a/api/tests/test_connection_secrets.py b/api/tests/test_connection_secrets.py index 1e97e6e6..94136bfc 100644 --- a/api/tests/test_connection_secrets.py +++ b/api/tests/test_connection_secrets.py @@ -216,6 +216,28 @@ def test_redactor(): assert "after the block: s3cr3t-value" in stream.getvalue() +def test_the_log_filter_leaves_records_it_does_not_change_alone(): + """9b D1 (57a5a54): uvicorn's access formatter unpacks a record's five args, so a record with nothing to hide keeps + them (emptying them logged a 'Logging error' traceback for every request).""" + from uvicorn.logging import AccessFormatter + + from app.connections import redact + + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False)) + handler.addFilter(redact.LogFilter()) + log = logging.getLogger(f"redact-access-{uuid.uuid4().hex}") + log.addHandler(handler) + log.propagate = False + args = ("10.0.0.1:5000", "GET", "/api/health", "1.1", 200) + with redact.secrets(["s3cr3t-value"]): + record = log.makeRecord(log.name, logging.INFO, __file__, 1, '%s - "%s %s HTTP/%s" %d', args, None) + log.handle(record) + assert record.args == args + assert '10.0.0.1:5000 - "GET /api/health HTTP/1.1" 200' in stream.getvalue() + + def test_validation_errors_never_echo_input(): from app import main diff --git a/api/tests/test_gpu_dispatch.py b/api/tests/test_gpu_dispatch.py index 5abed32f..c81b361e 100644 --- a/api/tests/test_gpu_dispatch.py +++ b/api/tests/test_gpu_dispatch.py @@ -147,6 +147,56 @@ def test_inadmissible_records_kueue_message(fake, dispatchers): assert _row(job)["args"]["gpu_admission"]["state"] == "admitted" +def test_a_workload_waiting_for_quota_is_in_line_not_inadmissible(fake, dispatchers): + """9b D1 (2ee089d): live Kueue v0.20.4 marks every queued Workload QuotaReserved False with reason WaitingForQuota; + that is a place in line, not "Can't queue for the GPU".""" + job = _job() + d = dispatchers() + d.reconcile() + name = _name(job, 1) + fake.wait_for_quota(name) + d.reconcile() + assert _row(job)["args"]["gpu_admission"]["state"] == "pending" + fake.inadmissible(name, "couldn't assign flavors: no flavor matches") + d.reconcile() + assert _row(job)["args"]["gpu_admission"]["state"] == "inadmissible" + + +@pytest.mark.parametrize( + "reason, waiting", + [("WaitingForQuota", True), ("Pending", True), ("Inadmissible", False), ("NoMatchingFlavor", False)], +) +def test_workload_state_reads_waiting_for_quota_from_the_reason(reason, waiting): + wl = { + "metadata": {"name": "job-x-0a1b2", "ownerReferences": [{"kind": "Job", "name": "x"}]}, + "status": {"conditions": [{"type": "QuotaReserved", "status": "False", "reason": reason, "message": "m"}]}, + } + state = kube.workload_state(wl) + assert (state["quota_reserved"], state["waiting_for_quota"], state["owner_name"]) == (False, waiting, "x") + + +def test_the_sweep_leaves_a_gpu_jobs_own_pod_to_its_job(fake, dispatchers): + """9b D1 (8c86e05): the pod template carries the Job's job-id and GPU labels, so the pod is listed as an object of + the job under another name; the sweep must not delete it (the Job would make a new one and the supervisor would + requeue the run as 'its GPU pod vanished').""" + job = _job() + d = dispatchers() + d.reconcile() + name = _name(job, 1) + fake.admit(name) + (pod,) = fake.job_pods(name) + listed = {(o["kind"], o["name"]): o for o in fake.objects()} + assert listed[("Pod", pod.metadata.name)]["job_id"] == job # the pod does carry the job's labels + d.reconcile() + d.reconcile() + assert ("Pod", pod.metadata.name) not in fake.deleted and fake.job_pods(name) + # the Job itself still goes when its job has ended, and its pod with it + db.execute("UPDATE meta.jobs SET status = 'cancelled', finished_at = now() WHERE id = %s", (job,)) + d.reconcile() + assert ("Job", name) in fake.deleted and not fake.job_pods(name) + assert ("Pod", pod.metadata.name) not in fake.deleted + + def test_ended_job_object_deleted(fake, dispatchers): gone, moved = _job(), _job() d = dispatchers() diff --git a/api/tests/test_gpu_supervise.py b/api/tests/test_gpu_supervise.py index 12e07424..675722ac 100644 --- a/api/tests/test_gpu_supervise.py +++ b/api/tests/test_gpu_supervise.py @@ -282,7 +282,8 @@ def test_a_cluster_error_on_create_waits_out_the_backoff(fake): fake.down(True) assert _main(job) == 3 row = _row(job) - assert row["status"] == "queued" and row["gpu_object"] is None and row["args"]["gpu_requeues"] == 1 + # requeued without burning a try: a cluster blip isn't the job's fault (supervise._submit, counts=False) + assert row["status"] == "queued" and row["gpu_object"] is None and row["args"]["gpu_requeues"] == 0 until = datetime.fromisoformat(row["args"]["gpu_blocked"]["until"]) assert abs(until - (datetime.now(timezone.utc) + timedelta(seconds=policy.BACKOFF_S))) < timedelta(seconds=10) assert "wasn't submitted" in row["args"]["gpu_blocked"]["reason"] diff --git a/api/tests/test_gpu_waiting.py b/api/tests/test_gpu_waiting.py index f896d60c..2924d062 100644 --- a/api/tests/test_gpu_waiting.py +++ b/api/tests/test_gpu_waiting.py @@ -209,7 +209,9 @@ def test_holder_ends_by_for_a_study_and_none_for_a_run(kq, app_, login_as): job = _gpu_job(_general(), me.user["id"], queue="rf-shared", obj="obj-s") kq.set_pending("rf-shared", [_item("obj-s", 0, 0)]) got = datetime.fromisoformat(_waiting(me, job)["in_use_by"]["ends_by"]) - assert got == started + timedelta(hours=get_settings().tune_max_hours_gpu) + # the study's whole budget: its hours plus the margin for its last trial (policy.study_max_exec_s) + assert got == started + timedelta(seconds=policy.study_max_exec_s()) + assert policy.study_max_exec_s() == get_settings().tune_max_hours_gpu * 3600 + 900 assert get_settings().tune_max_hours_gpu == 4 db.execute("UPDATE meta.jobs SET kind = 'run.train' WHERE id = %s", (holder,)) assert _waiting(me, job)["in_use_by"]["ends_by"] is None diff --git a/api/tests/test_mlflow.py b/api/tests/test_mlflow.py index ae8f38c7..f3e5a380 100644 --- a/api/tests/test_mlflow.py +++ b/api/tests/test_mlflow.py @@ -302,7 +302,11 @@ def _train_run(export_dir, phase="review", personal=False) -> int: recipe = db.fetch_one( "INSERT INTO meta.recipes (dataset_id, version, phase) VALUES (%s, 3, %s) RETURNING id", (ds, phase) )["id"] - user = db.fetch_one("INSERT INTO meta.users (name, email) VALUES ('Ana', 'ana@example.com') RETURNING id")["id"] + # Emails are unique since phase 11 (migration 046): every run's trainer is the one Ana. + found = db.fetch_one("SELECT id FROM meta.users WHERE lower(email) = 'ana@example.com'") + user = ( + found or db.fetch_one("INSERT INTO meta.users (name, email) VALUES ('Ana', 'ana@example.com') RETURNING id") + )["id"] assessor = user if personal else None source = f"assessment:{recipe}:{user}" if personal else f"recipe:{recipe}" xid = db.fetch_one( diff --git a/api/tests/test_oidc.py b/api/tests/test_oidc.py index 0ae2f493..12d2e172 100644 --- a/api/tests/test_oidc.py +++ b/api/tests/test_oidc.py @@ -85,6 +85,8 @@ def test_config_names_the_provider(oidc_env): "member_group": "rainforest-member", "problem": None, "accounts_from_env": False, + "password": False, + "oidc": True, } diff --git a/api/tests/test_rls_app.py b/api/tests/test_rls_app.py index d62ddd05..8d1c8247 100644 --- a/api/tests/test_rls_app.py +++ b/api/tests/test_rls_app.py @@ -333,7 +333,7 @@ def policies(conn): ).fetchall() paths = [p for p in sorted(db.MIGRATIONS_DIR.glob("*.sql")) if p.stem >= "008"] - assert paths[-1].stem.startswith("044_") + assert paths[-1].stem.startswith("046_") # a new migration: check that the replay still holds, then bump with db.owner_connection() as conn: before = policies(conn) assert conn.execute("SELECT count(*) AS n FROM meta.datasets").fetchone()["n"] >= 2 diff --git a/api/tests/test_route_matrix.py b/api/tests/test_route_matrix.py index a8b249cd..b896bab8 100644 --- a/api/tests/test_route_matrix.py +++ b/api/tests/test_route_matrix.py @@ -536,7 +536,10 @@ def _vault_rows(persona: str) -> dict[str, Row]: if not names or not names <= VAULT_IDS.keys() or row.outsider == OPEN: continue if template.startswith("/api/admin"): # not project routes: an admin is let in, anyone else is refused - out[key] = Row(F, OK, F, F, F, body=row.body, query=row.query) + # (a route's own gate stays: the reset link is a 404 in pick mode, whoever asks) + out[key] = Row( + F, row.admin_outsider if row.admin_outsider == GATE else OK, F, F, F, body=row.body, query=row.query + ) continue if template.startswith(("/api/auth", "/api/users", "/api/notices", "/predict")): continue From cb43bb5abf4c8a414b3ad4247e3561f7c7b4b20e Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 10:40:17 -0700 Subject: [PATCH 4/5] Test pass: a running GPU job still starting keeps its waiting line (9b fixes) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- ui/src/kit/kit.test.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ui/src/kit/kit.test.tsx b/ui/src/kit/kit.test.tsx index 8e2a6a11..a757a6c3 100644 --- a/ui/src/kit/kit.test.tsx +++ b/ui/src/kit/kit.test.tsx @@ -94,9 +94,14 @@ describe("JobStatus waiting", () => { expect(line.getAttribute("title")).toBe("(1 ahead in its team, Fraud)"); }); - it("shows nothing for a running job", () => { - const el = render(); - expect(el.querySelector('[data-testid="job-waiting"]')).toBeNull(); + // A claimed GPU job still starting (pulling, unschedulable) keeps its line; the API drops `waiting` once it runs. + it.each([ + ["running", w, true], + ["running", null, false], + ["done", w, false], + ] as const)("a %s job with waiting %s shows the line: %s", (status, waiting, shown) => { + const el = render(); + expect(el.querySelector('[data-testid="job-waiting"]') !== null).toBe(shown); }); }); From 751972eb2778b1605c73e0c04ab9b422a04b181f Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Mon, 16 Nov 2026 10:49:36 -0700 Subject: [PATCH 5/5] Test pass: migration replays on the shared database leave its policies as an upgrade would Run serially on one database, two replays broke later files: 031-033's replayed 041 (which drops every policy) and then only 042-044, so 045 and 046's tables lost theirs (test_passwords, test_every_meta_table_has_rls); and 010's replay put back its permissive SELECT policies, which 041 drops on an upgrade. Every migration from 041 on follows now, and 010's policies are dropped after its replay. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- api/tests/test_experiments.py | 9 +++++++++ api/tests/test_migrations_031_033.py | 12 ++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/api/tests/test_experiments.py b/api/tests/test_experiments.py index 031a3f80..74558455 100644 --- a/api/tests/test_experiments.py +++ b/api/tests/test_experiments.py @@ -275,6 +275,15 @@ def test_existing_exports_and_runs_move_into_one_imported_experiment(people): with owner_connection() as owner: # a migration replay is the owner's: the app role cannot create tables for _ in range(2): # idempotent owner.execute((MIGRATIONS_DIR / "010_experiments.sql").read_text()) + # 010's permissive SELECT policies are 041's to drop on an upgrade; left here, they would be OR-ed with the + # row-level security of every later test on this database. + for table, policy in ( + ("experiments", "experiments_public"), + ("exports", "exports_public"), + ("model_runs", "model_runs_public"), + ("analyses", "analyses_team_only"), + ): + owner.execute(f"DROP POLICY IF EXISTS {policy} ON meta.{table}") made = fetch_all("SELECT * FROM meta.experiments WHERE dataset_id = %s", (ds["id"],)) assert len(made) == 1 imp = made[0] diff --git a/api/tests/test_migrations_031_033.py b/api/tests/test_migrations_031_033.py index cf9cae0d..b4328dbe 100644 --- a/api/tests/test_migrations_031_033.py +++ b/api/tests/test_migrations_031_033.py @@ -79,10 +79,14 @@ def test_the_three_migrations_apply_again_without_changing_the_data(world): conn.execute(_migration(prefix).read_text()) # 031-033 are older than 041, which took the policies they make for its own (a replay of an old file would # leave their permissive ones beside 041's): the upgrade order is what a database sees, so 041 follows. - conn.execute(_migration("041").read_text()) - conn.execute(_migration("042").read_text()) # 041 dropped every policy; 042 makes the connections' again - conn.execute(_migration("043").read_text()) # and 043 its own tables' (field_about_history, tuning_trials, ...) - conn.execute(_migration("044").read_text()) # and 044 the GPU queue's, with the Lab stop-reason check + # 041 drops every policy; each later migration makes its own tables' again (042 the connections', 043 + # field_about_history and tuning_trials, 044 the GPU queue's with the Lab stop-reason check, 045 app_settings, + # 046 the password tables), so every one from 041 on follows, whatever the newest is. + from app.db import MIGRATIONS_DIR + + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.stem >= "041": + conn.execute(path.read_text()) # The current check (the replayed 033 narrowed it again), then the parked rows back. conn.execute( "ALTER TABLE meta.datasets DROP CONSTRAINT IF EXISTS datasets_file_format_check, "