Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ RAINFOREST_SECRET_KEY=
# MLflow tracking (docs/mlflow.md). Unset: the compose mlflow service at http://mlflow:5000/mlflow.
# Set it empty to turn logging off.
# MLFLOW_TRACKING_URI=
# false stops logging runs while the address stays set; admins also change it on Settings › Application.
# MLFLOW_LOGGING=true
# The address browsers open MLflow at, when it isn't the compose server behind /mlflow/ (an existing MLflow server).
# MLFLOW_BROWSE_URL=
# 0 removes the UI's /mlflow/ proxy (1 by default).
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,8 @@ jobs:
run: |
echo "COMPOSE_PROJECT_NAME=rf-ci-stack-${RUNNER_NAME,,}" >> "$GITHUB_ENV"
{ echo UI_PORT=127.0.0.1:0; echo API_PORT=0; echo DB_PORT=0; } >> "$GITHUB_ENV"
# The stack specs sign in by picking a user; password sign-in is the default, so turn the dev mode on.
{ echo RAINFOREST_AUTH_MODE=pick; echo RAINFOREST_DEV_PICK=true; } >> "$GITHUB_ENV"
- name: Remove a stack an earlier run left
run: docker compose down -v --remove-orphans
- name: Start the stack
Expand Down
34 changes: 27 additions & 7 deletions api/tests/fakes/kueue.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@
adds a pending Workload owned by it; a Job's pod appears when it is admitted (a Job without the label is not gated:
its pod appears at once). Creating a name that exists, deleting or not, raises `Conflict`.

What a test drives: `admit`, `evict`, `inadmissible`, `finish` (by the owner's name), `set_pending(lq, items)`,
A Job's pod is made from its template, as Kubernetes makes it: the template's labels (the GPU and job-id labels
included, so `objects()` lists the pod beside its Job) and an `ownerReferences` entry naming the Job.

What a test drives: `admit`, `evict`, `inadmissible`, `wait_for_quota`, `finish` (by the owner's name), `set_pending(lq, items)`,
`set_pod`, `terminating`, `remove` (a terminating object finally goes), `set_queue`, `installed`, `down(True)`.
What it checks: `created` (manifests), `deleted` ((kind, name)), `patched` ((kind, name, labels)), `ensured`.
"""
Expand Down Expand Up @@ -63,7 +66,7 @@ def _workload(self, owner: str) -> dict:

def admit(self, owner_name: str) -> None:
w = self._workload(owner_name)
w.update(admitted=True, evicted=None, quota_reserved=True, message=None)
w.update(admitted=True, evicted=None, quota_reserved=True, waiting_for_quota=False, message=None)
if owner_name in self.jobs and not self.job_pods(owner_name):
self._make_job_pod(owner_name)

Expand All @@ -83,7 +86,13 @@ def reconcile(self) -> None:
q.update(active=True, message=None, new=False)

def inadmissible(self, owner_name: str, message: str) -> None:
self._workload(owner_name).update(admitted=False, quota_reserved=False, message=message)
self._workload(owner_name).update(
admitted=False, quota_reserved=False, waiting_for_quota=False, message=message
)

def wait_for_quota(self, owner_name: str, message: str = "couldn't assign flavors to pod set main") -> None:
"""Live Kueue (v0.20): a Workload in line is QuotaReserved False with reason WaitingForQuota."""
self._workload(owner_name).update(admitted=False, quota_reserved=False, waiting_for_quota=True, message=message)

def finish(self, owner_name: str) -> None:
self._workload(owner_name).update(finished=True)
Expand Down Expand Up @@ -143,7 +152,14 @@ def _make_job_pod(self, job: str) -> None:
labels = {**(tpl.get("metadata", {}).get("labels") or {}), "batch.kubernetes.io/job-name": job}
self.core.create_namespaced_pod(
self.ns,
{"metadata": {"name": f"{job}-{self._suffix(job)}", "labels": labels}, "spec": copy.deepcopy(tpl["spec"])},
{
"metadata": {
"name": f"{job}-{self._suffix(job)}",
"labels": labels,
"ownerReferences": [{"apiVersion": "batch/v1", "kind": "Job", "name": job, "uid": f"uid-{job}"}],
},
"spec": copy.deepcopy(tpl["spec"]),
},
)

def _drop(self, kind: str, name: str) -> None:
Expand Down Expand Up @@ -199,7 +215,8 @@ def pending(self, local_queue: str) -> list[dict]:
out = []
for i in self.pending_items.get(local_queue, []):
w = self.wl.get(i.get("owner_name"))
if w is not None and (w["admitted"] or w["finished"] or w["quota_reserved"] is False):
inadmissible = w is not None and w["quota_reserved"] is False and not w.get("waiting_for_quota")
if w is not None and (w["admitted"] or w["finished"] or inadmissible):
continue
out.append(dict(i))
return out
Expand Down Expand Up @@ -233,6 +250,7 @@ def create(self, manifest: dict) -> None:
"evicted": None,
"finished": False,
"quota_reserved": None,
"waiting_for_quota": False,
"message": None,
}

Expand All @@ -258,7 +276,7 @@ def objects(self) -> list[dict]:
continue
pods = self.job_pods(name)
phase = pods[0].status.phase if pods else "Pending"
out.append(self._entry("Job", name, job["metadata"], phase, None, None, None))
out.append(self._entry("Job", name, job["metadata"], phase, None, None, None, None))
return out + self.pod_objects()

def pod_objects(self) -> list[dict]:
Expand All @@ -281,18 +299,20 @@ def pod_objects(self) -> list[dict]:
waiting,
self.unsched.get(name),
started,
next((r.kind for r in pod.metadata.owner_references or []), None),
)
)
return out

def _entry(self, kind, name, meta, phase, waiting, unschedulable, started) -> dict:
def _entry(self, kind, name, meta, phase, waiting, unschedulable, started, owner_kind) -> dict:
try:
job_id = int((meta.get("labels") or {}).get(kube.JOB_ID_LABEL, ""))
except ValueError:
job_id = None
return {
"kind": kind,
"name": name,
"owner_kind": owner_kind,
"job_id": job_id,
"queue": (meta.get("labels") or {}).get(kube.QUEUE_LABEL),
"deleting": (kind, name) in self.gone,
Expand Down
3 changes: 3 additions & 0 deletions api/tests/test_access_lint.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,9 @@ def test_secrets_table_only_in_store():
"sources/postgres.py", "sources/mongodb.py", "sources/s3.py", "sample_mongo.py", "sample_s3.py",
"connections/netguard.py",
"db.py", "worker.py", "jobevents.py", # Rainforest's own database: psycopg.connect
# The in-app settings (meta.app_settings, phase 10): read from the environment's database_url before the merged
# settings exist; db.py's pool is built from get_settings(), which calls this, so going through db.py would recurse.
"config.py",
} # fmt: skip


Expand Down
120 changes: 120 additions & 0 deletions api/tests/test_app_settings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
"""Settings › Application (phase 10): an admin's change overrides the env value in every process (here, at once in the
saving one), a reset drops the row so the env default applies again, a Lab choice above the deployment's limit is
refused with nothing saved, and every change is audited."""

import pytest
from fastapi.testclient import TestClient

from .team_helpers import app_ # noqa: F401 (fixture)


@pytest.fixture
def admin(app_, login_as): # noqa: F811
from app import config, db

def clean():
db.execute("DELETE FROM meta.app_settings")
config.invalidate_overrides()

clean()
yield login_as(TestClient(app_), "admin")
clean()


def _setting(body: dict, key: str) -> dict:
return next(s for s in body["settings"] if s["key"] == key)


def _audits(key: str) -> list[dict]:
from app import db

return db.fetch_all(
"SELECT actor, detail FROM meta.audit WHERE action = 'settings.change' AND detail ->> 'key' = %s ORDER BY id",
(key,),
)


def test_a_change_overrides_the_env_value_and_a_reset_brings_it_back(admin):
from app.config import env_settings, get_settings

default = env_settings().job_log_keep_days
before = _setting(admin.get("/api/admin/settings").json(), "job_log_keep_days")
assert (before["value"], before["default"], before["overridden"]) == (default, default, False)

r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": default + 7}})
assert r.status_code == 200, r.text
row = _setting(r.json(), "job_log_keep_days")
assert (row["value"], row["default"], row["overridden"], row["updated_by"]) == (default + 7, default, True, "Admin")
assert get_settings().job_log_keep_days == default + 7 # the saving process applies it without waiting
assert env_settings().job_log_keep_days == default # the deployment's own value is untouched

r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}})
row = _setting(r.json(), "job_log_keep_days")
assert (row["value"], row["overridden"], row["updated_by"]) == (default, False, None)
assert get_settings().job_log_keep_days == default
from app import db

assert db.fetch_one("SELECT 1 FROM meta.app_settings WHERE key = 'job_log_keep_days'") is None


def test_each_change_is_audited_with_its_old_and_new_value(admin):
from app.config import env_settings

default = env_settings().job_log_keep_days
n = len(_audits("job_log_keep_days"))
admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}})
admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}}) # no change, no row
admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 120}})
admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}})
rows = _audits("job_log_keep_days")[n:]
assert [(r["detail"]["from"], r["detail"]["to"]) for r in rows] == [(None, 90), (90, 120), (120, None)]
assert all(r["actor"] == admin.user["id"] for r in rows)
assert all(r["detail"]["env"] == "JOB_LOG_KEEP_DAYS" and r["detail"]["default"] == default for r in rows)


def test_a_lab_choice_above_the_deployments_limit_is_refused_and_nothing_is_saved(admin):
from app.config import env_settings, get_settings

env = env_settings()
cap = int(env.notebook_cpu_limit)
view = _setting(admin.get("/api/admin/settings").json(), "notebook_cpu_choices")
assert view["max"] <= cap and view["ceiling"] == f"NOTEBOOK_CPU_LIMIT = {env.notebook_cpu_limit}"
n = len(_audits("job_log_keep_days"))

r = admin.patch(
"/api/admin/settings", json={"values": {"job_log_keep_days": 45, "notebook_cpu_choices": [1, cap + 1]}}
)
assert r.status_code == 422
assert "above the deployment's NOTEBOOK_CPU_LIMIT" in r.json()["detail"]
# all or nothing: the valid change beside it is not saved either
assert get_settings().job_log_keep_days == env.job_log_keep_days
assert get_settings().notebook_cpu_choices == env.notebook_cpu_choices
assert len(_audits("job_log_keep_days")) == n

r = admin.patch("/api/admin/settings", json={"values": {"notebook_memory_choices_gb": [2, 4096]}})
assert r.status_code == 422 and "NOTEBOOK_MEMORY_LIMIT" in r.json()["detail"]

r = admin.patch("/api/admin/settings", json={"values": {"notebook_cpu_choices": [cap, 1, 1]}})
assert r.status_code == 200, r.text
assert get_settings().notebook_cpu_choices == sorted({1, cap}) # sorted, duplicates dropped


@pytest.mark.parametrize(
"values, says",
[
({"database_url": "postgresql://x/y"}, "can't be changed in the app"),
({"job_log_keep_days": 0}, "outside"),
({"mlflow_logging": "yes"}, "not a valid value"),
({"log_level": "loud"}, "one of"),
({"notebook_cpu_choices": []}, "at least one"),
],
)
def test_bad_values_are_refused_with_a_reason(admin, values, says):
r = admin.patch("/api/admin/settings", json={"values": values})
assert r.status_code == 422 and says in r.json()["detail"], r.text


def test_members_can_neither_read_nor_change_them(app_, admin, login_as): # noqa: F811
member = login_as(TestClient(app_), "owner")
assert member.get("/api/admin/settings").status_code == 403
assert member.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 9}}).status_code == 403
11 changes: 10 additions & 1 deletion api/tests/test_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def touch(self, user_id):
def env(tmp_path, monkeypatch):
monkeypatch.setenv("DATA_DIR", str(tmp_path))
monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick")
monkeypatch.setenv("RAINFOREST_DEV_PICK", "true") # phase 11: pick mode needs the development flag
for name in (
"RAINFOREST_SESSION_SECRET",
"RAINFOREST_ALLOWED_HOSTS",
Expand All @@ -70,6 +71,10 @@ def store(monkeypatch):
# No Postgres here: a user's scope is empty (these tests are about sessions, not about what a session may read).
monkeypatch.setattr(access, "scope", lambda user, fresh=False: access.Scope(user["id"], user["role"] == "admin", 0))
monkeypatch.setattr(auth_module, "_my_projects", lambda user_id: [])
monkeypatch.setattr(auth_module.passwords, "credentials", lambda user_id: None) # pick-mode users have none
# Phase 11 audits every sign-in (meta.audit); here the rows are kept in memory.
fake.sign_ins = []
monkeypatch.setattr(auth_module, "_audit_sign_in", lambda user_id, via, ip: fake.sign_ins.append((user_id, via)))
return fake


Expand Down Expand Up @@ -102,7 +107,7 @@ def test_protected_routes_need_a_session(client):
assert r.status_code == 401


def test_pick_me_logout(client):
def test_pick_me_logout(client, store):
cfg = client.get("/api/auth/config").json()
assert cfg["mode"] == "pick" and cfg["problem"] is None
assert client.get("/api/auth/users").json() == [
Expand All @@ -112,6 +117,7 @@ def test_pick_me_logout(client):
assert _pick(client, 99).status_code == 404
r = _pick(client, 2)
assert r.status_code == 200 and r.json()["name"] == "Dana"
assert store.sign_ins == [(2, "pick")] # audited; the refused id 99 is not
cookie = r.headers["set-cookie"]
assert cookie.startswith(f"{auth.COOKIE}=") and "HttpOnly" in cookie and "SameSite=strict" in cookie
assert "Max-Age=604800" in cookie and "Path=/" in cookie and "Secure" not in cookie
Expand All @@ -124,6 +130,9 @@ def test_pick_me_logout(client):
"avatar_url": None,
"mode": "pick",
"can_rename": True,
"has_password": False,
"must_change_password": False,
"can_change_password": False,
"personal": None,
"projects": [],
"can_create_projects": True,
Expand Down
22 changes: 22 additions & 0 deletions api/tests/test_connection_secrets.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,28 @@ def test_redactor():
assert "after the block: s3cr3t-value" in stream.getvalue()


def test_the_log_filter_leaves_records_it_does_not_change_alone():
"""9b D1 (57a5a54): uvicorn's access formatter unpacks a record's five args, so a record with nothing to hide keeps
them (emptying them logged a 'Logging error' traceback for every request)."""
from uvicorn.logging import AccessFormatter

from app.connections import redact

stream = io.StringIO()
handler = logging.StreamHandler(stream)
handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False))
handler.addFilter(redact.LogFilter())
log = logging.getLogger(f"redact-access-{uuid.uuid4().hex}")
log.addHandler(handler)
log.propagate = False
args = ("10.0.0.1:5000", "GET", "/api/health", "1.1", 200)
with redact.secrets(["s3cr3t-value"]):
record = log.makeRecord(log.name, logging.INFO, __file__, 1, '%s - "%s %s HTTP/%s" %d', args, None)
log.handle(record)
assert record.args == args
assert '10.0.0.1:5000 - "GET /api/health HTTP/1.1" 200' in stream.getvalue()


def test_validation_errors_never_echo_input():
from app import main

Expand Down
9 changes: 9 additions & 0 deletions api/tests/test_experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,15 @@ def test_existing_exports_and_runs_move_into_one_imported_experiment(people):
with owner_connection() as owner: # a migration replay is the owner's: the app role cannot create tables
for _ in range(2): # idempotent
owner.execute((MIGRATIONS_DIR / "010_experiments.sql").read_text())
# 010's permissive SELECT policies are 041's to drop on an upgrade; left here, they would be OR-ed with the
# row-level security of every later test on this database.
for table, policy in (
("experiments", "experiments_public"),
("exports", "exports_public"),
("model_runs", "model_runs_public"),
("analyses", "analyses_team_only"),
):
owner.execute(f"DROP POLICY IF EXISTS {policy} ON meta.{table}")
made = fetch_all("SELECT * FROM meta.experiments WHERE dataset_id = %s", (ds["id"],))
assert len(made) == 1
imp = made[0]
Expand Down
50 changes: 50 additions & 0 deletions api/tests/test_gpu_dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,56 @@ def test_inadmissible_records_kueue_message(fake, dispatchers):
assert _row(job)["args"]["gpu_admission"]["state"] == "admitted"


def test_a_workload_waiting_for_quota_is_in_line_not_inadmissible(fake, dispatchers):
"""9b D1 (2ee089d): live Kueue v0.20.4 marks every queued Workload QuotaReserved False with reason WaitingForQuota;
that is a place in line, not "Can't queue for the GPU"."""
job = _job()
d = dispatchers()
d.reconcile()
name = _name(job, 1)
fake.wait_for_quota(name)
d.reconcile()
assert _row(job)["args"]["gpu_admission"]["state"] == "pending"
fake.inadmissible(name, "couldn't assign flavors: no flavor matches")
d.reconcile()
assert _row(job)["args"]["gpu_admission"]["state"] == "inadmissible"


@pytest.mark.parametrize(
"reason, waiting",
[("WaitingForQuota", True), ("Pending", True), ("Inadmissible", False), ("NoMatchingFlavor", False)],
)
def test_workload_state_reads_waiting_for_quota_from_the_reason(reason, waiting):
wl = {
"metadata": {"name": "job-x-0a1b2", "ownerReferences": [{"kind": "Job", "name": "x"}]},
"status": {"conditions": [{"type": "QuotaReserved", "status": "False", "reason": reason, "message": "m"}]},
}
state = kube.workload_state(wl)
assert (state["quota_reserved"], state["waiting_for_quota"], state["owner_name"]) == (False, waiting, "x")


def test_the_sweep_leaves_a_gpu_jobs_own_pod_to_its_job(fake, dispatchers):
"""9b D1 (8c86e05): the pod template carries the Job's job-id and GPU labels, so the pod is listed as an object of
the job under another name; the sweep must not delete it (the Job would make a new one and the supervisor would
requeue the run as 'its GPU pod vanished')."""
job = _job()
d = dispatchers()
d.reconcile()
name = _name(job, 1)
fake.admit(name)
(pod,) = fake.job_pods(name)
listed = {(o["kind"], o["name"]): o for o in fake.objects()}
assert listed[("Pod", pod.metadata.name)]["job_id"] == job # the pod does carry the job's labels
d.reconcile()
d.reconcile()
assert ("Pod", pod.metadata.name) not in fake.deleted and fake.job_pods(name)
# the Job itself still goes when its job has ended, and its pod with it
db.execute("UPDATE meta.jobs SET status = 'cancelled', finished_at = now() WHERE id = %s", (job,))
d.reconcile()
assert ("Job", name) in fake.deleted and not fake.job_pods(name)
assert ("Pod", pod.metadata.name) not in fake.deleted


def test_ended_job_object_deleted(fake, dispatchers):
gone, moved = _job(), _job()
d = dispatchers()
Expand Down
Loading
Loading