diff --git a/.env.example b/.env.example index 27c4428..b9d0e86 100644 --- a/.env.example +++ b/.env.example @@ -127,6 +127,8 @@ RAINFOREST_SECRET_KEY= # MLflow tracking (docs/mlflow.md). Unset: the compose mlflow service at http://mlflow:5000/mlflow. # Set it empty to turn logging off. # MLFLOW_TRACKING_URI= +# false stops logging runs while the address stays set; admins also change it on Settings › Application. +# MLFLOW_LOGGING=true # The address browsers open MLflow at, when it isn't the compose server behind /mlflow/ (an existing MLflow server). # MLFLOW_BROWSE_URL= # 0 removes the UI's /mlflow/ proxy (1 by default). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b27218c..9ec2eb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,8 @@ jobs: run: | echo "COMPOSE_PROJECT_NAME=rf-ci-stack-${RUNNER_NAME,,}" >> "$GITHUB_ENV" { echo UI_PORT=127.0.0.1:0; echo API_PORT=0; echo DB_PORT=0; } >> "$GITHUB_ENV" + # The stack specs sign in by picking a user; password sign-in is the default, so turn the dev mode on. + { echo RAINFOREST_AUTH_MODE=pick; echo RAINFOREST_DEV_PICK=true; } >> "$GITHUB_ENV" - name: Remove a stack an earlier run left run: docker compose down -v --remove-orphans - name: Start the stack diff --git a/api/tests/fakes/kueue.py b/api/tests/fakes/kueue.py index 763e63f..b545519 100644 --- a/api/tests/fakes/kueue.py +++ b/api/tests/fakes/kueue.py @@ -5,7 +5,10 @@ adds a pending Workload owned by it; a Job's pod appears when it is admitted (a Job without the label is not gated: its pod appears at once). Creating a name that exists, deleting or not, raises `Conflict`. -What a test drives: `admit`, `evict`, `inadmissible`, `finish` (by the owner's name), `set_pending(lq, items)`, +A Job's pod is made from its template, as Kubernetes makes it: the template's labels (the GPU and job-id labels +included, so `objects()` lists the pod beside its Job) and an `ownerReferences` entry naming the Job. + +What a test drives: `admit`, `evict`, `inadmissible`, `wait_for_quota`, `finish` (by the owner's name), `set_pending(lq, items)`, `set_pod`, `terminating`, `remove` (a terminating object finally goes), `set_queue`, `installed`, `down(True)`. What it checks: `created` (manifests), `deleted` ((kind, name)), `patched` ((kind, name, labels)), `ensured`. """ @@ -63,7 +66,7 @@ def _workload(self, owner: str) -> dict: def admit(self, owner_name: str) -> None: w = self._workload(owner_name) - w.update(admitted=True, evicted=None, quota_reserved=True, message=None) + w.update(admitted=True, evicted=None, quota_reserved=True, waiting_for_quota=False, message=None) if owner_name in self.jobs and not self.job_pods(owner_name): self._make_job_pod(owner_name) @@ -83,7 +86,13 @@ def reconcile(self) -> None: q.update(active=True, message=None, new=False) def inadmissible(self, owner_name: str, message: str) -> None: - self._workload(owner_name).update(admitted=False, quota_reserved=False, message=message) + self._workload(owner_name).update( + admitted=False, quota_reserved=False, waiting_for_quota=False, message=message + ) + + def wait_for_quota(self, owner_name: str, message: str = "couldn't assign flavors to pod set main") -> None: + """Live Kueue (v0.20): a Workload in line is QuotaReserved False with reason WaitingForQuota.""" + self._workload(owner_name).update(admitted=False, quota_reserved=False, waiting_for_quota=True, message=message) def finish(self, owner_name: str) -> None: self._workload(owner_name).update(finished=True) @@ -143,7 +152,14 @@ def _make_job_pod(self, job: str) -> None: labels = {**(tpl.get("metadata", {}).get("labels") or {}), "batch.kubernetes.io/job-name": job} self.core.create_namespaced_pod( self.ns, - {"metadata": {"name": f"{job}-{self._suffix(job)}", "labels": labels}, "spec": copy.deepcopy(tpl["spec"])}, + { + "metadata": { + "name": f"{job}-{self._suffix(job)}", + "labels": labels, + "ownerReferences": [{"apiVersion": "batch/v1", "kind": "Job", "name": job, "uid": f"uid-{job}"}], + }, + "spec": copy.deepcopy(tpl["spec"]), + }, ) def _drop(self, kind: str, name: str) -> None: @@ -199,7 +215,8 @@ def pending(self, local_queue: str) -> list[dict]: out = [] for i in self.pending_items.get(local_queue, []): w = self.wl.get(i.get("owner_name")) - if w is not None and (w["admitted"] or w["finished"] or w["quota_reserved"] is False): + inadmissible = w is not None and w["quota_reserved"] is False and not w.get("waiting_for_quota") + if w is not None and (w["admitted"] or w["finished"] or inadmissible): continue out.append(dict(i)) return out @@ -233,6 +250,7 @@ def create(self, manifest: dict) -> None: "evicted": None, "finished": False, "quota_reserved": None, + "waiting_for_quota": False, "message": None, } @@ -258,7 +276,7 @@ def objects(self) -> list[dict]: continue pods = self.job_pods(name) phase = pods[0].status.phase if pods else "Pending" - out.append(self._entry("Job", name, job["metadata"], phase, None, None, None)) + out.append(self._entry("Job", name, job["metadata"], phase, None, None, None, None)) return out + self.pod_objects() def pod_objects(self) -> list[dict]: @@ -281,11 +299,12 @@ def pod_objects(self) -> list[dict]: waiting, self.unsched.get(name), started, + next((r.kind for r in pod.metadata.owner_references or []), None), ) ) return out - def _entry(self, kind, name, meta, phase, waiting, unschedulable, started) -> dict: + def _entry(self, kind, name, meta, phase, waiting, unschedulable, started, owner_kind) -> dict: try: job_id = int((meta.get("labels") or {}).get(kube.JOB_ID_LABEL, "")) except ValueError: @@ -293,6 +312,7 @@ def _entry(self, kind, name, meta, phase, waiting, unschedulable, started) -> di return { "kind": kind, "name": name, + "owner_kind": owner_kind, "job_id": job_id, "queue": (meta.get("labels") or {}).get(kube.QUEUE_LABEL), "deleting": (kind, name) in self.gone, diff --git a/api/tests/test_access_lint.py b/api/tests/test_access_lint.py index 7b033ee..2a702ce 100644 --- a/api/tests/test_access_lint.py +++ b/api/tests/test_access_lint.py @@ -331,6 +331,9 @@ def test_secrets_table_only_in_store(): "sources/postgres.py", "sources/mongodb.py", "sources/s3.py", "sample_mongo.py", "sample_s3.py", "connections/netguard.py", "db.py", "worker.py", "jobevents.py", # Rainforest's own database: psycopg.connect + # The in-app settings (meta.app_settings, phase 10): read from the environment's database_url before the merged + # settings exist; db.py's pool is built from get_settings(), which calls this, so going through db.py would recurse. + "config.py", } # fmt: skip diff --git a/api/tests/test_app_settings.py b/api/tests/test_app_settings.py new file mode 100644 index 0000000..c5224ba --- /dev/null +++ b/api/tests/test_app_settings.py @@ -0,0 +1,120 @@ +"""Settings › Application (phase 10): an admin's change overrides the env value in every process (here, at once in the +saving one), a reset drops the row so the env default applies again, a Lab choice above the deployment's limit is +refused with nothing saved, and every change is audited.""" + +import pytest +from fastapi.testclient import TestClient + +from .team_helpers import app_ # noqa: F401 (fixture) + + +@pytest.fixture +def admin(app_, login_as): # noqa: F811 + from app import config, db + + def clean(): + db.execute("DELETE FROM meta.app_settings") + config.invalidate_overrides() + + clean() + yield login_as(TestClient(app_), "admin") + clean() + + +def _setting(body: dict, key: str) -> dict: + return next(s for s in body["settings"] if s["key"] == key) + + +def _audits(key: str) -> list[dict]: + from app import db + + return db.fetch_all( + "SELECT actor, detail FROM meta.audit WHERE action = 'settings.change' AND detail ->> 'key' = %s ORDER BY id", + (key,), + ) + + +def test_a_change_overrides_the_env_value_and_a_reset_brings_it_back(admin): + from app.config import env_settings, get_settings + + default = env_settings().job_log_keep_days + before = _setting(admin.get("/api/admin/settings").json(), "job_log_keep_days") + assert (before["value"], before["default"], before["overridden"]) == (default, default, False) + + r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": default + 7}}) + assert r.status_code == 200, r.text + row = _setting(r.json(), "job_log_keep_days") + assert (row["value"], row["default"], row["overridden"], row["updated_by"]) == (default + 7, default, True, "Admin") + assert get_settings().job_log_keep_days == default + 7 # the saving process applies it without waiting + assert env_settings().job_log_keep_days == default # the deployment's own value is untouched + + r = admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}}) + row = _setting(r.json(), "job_log_keep_days") + assert (row["value"], row["overridden"], row["updated_by"]) == (default, False, None) + assert get_settings().job_log_keep_days == default + from app import db + + assert db.fetch_one("SELECT 1 FROM meta.app_settings WHERE key = 'job_log_keep_days'") is None + + +def test_each_change_is_audited_with_its_old_and_new_value(admin): + from app.config import env_settings + + default = env_settings().job_log_keep_days + n = len(_audits("job_log_keep_days")) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}}) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 90}}) # no change, no row + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 120}}) + admin.patch("/api/admin/settings", json={"values": {"job_log_keep_days": None}}) + rows = _audits("job_log_keep_days")[n:] + assert [(r["detail"]["from"], r["detail"]["to"]) for r in rows] == [(None, 90), (90, 120), (120, None)] + assert all(r["actor"] == admin.user["id"] for r in rows) + assert all(r["detail"]["env"] == "JOB_LOG_KEEP_DAYS" and r["detail"]["default"] == default for r in rows) + + +def test_a_lab_choice_above_the_deployments_limit_is_refused_and_nothing_is_saved(admin): + from app.config import env_settings, get_settings + + env = env_settings() + cap = int(env.notebook_cpu_limit) + view = _setting(admin.get("/api/admin/settings").json(), "notebook_cpu_choices") + assert view["max"] <= cap and view["ceiling"] == f"NOTEBOOK_CPU_LIMIT = {env.notebook_cpu_limit}" + n = len(_audits("job_log_keep_days")) + + r = admin.patch( + "/api/admin/settings", json={"values": {"job_log_keep_days": 45, "notebook_cpu_choices": [1, cap + 1]}} + ) + assert r.status_code == 422 + assert "above the deployment's NOTEBOOK_CPU_LIMIT" in r.json()["detail"] + # all or nothing: the valid change beside it is not saved either + assert get_settings().job_log_keep_days == env.job_log_keep_days + assert get_settings().notebook_cpu_choices == env.notebook_cpu_choices + assert len(_audits("job_log_keep_days")) == n + + r = admin.patch("/api/admin/settings", json={"values": {"notebook_memory_choices_gb": [2, 4096]}}) + assert r.status_code == 422 and "NOTEBOOK_MEMORY_LIMIT" in r.json()["detail"] + + r = admin.patch("/api/admin/settings", json={"values": {"notebook_cpu_choices": [cap, 1, 1]}}) + assert r.status_code == 200, r.text + assert get_settings().notebook_cpu_choices == sorted({1, cap}) # sorted, duplicates dropped + + +@pytest.mark.parametrize( + "values, says", + [ + ({"database_url": "postgresql://x/y"}, "can't be changed in the app"), + ({"job_log_keep_days": 0}, "outside"), + ({"mlflow_logging": "yes"}, "not a valid value"), + ({"log_level": "loud"}, "one of"), + ({"notebook_cpu_choices": []}, "at least one"), + ], +) +def test_bad_values_are_refused_with_a_reason(admin, values, says): + r = admin.patch("/api/admin/settings", json={"values": values}) + assert r.status_code == 422 and says in r.json()["detail"], r.text + + +def test_members_can_neither_read_nor_change_them(app_, admin, login_as): # noqa: F811 + member = login_as(TestClient(app_), "owner") + assert member.get("/api/admin/settings").status_code == 403 + assert member.patch("/api/admin/settings", json={"values": {"job_log_keep_days": 9}}).status_code == 403 diff --git a/api/tests/test_auth.py b/api/tests/test_auth.py index 5a5eb5f..2b4260a 100644 --- a/api/tests/test_auth.py +++ b/api/tests/test_auth.py @@ -49,6 +49,7 @@ def touch(self, user_id): def env(tmp_path, monkeypatch): monkeypatch.setenv("DATA_DIR", str(tmp_path)) monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + monkeypatch.setenv("RAINFOREST_DEV_PICK", "true") # phase 11: pick mode needs the development flag for name in ( "RAINFOREST_SESSION_SECRET", "RAINFOREST_ALLOWED_HOSTS", @@ -70,6 +71,10 @@ def store(monkeypatch): # No Postgres here: a user's scope is empty (these tests are about sessions, not about what a session may read). monkeypatch.setattr(access, "scope", lambda user, fresh=False: access.Scope(user["id"], user["role"] == "admin", 0)) monkeypatch.setattr(auth_module, "_my_projects", lambda user_id: []) + monkeypatch.setattr(auth_module.passwords, "credentials", lambda user_id: None) # pick-mode users have none + # Phase 11 audits every sign-in (meta.audit); here the rows are kept in memory. + fake.sign_ins = [] + monkeypatch.setattr(auth_module, "_audit_sign_in", lambda user_id, via, ip: fake.sign_ins.append((user_id, via))) return fake @@ -102,7 +107,7 @@ def test_protected_routes_need_a_session(client): assert r.status_code == 401 -def test_pick_me_logout(client): +def test_pick_me_logout(client, store): cfg = client.get("/api/auth/config").json() assert cfg["mode"] == "pick" and cfg["problem"] is None assert client.get("/api/auth/users").json() == [ @@ -112,6 +117,7 @@ def test_pick_me_logout(client): assert _pick(client, 99).status_code == 404 r = _pick(client, 2) assert r.status_code == 200 and r.json()["name"] == "Dana" + assert store.sign_ins == [(2, "pick")] # audited; the refused id 99 is not cookie = r.headers["set-cookie"] assert cookie.startswith(f"{auth.COOKIE}=") and "HttpOnly" in cookie and "SameSite=strict" in cookie assert "Max-Age=604800" in cookie and "Path=/" in cookie and "Secure" not in cookie @@ -124,6 +130,9 @@ def test_pick_me_logout(client): "avatar_url": None, "mode": "pick", "can_rename": True, + "has_password": False, + "must_change_password": False, + "can_change_password": False, "personal": None, "projects": [], "can_create_projects": True, diff --git a/api/tests/test_connection_secrets.py b/api/tests/test_connection_secrets.py index 1e97e6e..94136bf 100644 --- a/api/tests/test_connection_secrets.py +++ b/api/tests/test_connection_secrets.py @@ -216,6 +216,28 @@ def test_redactor(): assert "after the block: s3cr3t-value" in stream.getvalue() +def test_the_log_filter_leaves_records_it_does_not_change_alone(): + """9b D1 (57a5a54): uvicorn's access formatter unpacks a record's five args, so a record with nothing to hide keeps + them (emptying them logged a 'Logging error' traceback for every request).""" + from uvicorn.logging import AccessFormatter + + from app.connections import redact + + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(AccessFormatter('%(client_addr)s - "%(request_line)s" %(status_code)s', use_colors=False)) + handler.addFilter(redact.LogFilter()) + log = logging.getLogger(f"redact-access-{uuid.uuid4().hex}") + log.addHandler(handler) + log.propagate = False + args = ("10.0.0.1:5000", "GET", "/api/health", "1.1", 200) + with redact.secrets(["s3cr3t-value"]): + record = log.makeRecord(log.name, logging.INFO, __file__, 1, '%s - "%s %s HTTP/%s" %d', args, None) + log.handle(record) + assert record.args == args + assert '10.0.0.1:5000 - "GET /api/health HTTP/1.1" 200' in stream.getvalue() + + def test_validation_errors_never_echo_input(): from app import main diff --git a/api/tests/test_experiments.py b/api/tests/test_experiments.py index 031a3f8..7455845 100644 --- a/api/tests/test_experiments.py +++ b/api/tests/test_experiments.py @@ -275,6 +275,15 @@ def test_existing_exports_and_runs_move_into_one_imported_experiment(people): with owner_connection() as owner: # a migration replay is the owner's: the app role cannot create tables for _ in range(2): # idempotent owner.execute((MIGRATIONS_DIR / "010_experiments.sql").read_text()) + # 010's permissive SELECT policies are 041's to drop on an upgrade; left here, they would be OR-ed with the + # row-level security of every later test on this database. + for table, policy in ( + ("experiments", "experiments_public"), + ("exports", "exports_public"), + ("model_runs", "model_runs_public"), + ("analyses", "analyses_team_only"), + ): + owner.execute(f"DROP POLICY IF EXISTS {policy} ON meta.{table}") made = fetch_all("SELECT * FROM meta.experiments WHERE dataset_id = %s", (ds["id"],)) assert len(made) == 1 imp = made[0] diff --git a/api/tests/test_gpu_dispatch.py b/api/tests/test_gpu_dispatch.py index 5abed32..c81b361 100644 --- a/api/tests/test_gpu_dispatch.py +++ b/api/tests/test_gpu_dispatch.py @@ -147,6 +147,56 @@ def test_inadmissible_records_kueue_message(fake, dispatchers): assert _row(job)["args"]["gpu_admission"]["state"] == "admitted" +def test_a_workload_waiting_for_quota_is_in_line_not_inadmissible(fake, dispatchers): + """9b D1 (2ee089d): live Kueue v0.20.4 marks every queued Workload QuotaReserved False with reason WaitingForQuota; + that is a place in line, not "Can't queue for the GPU".""" + job = _job() + d = dispatchers() + d.reconcile() + name = _name(job, 1) + fake.wait_for_quota(name) + d.reconcile() + assert _row(job)["args"]["gpu_admission"]["state"] == "pending" + fake.inadmissible(name, "couldn't assign flavors: no flavor matches") + d.reconcile() + assert _row(job)["args"]["gpu_admission"]["state"] == "inadmissible" + + +@pytest.mark.parametrize( + "reason, waiting", + [("WaitingForQuota", True), ("Pending", True), ("Inadmissible", False), ("NoMatchingFlavor", False)], +) +def test_workload_state_reads_waiting_for_quota_from_the_reason(reason, waiting): + wl = { + "metadata": {"name": "job-x-0a1b2", "ownerReferences": [{"kind": "Job", "name": "x"}]}, + "status": {"conditions": [{"type": "QuotaReserved", "status": "False", "reason": reason, "message": "m"}]}, + } + state = kube.workload_state(wl) + assert (state["quota_reserved"], state["waiting_for_quota"], state["owner_name"]) == (False, waiting, "x") + + +def test_the_sweep_leaves_a_gpu_jobs_own_pod_to_its_job(fake, dispatchers): + """9b D1 (8c86e05): the pod template carries the Job's job-id and GPU labels, so the pod is listed as an object of + the job under another name; the sweep must not delete it (the Job would make a new one and the supervisor would + requeue the run as 'its GPU pod vanished').""" + job = _job() + d = dispatchers() + d.reconcile() + name = _name(job, 1) + fake.admit(name) + (pod,) = fake.job_pods(name) + listed = {(o["kind"], o["name"]): o for o in fake.objects()} + assert listed[("Pod", pod.metadata.name)]["job_id"] == job # the pod does carry the job's labels + d.reconcile() + d.reconcile() + assert ("Pod", pod.metadata.name) not in fake.deleted and fake.job_pods(name) + # the Job itself still goes when its job has ended, and its pod with it + db.execute("UPDATE meta.jobs SET status = 'cancelled', finished_at = now() WHERE id = %s", (job,)) + d.reconcile() + assert ("Job", name) in fake.deleted and not fake.job_pods(name) + assert ("Pod", pod.metadata.name) not in fake.deleted + + def test_ended_job_object_deleted(fake, dispatchers): gone, moved = _job(), _job() d = dispatchers() diff --git a/api/tests/test_gpu_supervise.py b/api/tests/test_gpu_supervise.py index 12e0742..675722a 100644 --- a/api/tests/test_gpu_supervise.py +++ b/api/tests/test_gpu_supervise.py @@ -282,7 +282,8 @@ def test_a_cluster_error_on_create_waits_out_the_backoff(fake): fake.down(True) assert _main(job) == 3 row = _row(job) - assert row["status"] == "queued" and row["gpu_object"] is None and row["args"]["gpu_requeues"] == 1 + # requeued without burning a try: a cluster blip isn't the job's fault (supervise._submit, counts=False) + assert row["status"] == "queued" and row["gpu_object"] is None and row["args"]["gpu_requeues"] == 0 until = datetime.fromisoformat(row["args"]["gpu_blocked"]["until"]) assert abs(until - (datetime.now(timezone.utc) + timedelta(seconds=policy.BACKOFF_S))) < timedelta(seconds=10) assert "wasn't submitted" in row["args"]["gpu_blocked"]["reason"] diff --git a/api/tests/test_gpu_waiting.py b/api/tests/test_gpu_waiting.py index f896d60..2924d06 100644 --- a/api/tests/test_gpu_waiting.py +++ b/api/tests/test_gpu_waiting.py @@ -209,7 +209,9 @@ def test_holder_ends_by_for_a_study_and_none_for_a_run(kq, app_, login_as): job = _gpu_job(_general(), me.user["id"], queue="rf-shared", obj="obj-s") kq.set_pending("rf-shared", [_item("obj-s", 0, 0)]) got = datetime.fromisoformat(_waiting(me, job)["in_use_by"]["ends_by"]) - assert got == started + timedelta(hours=get_settings().tune_max_hours_gpu) + # the study's whole budget: its hours plus the margin for its last trial (policy.study_max_exec_s) + assert got == started + timedelta(seconds=policy.study_max_exec_s()) + assert policy.study_max_exec_s() == get_settings().tune_max_hours_gpu * 3600 + 900 assert get_settings().tune_max_hours_gpu == 4 db.execute("UPDATE meta.jobs SET kind = 'run.train' WHERE id = %s", (holder,)) assert _waiting(me, job)["in_use_by"]["ends_by"] is None diff --git a/api/tests/test_migrations_031_033.py b/api/tests/test_migrations_031_033.py index cf9cae0..b4328db 100644 --- a/api/tests/test_migrations_031_033.py +++ b/api/tests/test_migrations_031_033.py @@ -79,10 +79,14 @@ def test_the_three_migrations_apply_again_without_changing_the_data(world): conn.execute(_migration(prefix).read_text()) # 031-033 are older than 041, which took the policies they make for its own (a replay of an old file would # leave their permissive ones beside 041's): the upgrade order is what a database sees, so 041 follows. - conn.execute(_migration("041").read_text()) - conn.execute(_migration("042").read_text()) # 041 dropped every policy; 042 makes the connections' again - conn.execute(_migration("043").read_text()) # and 043 its own tables' (field_about_history, tuning_trials, ...) - conn.execute(_migration("044").read_text()) # and 044 the GPU queue's, with the Lab stop-reason check + # 041 drops every policy; each later migration makes its own tables' again (042 the connections', 043 + # field_about_history and tuning_trials, 044 the GPU queue's with the Lab stop-reason check, 045 app_settings, + # 046 the password tables), so every one from 041 on follows, whatever the newest is. + from app.db import MIGRATIONS_DIR + + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.stem >= "041": + conn.execute(path.read_text()) # The current check (the replayed 033 narrowed it again), then the parked rows back. conn.execute( "ALTER TABLE meta.datasets DROP CONSTRAINT IF EXISTS datasets_file_format_check, " diff --git a/api/tests/test_mlflow.py b/api/tests/test_mlflow.py index ae8f38c..f3e5a38 100644 --- a/api/tests/test_mlflow.py +++ b/api/tests/test_mlflow.py @@ -302,7 +302,11 @@ def _train_run(export_dir, phase="review", personal=False) -> int: recipe = db.fetch_one( "INSERT INTO meta.recipes (dataset_id, version, phase) VALUES (%s, 3, %s) RETURNING id", (ds, phase) )["id"] - user = db.fetch_one("INSERT INTO meta.users (name, email) VALUES ('Ana', 'ana@example.com') RETURNING id")["id"] + # Emails are unique since phase 11 (migration 046): every run's trainer is the one Ana. + found = db.fetch_one("SELECT id FROM meta.users WHERE lower(email) = 'ana@example.com'") + user = ( + found or db.fetch_one("INSERT INTO meta.users (name, email) VALUES ('Ana', 'ana@example.com') RETURNING id") + )["id"] assessor = user if personal else None source = f"assessment:{recipe}:{user}" if personal else f"recipe:{recipe}" xid = db.fetch_one( diff --git a/api/tests/test_oidc.py b/api/tests/test_oidc.py index 0ae2f49..12d2e17 100644 --- a/api/tests/test_oidc.py +++ b/api/tests/test_oidc.py @@ -85,6 +85,8 @@ def test_config_names_the_provider(oidc_env): "member_group": "rainforest-member", "problem": None, "accounts_from_env": False, + "password": False, + "oidc": True, } diff --git a/api/tests/test_passwords.py b/api/tests/test_passwords.py new file mode 100644 index 0000000..e4d198b --- /dev/null +++ b/api/tests/test_passwords.py @@ -0,0 +1,512 @@ +"""Phase 11: password accounts. Hashing and the password rules, the sign-in throttle (account and address), one answer +for every refusal, the one-time reset / invite / setup links, must-change-on-first-sign-in, and pick mode needing the +dev flag. Needs the test database; each test signs in from an address of its own so throttles never meet.""" + +import asyncio +import hashlib +import uuid + +import pytest +from fastapi.testclient import TestClient + +from app import auth, db, passwords, users +from app.config import get_settings +from app.main import app + +from .team_helpers import DB_URL, app_ # noqa: F401 + +pytestmark = pytest.mark.skipif(not DB_URL, reason="set RAINFOREST_TEST_DATABASE_URL to run the API tests") + +PW = "Lilac-Harbour-Tangent-77" +NEW_PW = "Quartz-Meadow-Lantern-42" +BAD = {"detail": passwords.BAD_CREDENTIALS} + + +@pytest.fixture(autouse=True) +def password_mode(app_, monkeypatch): + """Password sign-in (the default), pick off, one proxy hop.""" + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "password") + monkeypatch.delenv("RAINFOREST_DEV_PICK", raising=False) + monkeypatch.delenv("RAINFOREST_CLIENT_IP_HOPS", raising=False) + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +def _tag() -> str: + return uuid.uuid4().hex[:10] + + +def ip() -> str: + return f"ip-{_tag()}" + + +def make(*, role="member", password: str | None = PW, must_change=False, active=True, email="") -> dict: + """A password account (no password when `password` is None); returns the user with its email.""" + email = email or f"pw-{_tag()}@example.com" + user = users.create(f"Pw {_tag()}", email, role) + with db.connection(system=True) as conn: + if password is not None: + passwords.set_password(conn, user["id"], password, must_change=must_change) + if not active: + conn.execute("UPDATE meta.users SET active = false WHERE id = %s", (user["id"],)) + return {**user, "email": email} + + +def sql(query: str, params=()) -> list[dict]: + with db.connection(system=True) as conn: + return conn.execute(query, params).fetchall() + + +def login(client, email, password, address=None, **headers): + return client.post( + "/api/auth/login", + json={"email": email, "password": password}, + headers={"X-Forwarded-For": address or ip(), **headers}, + ) + + +def throttle(kind: str, key: str) -> dict | None: + rows = sql("SELECT * FROM meta.sign_in_throttle WHERE kind = %s AND key = %s", (kind, key)) + return rows[0] if rows else None + + +def account_key(email: str) -> str: + return hashlib.sha256(email.strip().lower().encode()).hexdigest() + + +def link_for(user_id: int, purpose="reset", created_by=None) -> str: + with db.connection(system=True) as conn: + return passwords.issue_link(conn, user_id, purpose, created_by)["url"].split("token=", 1)[1] + + +@pytest.fixture +def client(app_): + with TestClient(app_) as c: + yield c + + +def admin_client(app_) -> TestClient: + admin = make(role="admin") + c = TestClient(app_) + r = login(c, admin["email"], PW) + assert r.status_code == 200, r.text + return c + + +# ------------------------------------------------------------------------------------------- hashing +def test_hash_is_argon2id_and_never_the_plaintext(): + user = make() + (row,) = sql("SELECT password_hash FROM meta.user_credentials WHERE user_id = %s", (user["id"],)) + assert row["password_hash"].startswith("$argon2id$") + assert PW not in row["password_hash"] + assert passwords.hash_password(PW) != passwords.hash_password(PW) # salted + + +def test_verify_right_wrong_and_missing(): + stored = passwords.hash_password(PW) + assert passwords.verify(stored, PW) + assert not passwords.verify(stored, PW + "x") + assert not passwords.verify(None, PW) + assert not passwords.verify("not a hash", PW) + + +def test_login_with_the_right_password_signs_in(client): + user = make() + r = login(client, user["email"].upper(), PW) # an email is matched without regard to case + assert r.status_code == 200, r.text + assert r.json()["id"] == user["id"] and r.json()["must_change_password"] is False + assert client.get("/api/auth/me").json()["id"] == user["id"] + + +@pytest.mark.parametrize( + "password,needle", + [("short1!", "at least"), ("password1234", "too common"), ("aaaaaaaaaaaaaa", "too common"), ("x" * 300, "at most")], +) +def test_refused_passwords(password, needle): + assert needle in passwords.problem(password) + + +def test_a_password_made_of_the_email_or_name_is_refused(): + assert "email or name" in passwords.problem("jane.doe@example.com", email="jane.doe@example.com") + assert "email or name" in passwords.problem("BartholomewSmith1!", email="a@b.com", name="Bartholomew Smith") + assert passwords.problem(PW, email="a@b.com", name="Jane") is None + + +def test_weak_passwords_are_refused_where_they_are_set(app_): + admin = admin_client(app_) + r = admin.post("/api/admin/people", json={"name": "Weak", "email": f"w-{_tag()}@example.com", "password": "short"}) + assert r.status_code == 422 + r = admin.post( + "/api/admin/people", json={"name": "Weak", "email": f"w-{_tag()}@example.com", "password": "password1234"} + ) + assert r.status_code == 422 and "common" in r.json()["detail"] + user = make() + token = link_for(user["id"]) + c = TestClient(app_) + r = c.post("/api/auth/password-token", json={"token": token, "password": "password1234"}) + assert r.status_code == 422 + assert c.post("/api/auth/password-token/check", json={"token": token}).status_code == 200 # not used up + + +# ------------------------------------------------------------------------------------------ lockout +def test_five_wrong_passwords_lock_the_account_and_even_the_right_one_is_refused(client): + user = make() + address = ip() + for _ in range(5): + assert login(client, user["email"], "Wrong-Password-123", address).status_code == 401 + assert throttle("account", account_key(user["email"]))["locked_until"] is not None + r = login(client, user["email"], PW, address) # the right password, still locked + assert r.status_code == 429 and "Try again in 15 minutes" in r.json()["detail"] + r = login(client, user["email"], PW, ip()) # a different address, same account + assert r.status_code == 429 + + +def test_the_lock_ends(client): + user = make() + for _ in range(5): + login(client, user["email"], "Wrong-Password-123") + sql( + "UPDATE meta.sign_in_throttle SET locked_until = now() - interval '1 second' WHERE kind = 'account' AND key = %s " + "RETURNING key", + (account_key(user["email"]),), + ) + assert login(client, user["email"], PW).status_code == 200 + assert throttle("account", account_key(user["email"])) is None # a success clears the count + + +def test_a_success_resets_the_failure_count(client): + user = make() + for _ in range(4): + login(client, user["email"], "Wrong-Password-123") + assert login(client, user["email"], PW).status_code == 200 + for _ in range(4): + assert login(client, user["email"], "Wrong-Password-123").status_code == 401 + assert login(client, user["email"], PW).status_code == 200 + + +def test_the_lock_is_stored_before_the_refusal(client): + """The sixth attempt is refused with 429, and the lock it sets must survive the refusal (a43613e: raising inside the + transaction rolled it back).""" + user = make() + key = account_key(user["email"]) + sql("INSERT INTO meta.sign_in_throttle (kind, key, failures) VALUES ('account', %s, 5) RETURNING key", (key,)) + r = login(client, user["email"], PW) + assert r.status_code == 429 + row = throttle("account", key) + assert row is not None and row["locked_until"] is not None + assert login(client, user["email"], PW).status_code == 429 + + +def test_an_address_that_fails_twenty_times_is_locked(client): + address = ip() + for i in range(passwords.IP_LIMIT): + assert login(client, f"nobody-{i}-{_tag()}@example.com", "Wrong-Password-123", address).status_code == 401 + assert throttle("ip", address)["locked_until"] is not None + user = make() + assert login(client, user["email"], PW, address).status_code == 429 + assert login(client, user["email"], PW, ip()).status_code == 200 + + +def test_the_client_address_is_the_forwarded_entry_hops_from_the_right(client, monkeypatch): + user = make() + mine, theirs = ip(), ip() + # One hop (the default): the last entry; what a client puts before it is ignored. + assert login(client, user["email"], "Wrong-Password-123", f"{theirs}, {mine}").status_code == 401 + assert throttle("ip", mine) and throttle("ip", theirs) is None + # Two hops: the second from the right. + monkeypatch.setenv("RAINFOREST_CLIENT_IP_HOPS", "2") + get_settings.cache_clear() + mine2, proxy = ip(), ip() + login(client, user["email"], "Wrong-Password-123", f"{mine2}, {proxy}") + assert throttle("ip", mine2) and throttle("ip", proxy) is None + # Zero hops: the peer, whatever the header says. + monkeypatch.setenv("RAINFOREST_CLIENT_IP_HOPS", "0") + get_settings.cache_clear() + spoof = ip() + login(client, user["email"], "Wrong-Password-123", spoof) + assert throttle("ip", spoof) is None + + +def test_a_spoofed_leftmost_entry_does_not_dodge_the_address_lock(client): + real = ip() + for i in range(passwords.IP_LIMIT): + login(client, f"nobody-{i}-{_tag()}@example.com", "Wrong-Password-123", f"spoof-{i}, {real}") + user = make() + assert login(client, user["email"], PW, f"another, {real}").status_code == 429 + + +# --------------------------------------------------------------------------------------- enumeration +def test_every_refusal_looks_the_same(client): + known = make() + inactive = make(active=False) + no_password = make(password=None) + cases = [ + login(client, f"ghost-{_tag()}@example.com", PW), + login(client, known["email"], "Wrong-Password-123"), + login(client, inactive["email"], PW), + login(client, no_password["email"], PW), + ] + assert {r.status_code for r in cases} == {401} + assert all(r.json() == BAD for r in cases) + assert not any("set-cookie" in r.headers for r in cases) + + +def test_an_unknown_email_locks_like_a_real_one(client): + ghost = f"ghost-{_tag()}@example.com" + real = make() + ghost_codes = [login(client, ghost, "Wrong-Password-123").status_code for _ in range(7)] + real_codes = [login(client, real["email"], "Wrong-Password-123").status_code for _ in range(7)] + assert ghost_codes == real_codes == [401] * 5 + [429] * 2 + bodies = { + login(client, ghost, "x").json()["detail"], + login(client, real["email"], "x").json()["detail"], + } + assert len(bodies) == 1 + + +def test_the_failures_are_audited_with_a_reason(client): + known = make() + login(client, known["email"], "Wrong-Password-123") + rows = sql( + "SELECT detail->>'reason' AS reason FROM meta.audit WHERE action = 'auth.sign_in_failed' AND subject_user = %s", + (known["id"],), + ) + assert [r["reason"] for r in rows] == ["wrong"] + + +# -------------------------------------------------------------------------------------- one-time links +def test_a_link_is_stored_hashed_and_works_once(client, app_): + user = make() + token = link_for(user["id"]) + stored = sql("SELECT hash, purpose FROM meta.password_tokens WHERE user_id = %s", (user["id"],)) + assert [r["hash"] for r in stored] == [hashlib.sha256(token.encode()).hexdigest()] and token not in stored[0][ + "hash" + ] + info = client.post("/api/auth/password-token/check", json={"token": token}) + assert info.status_code == 200 + assert info.json() == { + "name": user["name"], + "email": user["email"], + "purpose": "reset", + "needs_email": False, + } + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 200 and r.json()["id"] == user["id"] + assert client.get("/api/auth/me").json()["id"] == user["id"] # signed in by it + # Used: both the check and a second redemption are refused, with the same message. + again = TestClient(app_) + for r in ( + again.post("/api/auth/password-token/check", json={"token": token}), + again.post("/api/auth/password-token", json={"token": token, "password": "Another-Fine-Pass-99"}), + ): + assert r.status_code == 404 and r.json()["detail"] == passwords.BAD_LINK + assert login(again, user["email"], NEW_PW).status_code == 200 + assert login(again, user["email"], PW).status_code == 401 + + +def test_an_expired_link_is_refused(client): + user = make() + token = link_for(user["id"]) + sql( + "UPDATE meta.password_tokens SET expires_at = now() - interval '1 second' WHERE user_id = %s RETURNING id", + (user["id"],), + ) + assert client.post("/api/auth/password-token/check", json={"token": token}).status_code == 404 + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 404 and r.json()["detail"] == passwords.BAD_LINK + assert login(client, user["email"], PW).status_code == 200 # the old password is untouched + + +def test_lifetimes_by_purpose(): + user = make() + with db.connection(system=True) as conn: + for purpose, hours in passwords.TOKEN_HOURS.items(): + link = passwords.issue_link(conn, user["id"], purpose, None) + row = conn.execute( + "SELECT extract(epoch FROM expires_at - now()) / 3600 AS h FROM meta.password_tokens WHERE hash = %s", + (passwords._token_hash(link["url"].split("token=", 1)[1]),), + ).fetchone() + assert hours - 0.1 < row["h"] <= hours + assert (passwords.TOKEN_HOURS["reset"], passwords.TOKEN_HOURS["invite"]) == (24, 72) + + +def test_a_new_link_revokes_the_earlier_one(client): + user = make() + old = link_for(user["id"]) + new = link_for(user["id"], "invite") + assert client.post("/api/auth/password-token/check", json={"token": old}).status_code == 404 + assert client.post("/api/auth/password-token/check", json={"token": new}).json()["purpose"] == "invite" + + +def test_unknown_garbage_and_inactive_links_are_refused(client): + assert client.post("/api/auth/password-token/check", json={"token": "no-such-token"}).status_code == 404 + gone = make(active=False) + token = link_for(gone["id"]) + assert client.post("/api/auth/password-token/check", json={"token": token}).status_code == 404 + r = client.post("/api/auth/password-token", json={"token": "x" * 129, "password": NEW_PW}) + assert r.status_code == 422 + + +def test_redeeming_signs_the_person_out_elsewhere_and_clears_the_lock(app_): + user = make() + elsewhere = TestClient(app_) + assert login(elsewhere, user["email"], PW).status_code == 200 + for _ in range(5): + login(TestClient(app_), user["email"], "Wrong-Password-123") + assert throttle("account", account_key(user["email"]))["locked_until"] is not None + c = TestClient(app_) + token = link_for(user["id"]) + assert c.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}).status_code == 200 + assert elsewhere.get("/api/auth/me").status_code == 401 # session_version moved on + assert c.get("/api/auth/me").status_code == 200 + assert throttle("account", account_key(user["email"])) is None # the reset lifts a lock + + +def test_a_setup_link_for_an_account_without_an_email_asks_for_one(client, app_): + made = users.create(f"Setup {_tag()}", None, "admin") + token = link_for(made["id"], "setup") + info = client.post("/api/auth/password-token/check", json={"token": token}).json() + assert info["needs_email"] is True and info["purpose"] == "setup" + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 422 and "email" in r.json()["detail"] + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": "nope"}) + assert r.status_code == 422 + taken = make() + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": taken["email"]}) + assert r.status_code == 422 and "already" in r.json()["detail"] + email = f"setup-{_tag()}@example.com" + r = client.post("/api/auth/password-token", json={"token": token, "password": NEW_PW, "email": email}) + assert r.status_code == 200 and r.json()["email"] == email + assert login(TestClient(app_), email, NEW_PW).status_code == 200 + + +# ------------------------------------------------------------------ admin: accounts and links, must change +def test_an_admin_creates_a_person_with_an_initial_password_that_must_change(app_): + admin = admin_client(app_) + email = f"init-{_tag()}@example.com" + r = admin.post("/api/admin/people", json={"name": "Init", "email": email, "password": PW}) + assert r.status_code == 201, r.text + assert r.json()["setup_url"] is None + person = TestClient(app_) + r = login(person, email, PW) + assert r.status_code == 200 and r.json()["must_change_password"] is True + assert person.get("/api/auth/me").json()["must_change_password"] is True + blocked = person.get("/api/projects") + assert blocked.status_code == 403 and blocked.json()["detail"]["code"] == "must_change_password" + # Same password again is refused; a new one clears the flag and the rest of the app opens. + r = person.post("/api/auth/password", json={"current": PW, "new": PW}) + assert r.status_code == 422 + r = person.post("/api/auth/password", json={"current": "Wrong-Password-123", "new": NEW_PW}) + assert r.status_code == 400 and r.json()["detail"] == passwords.BAD_CURRENT + r = person.post("/api/auth/password", json={"current": PW, "new": NEW_PW}) + assert r.status_code == 200 and r.json()["must_change_password"] is False + assert person.get("/api/projects").status_code == 200 + assert login(TestClient(app_), email, NEW_PW).json()["must_change_password"] is False + + +def test_wrong_current_passwords_are_throttled_like_sign_in(app_): + person = make() + c = TestClient(app_) + assert login(c, person["email"], PW).status_code == 200 + codes = [ + c.post("/api/auth/password", json={"current": "Wrong-Password-123", "new": NEW_PW}).status_code + for _ in range(6) + ] + assert codes == [400] * 5 + [429] + + +def test_an_invite_link_is_returned_once_and_sets_the_password(app_): + admin = admin_client(app_) + email = f"inv-{_tag()}@example.com" + r = admin.post("/api/admin/people", json={"name": "Invitee", "email": email}) + assert r.status_code == 201, r.text + token = r.json()["setup_url"].split("token=", 1)[1] + assert r.json()["setup_url"].startswith("/set-password#token=") and r.json()["setup_expires_at"] + stored = sql( + "SELECT hash, purpose FROM meta.password_tokens WHERE hash = %s", (hashlib.sha256(token.encode()).hexdigest(),) + ) + assert stored and stored[0]["purpose"] == "invite" + assert token not in str(sql("SELECT * FROM meta.password_tokens WHERE user_id = %s", (r.json()["id"],))) + person = TestClient(app_) + assert login(person, email, PW).status_code == 401 # no password yet + r = person.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}) + assert r.status_code == 200 and r.json()["must_change_password"] is False + assert person.get("/api/projects").status_code == 200 + + +def test_people_need_an_email_and_a_unique_one(app_): + admin = admin_client(app_) + assert admin.post("/api/admin/people", json={"name": "NoMail"}).status_code == 422 + taken = make() + r = admin.post("/api/admin/people", json={"name": "Dup", "email": taken["email"].upper()}) + assert r.status_code == 409 + + +def test_a_reset_link_leaves_the_old_password_working_until_used(app_): + admin = admin_client(app_) + person = make() + r = admin.post(f"/api/admin/people/{person['id']}/reset-link") + assert r.status_code == 200, r.text + token = r.json()["url"].split("token=", 1)[1] + c = TestClient(app_) + assert login(c, person["email"], PW).status_code == 200 + assert c.post("/api/auth/password-token", json={"token": token, "password": NEW_PW}).status_code == 200 + assert login(TestClient(app_), person["email"], PW).status_code == 401 + assert login(TestClient(app_), person["email"], NEW_PW).status_code == 200 + + +def test_reset_links_are_admin_only_and_refuse_unsuitable_people(app_): + admin = admin_client(app_) + member = make() + other = make() + c = TestClient(app_) + login(c, member["email"], PW) + assert c.post(f"/api/admin/people/{other['id']}/reset-link").status_code == 403 + assert TestClient(app_).post(f"/api/admin/people/{other['id']}/reset-link").status_code == 401 + assert admin.post("/api/admin/people/999999999/reset-link").status_code == 404 + assert admin.post(f"/api/admin/people/{make(active=False)['id']}/reset-link").status_code == 409 + no_email = users.create(f"NoMail {_tag()}", None, "member") + assert admin.post(f"/api/admin/people/{no_email['id']}/reset-link").status_code == 409 + + +# --------------------------------------------------------------------------------------------- pick mode +def test_pick_without_the_dev_flag_is_refused(client, monkeypatch): + user = make() + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + get_settings.cache_clear() + assert "RAINFOREST_DEV_PICK" in auth.config_problem() + assert client.post("/api/auth/pick", json={"user_id": user["id"]}).status_code in (404, 503) + assert client.get("/api/auth/users").status_code in (404, 503) + assert client.get("/api/auth/me").status_code == 503 # every other route says why + assert not client.cookies + assert client.get("/api/auth/config").json()["problem"] + + +def test_the_server_does_not_start_in_pick_mode_without_the_flag(monkeypatch): + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + get_settings.cache_clear() + + async def start(): + async with app.router.lifespan_context(app): + pass + + with pytest.raises(SystemExit, match="RAINFOREST_DEV_PICK"): + asyncio.run(start()) + + +def test_pick_works_with_the_flag(client, monkeypatch): + user = make() + monkeypatch.setenv("RAINFOREST_AUTH_MODE", "pick") + monkeypatch.setenv("RAINFOREST_DEV_PICK", "true") + get_settings.cache_clear() + assert any(u["id"] == user["id"] for u in client.get("/api/auth/users").json()) + r = client.post("/api/auth/pick", json={"user_id": user["id"]}) + assert r.status_code == 200 and r.json()["mode"] == "pick" + assert client.get("/api/auth/me").json()["id"] == user["id"] + + +def test_pick_and_login_are_off_in_the_wrong_mode(client): + user = make() + assert client.post("/api/auth/pick", json={"user_id": user["id"]}).status_code == 404 + assert client.get("/api/auth/users").status_code == 404 diff --git a/api/tests/test_rls_app.py b/api/tests/test_rls_app.py index d62ddd0..8d1c824 100644 --- a/api/tests/test_rls_app.py +++ b/api/tests/test_rls_app.py @@ -333,7 +333,7 @@ def policies(conn): ).fetchall() paths = [p for p in sorted(db.MIGRATIONS_DIR.glob("*.sql")) if p.stem >= "008"] - assert paths[-1].stem.startswith("044_") + assert paths[-1].stem.startswith("046_") # a new migration: check that the replay still holds, then bump with db.owner_connection() as conn: before = policies(conn) assert conn.execute("SELECT count(*) AS n FROM meta.datasets").fetchone()["n"] >= 2 diff --git a/api/tests/test_route_matrix.py b/api/tests/test_route_matrix.py index a8b249c..b896bab 100644 --- a/api/tests/test_route_matrix.py +++ b/api/tests/test_route_matrix.py @@ -536,7 +536,10 @@ def _vault_rows(persona: str) -> dict[str, Row]: if not names or not names <= VAULT_IDS.keys() or row.outsider == OPEN: continue if template.startswith("/api/admin"): # not project routes: an admin is let in, anyone else is refused - out[key] = Row(F, OK, F, F, F, body=row.body, query=row.query) + # (a route's own gate stays: the reset link is a 404 in pick mode, whoever asks) + out[key] = Row( + F, row.admin_outsider if row.admin_outsider == GATE else OK, F, F, F, body=row.body, query=row.query + ) continue if template.startswith(("/api/auth", "/api/users", "/api/notices", "/predict")): continue diff --git a/deploy/chart/rainforest/tests/api_test.yaml b/deploy/chart/rainforest/tests/api_test.yaml index 16e6194..c18004c 100644 --- a/deploy/chart/rainforest/tests/api_test.yaml +++ b/deploy/chart/rainforest/tests/api_test.yaml @@ -329,14 +329,30 @@ tests: path: data.RAINFOREST_OIDC_DISCOVERY_URL value: http://keycloak:8080/.well-known/openid-configuration + # Phase 11: built-in accounts are the default, and pick mode never turns on without devPick. + - it: defaults to password mode with the development pick flag off + template: configmap.yaml + asserts: + - equal: + path: data.RAINFOREST_AUTH_MODE + value: password + - equal: + path: data.RAINFOREST_DEV_PICK + value: "false" + - it: carries pick mode's user list template: configmap.yaml set: + auth.mode: pick + auth.devPick: true auth.users: "tanner:Tanner:admin" asserts: - equal: path: data.RAINFOREST_AUTH_MODE value: pick + - equal: + path: data.RAINFOREST_DEV_PICK + value: "true" - equal: path: data.RAINFOREST_USERS value: "tanner:Tanner:admin" diff --git a/deploy/smoke.sh b/deploy/smoke.sh index a9c5618..d419222 100755 --- a/deploy/smoke.sh +++ b/deploy/smoke.sh @@ -21,6 +21,8 @@ # # Environment: # BASE seed mode: the URL of a running stack's UI (pick sign-in, user "Admin"); nothing is built or deleted. +# Password sign-in is the default, so start that stack with RAINFOREST_AUTH_MODE=pick and +# RAINFOREST_DEV_PICK=true (development only; the chart path sets both from ci/dev-values.yaml). # REGISTRY image repository prefix: /{api,ui,mlflow,notebook}: (default: rainforest) # IMAGE_TAG tag of the four images (default: smoke) # BUILD 1: build the images from api/, ui/, mlflow/ and notebook/ (default). 0: use images already present @@ -299,7 +301,8 @@ check_reachable() { [ "$SECONDS" -ge "$deadline" ] && fail "$BASE/api/auth/config does not answer after ${FORWARD_TIMEOUT}s" sleep 1 done - jq -e '.mode == "pick"' "$WORK/config" >/dev/null || fail "unexpected /api/auth/config: $(cat "$WORK/config")" + jq -e '.mode == "pick"' "$WORK/config" >/dev/null \ + || fail "the stack must run RAINFOREST_AUTH_MODE=pick with RAINFOREST_DEV_PICK=true; /api/auth/config: $(cat "$WORK/config")" log "seeding $BASE: $(jq -c . "$WORK/config")" } diff --git a/docker-compose.yml b/docker-compose.yml index 2d0c490..0c38991 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ x-api-env: &api-env RAINFOREST_CONNECT_DENY: ${RAINFOREST_CONNECT_DENY:-127.0.0.0/8,::1/128,0.0.0.0/8,169.254.0.0/16,fe80::/10,fd00:ec2::254/128,100.100.100.200/32,168.63.129.16/32,224.0.0.0/4,255.255.255.255/32,ff00::/8,fc00::/7} # Empty turns MLflow logging off. The server serves everything, its REST API too, under /mlflow. MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI-http://mlflow:5000/mlflow} + MLFLOW_LOGGING: ${MLFLOW_LOGGING:-true} MLFLOW_BROWSE_URL: ${MLFLOW_BROWSE_URL:-} MLFLOW_EXPERIMENT_PREFIX: ${MLFLOW_EXPERIMENT_PREFIX:-rainforest} # MLflow's own pages cannot apply project access: closed unless true (docs/mlflow.md). diff --git a/docs/user-guide.md b/docs/user-guide.md index bf64b1e..752ed51 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -786,7 +786,7 @@ text and on the job's page), on the run, and on the Lab's card: - *Waiting for the GPU · about 3 ahead*: three jobs are in front of it. *About* means the count is Kueue's and moves as teams use the card; the built-in queue gives an exact count and drops *about*. -- *first in your team, Fraud* or *2 ahead in your team, Fraud*: its place within its team's line. Work from projects +- *first in its team, Fraud* or *2 ahead in its team, Fraud*: its place within its team's line. Work from projects without a team waits in the shared queue (*first in the shared queue*). - *in use by Ada's Lab (Risk), at most 1 h 40 min left*: who holds the card. *At most* is an upper bound, because a Lab or study can end sooner; a holder past its bound reads *finishing*. Work from a project you can't see is named diff --git a/ui/e2e/fixture/gpuQueue.spec.ts b/ui/e2e/fixture/gpuQueue.spec.ts index e353ddd..d5c576f 100644 --- a/ui/e2e/fixture/gpuQueue.spec.ts +++ b/ui/e2e/fixture/gpuQueue.spec.ts @@ -28,7 +28,7 @@ test("the Jobs page shows a waiting line with its team line and who orders it", const row = page.locator('[data-testid="job-row"][data-job-id="9101"]'); await expect(row).toBeVisible(); await expect(row.getByTestId("job-waiting")).toHaveText("Waiting for the GPU · about 2 ahead"); - await expect(row.getByTestId("job-waiting")).toHaveAttribute("title", /first in your team, Fraud/); + await expect(row.getByTestId("job-waiting")).toHaveAttribute("title", /first in its team, Fraud/); await expect(row.getByTestId("job-waiting")).toHaveAttribute( "title", /in use by Tuned · XGBoost study \(Risk\), at most/, @@ -48,7 +48,7 @@ test("a GPU Lab waits in line and starts on the CPU instead", async ({ page }) = await page.goto("/notebooks"); const card = page.getByTestId("lab-card"); await expect(page.getByTestId("lab-status")).toHaveText("Queued"); - await expect(page.getByTestId("lab-waiting-detail")).toContainText("first in your team, Fraud"); + await expect(page.getByTestId("lab-waiting-detail")).toContainText("first in its team, Fraud"); await card.getByTestId("lab-to-cpu").click(); await expect(page.getByRole("dialog")).toContainText("Start your Lab on the CPU instead of waiting for the GPU?"); await page.getByRole("dialog").getByRole("button", { name: "Start on CPU instead" }).click(); diff --git a/ui/src/gpuQueue.test.ts b/ui/src/gpuQueue.test.ts index 580f80e..3aa220b 100644 --- a/ui/src/gpuQueue.test.ts +++ b/ui/src/gpuQueue.test.ts @@ -32,10 +32,10 @@ describe("waitingDetail", () => { it("waiting: the team's place and the holder with the time it has left", () => { const holder = { label: "Tuned · XGBoost study", team: "Risk", since: at(8, 0), ends_by: at(11, 40) }; expect(detail({ in_use_by: holder })).toBe( - "(first in your team, Fraud) — in use by Tuned · XGBoost study (Risk), at most 1 h 40 min left", + "(first in its team, Fraud) — in use by Tuned · XGBoost study (Risk), at most 1 h 40 min left", ); expect(detail({ ahead_in_team: 3, in_use_by: { ...holder, ends_by: null, team: null } })).toBe( - "(3 ahead in your team, Fraud) — in use by Tuned · XGBoost study", + "(3 ahead in its team, Fraud) — in use by Tuned · XGBoost study", ); }); it("next: who holds it, since when", () => { diff --git a/ui/src/kit/kit.test.tsx b/ui/src/kit/kit.test.tsx index d476272..a757a6c 100644 --- a/ui/src/kit/kit.test.tsx +++ b/ui/src/kit/kit.test.tsx @@ -91,12 +91,17 @@ describe("JobStatus waiting", () => { const el = render(); const line = el.querySelector('[data-testid="job-waiting"]')!; expect(line.textContent).toBe("Waiting for the GPU · about 2 ahead"); - expect(line.getAttribute("title")).toBe("(1 ahead in your team, Fraud)"); + expect(line.getAttribute("title")).toBe("(1 ahead in its team, Fraud)"); }); - it("shows nothing for a running job", () => { - const el = render(); - expect(el.querySelector('[data-testid="job-waiting"]')).toBeNull(); + // A claimed GPU job still starting (pulling, unschedulable) keeps its line; the API drops `waiting` once it runs. + it.each([ + ["running", w, true], + ["running", null, false], + ["done", w, false], + ] as const)("a %s job with waiting %s shows the line: %s", (status, waiting, shown) => { + const el = render(); + expect(el.querySelector('[data-testid="job-waiting"]') !== null).toBe(shown); }); }); diff --git a/ui/src/pages/notebooks/notebooksPage.test.tsx b/ui/src/pages/notebooks/notebooksPage.test.tsx index 2e46d98..9464b11 100644 --- a/ui/src/pages/notebooks/notebooksPage.test.tsx +++ b/ui/src/pages/notebooks/notebooksPage.test.tsx @@ -289,7 +289,7 @@ describe("GPU Labs", () => { await mount("/notebooks", ); expect(byId("lab-waiting")?.textContent).toBe("Waiting for the GPU · about 2 ahead"); expect(text()).not.toContain("Waiting for a slot"); - expect(byId("lab-waiting-detail")?.textContent).toBe("(1 ahead in your team, Fraud) — in use by Ann's Lab (Risk)"); + expect(byId("lab-waiting-detail")?.textContent).toBe("(1 ahead in its team, Fraud) — in use by Ann's Lab (Risk)"); expect(byId("stop-lab")?.textContent).toBe("Cancel"); await act(async () => byId("lab-to-cpu")!.click()); await settle(); diff --git a/ui/src/theme.test.ts b/ui/src/theme.test.ts index bf6baad..aa75a48 100644 --- a/ui/src/theme.test.ts +++ b/ui/src/theme.test.ts @@ -38,8 +38,10 @@ describe("theme preference", () => { }); describe("settings sections", () => { - it("shows Tags to admins and project owners and Storage to everyone", () => { - expect(visibleSections({ role: "admin" }).map((s) => s.key)).toEqual(expect.arrayContaining(["tags", "storage"])); + it("shows Tags to admins and project owners, Storage to everyone, and Application only to admins", () => { + expect(visibleSections({ role: "admin" }).map((s) => s.key)).toEqual( + expect.arrayContaining(["tags", "storage", "gpu-queue", "application"]), + ); for (const role of ["member"] as const) { expect(visibleSections({ role }).map((s) => s.key)).toEqual([ "profile", @@ -48,7 +50,6 @@ describe("settings sections", () => { "notebooks", "storage", "integrations", - "application", ]); } });