Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
bf54642
Phase 9 S1: migration 043, RLS and job plumbing
LikelyCause Nov 16, 2026
0198ebc
Phase 9 B1: FeatureFrame and native categoricals in the export
LikelyCause Nov 16, 2026
defceb2
Phase 9 U1: modeling types, client, fixtures and helpers
LikelyCause Nov 16, 2026
f454a4d
Phase 9 U2: the device switch, the hardware facts and run on the CPU …
LikelyCause Nov 16, 2026
7e99cfd
Phase 9 B1 fixes: native lookup built once, compact grouped, edge cas…
LikelyCause Nov 16, 2026
1ee4094
Phase 9 U3: native categoricals in decisions and the model picker
LikelyCause Nov 16, 2026
0b162a0
Phase 9 U3 fixes: encode via withOption, shared experiment access rea…
LikelyCause Nov 16, 2026
0c52dd3
Phase 9 U3 fixes: encodeFields test mocks the catalog settings
LikelyCause Nov 16, 2026
70116b8
Phase 9 B2: native categoricals in training, scoring, bundles and the…
LikelyCause Nov 16, 2026
4d22f9e
Phase 9 U4: the scores table, the reference class and Types I–IV
LikelyCause Nov 16, 2026
a2c901d
Phase 9 U4 fixes: legend swatches, per-target reference, evaluation g…
LikelyCause Nov 16, 2026
ea522a3
Phase 9 U5: about this field
LikelyCause Nov 16, 2026
5b297fa
Phase 9 U5 fixes: about drafts keyed by field, single-line descriptio…
LikelyCause Nov 16, 2026
2f5989c
Phase 9 B2 fixes: one null rule (input_schema null_tokens), diabetes …
LikelyCause Nov 16, 2026
f574e3b
Phase 9 B3: GPU runs take the GPU slot; the switch's data, the hardwa…
LikelyCause Nov 16, 2026
5e9965a
Phase 9 U6: tune with Optuna and the study view
LikelyCause Nov 16, 2026
d454f68
Phase 9 B3 fixes: pinned 403 and 422 copy, owner-move assertions, own…
LikelyCause Nov 16, 2026
59982db
Phase 9 U6 fixes: study ends refresh, gated Stop, none in the space, …
LikelyCause Nov 16, 2026
c738139
Phase 9 U7: drop and re-run, derived versions in review, lineage
LikelyCause Nov 16, 2026
fc2d08c
Phase 9 U7 fixes: derive access by the run's own role, one derivation…
LikelyCause Nov 16, 2026
7d87c5a
Phase 9 B4: one metrics function for runs and evaluations; micro, fal…
LikelyCause Nov 16, 2026
02f357b
Phase 9 B5: field descriptions and notes
LikelyCause Nov 16, 2026
b7404d4
Phase 9 B6: baseline exchange /2 carries field descriptions and notes
LikelyCause Nov 16, 2026
f1dd664
Phase 9 B6 fixes: dry run previews the about fill (plan_fill), skip r…
LikelyCause Nov 16, 2026
aa49136
Phase 9 B7: Optuna, default search spaces and starting a tuning study
LikelyCause Nov 16, 2026
739a19d
Phase 9 B7: openapi.json keeps /api/auth/check's operation ids (FastA…
LikelyCause Nov 16, 2026
fe52b38
Phase 9 B8: the run.tune job: Optuna trials as rows, pruning, resume,…
LikelyCause Nov 16, 2026
0888712
Phase 9 B8 fixes: no trial left running at any end (one conclude), a …
LikelyCause Nov 16, 2026
32a8486
Phase 9 B9: derive a baseline version from a run; pending derivations…
LikelyCause Nov 16, 2026
3ae1c6c
Phase 9 B9 fixes: finalize rights from the locked version; a pending …
LikelyCause Nov 16, 2026
00db83e
Phase 9 B10: re-run from a derived version; then_run; automatic rebas…
LikelyCause Nov 16, 2026
3c28b44
Phase 9 B10 fixes: restore B9's finalize role re-check; then_run neve…
LikelyCause Nov 16, 2026
0ff00a8
Phase 9 B11: suite, route matrix, docs and smoke
LikelyCause Nov 16, 2026
8bcd5aa
Merge branch 'phase-9-api' into phase-9-ui
LikelyCause Nov 16, 2026
0cb8a63
Phase 9 U8: generated types, carry-over fixes, UI tests and the user …
LikelyCause Nov 16, 2026
3c90c7f
Phase 9 final review fixes: run visibility on derive/rerun, tune orde…
LikelyCause Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ RAINFOREST_SECRET_KEY=
# OVR_MIN_POSITIVES=10
# OVR_WEAK_PR_AUC=0.5
# BATCH_MAX_RUNS=500
# TUNE_MAX_TRIALS=1000
# TUNE_MAX_TRIALS_GPU=200
# TUNE_MAX_HOURS_GPU=4
# PREDICT_MAX_RECORDS=1000
# SERVE_PORT=8010
# SERVE_URL=http://serve:8010
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
api:
name: API tests
runs-on: ubuntu-24.04
timeout-minutes: 20
timeout-minutes: 45
services:
postgres:
image: postgres:16
Expand Down
3 changes: 3 additions & 0 deletions api/app/analysis/compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ def run_map(
"hover_fields": [c.name for c in hover],
"hover_values": [list(row) for row in zip(*values)] if values else [[] for _ in range(p.n)],
"skipped": p.skipped,
**({"notes": p.notes} if p.notes else {}),
}


Expand Down Expand Up @@ -202,6 +203,7 @@ def run_tests(p: Prepared, check: Check = _noop) -> dict:
"n": n,
"rows": out,
"skipped": skipped,
**({"notes": p.notes} if p.notes else {}),
}


Expand Down Expand Up @@ -383,6 +385,7 @@ def run_importance(p: Prepared, check: Check = _noop) -> dict:
"components": components,
},
"skipped": p.skipped,
**({"notes": p.notes} if p.notes else {}),
}


Expand Down
17 changes: 14 additions & 3 deletions api/app/analysis/prep.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ class Prepared:
target_kind: str | None = None # categorical | numeric
y: np.ndarray | None = None # int class codes or float values, aligned with column rows
classes: list[str] = field(default_factory=list)
notes: list[str] = field(default_factory=list) # what a result should say about how a field was read

@property
def n(self) -> int:
Expand Down Expand Up @@ -196,16 +197,26 @@ def prepare_export(export_dir: Path, manifest: dict, target: str | None, sample:
c for c in pl.read_parquet_schema(path) if c not in (ROW_ID, SPLIT_COL, ORIGIN_DATASET, ORIGIN_ROW, *targets)
]
df = read_rows(path, names, train[idx])
columns, skipped = [], []
columns, skipped, notes = [], [], []
levels = {k: {v: float(i) for i, v in enumerate(lv)} for k, lv in (manifest.get("category_levels") or {}).items()}
source = manifest.get("feature_source_field") or {}
for name in names:
col = numeric_column(name, None, df[name].cast(pl.Float64).fill_nan(None).to_numpy())
if name in levels:
# A native category column: its codes (the export's own), missing and unseen as NaN.
raw = df[name].cast(pl.String).replace_strict(levels[name], default=None, return_dtype=pl.Float64)
notes.append(
f"{source.get(name, name)} is passed through as a category; this study coded it as ordinal codes."
)
else:
raw = df[name].cast(pl.Float64)
col = numeric_column(name, None, raw.fill_nan(None).to_numpy())
if col is None:
skipped.append({"name": name, "reason": "constant in sample"})
else:
columns.append(col)
classes = [str(c) for c in (manifest.get("target_classes") or {}).get(target, [])] if target else []
y = None if y_full is None else y_full[idx]
return Prepared(columns, skipped, target, kind, y, classes)
return Prepared(columns, skipped, target, kind, y, classes, list(dict.fromkeys(notes)))


def design_matrix(columns: list[Column]) -> tuple[np.ndarray, list[str], list[int]]:
Expand Down
1 change: 1 addition & 0 deletions api/app/analysis/studies.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ def run_study(p: Prepared, params: dict, check: Check = _noop) -> dict:
"clusters": None,
**body,
"skipped": p.skipped,
**({"notes": p.notes} if p.notes else {}),
}


Expand Down
9 changes: 8 additions & 1 deletion api/app/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -565,12 +565,19 @@ def revoke_token(token_id: int, user: CurrentUser) -> dict:
return row


@router.api_route("/check", methods=["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"], status_code=204)
def check() -> Response:
"""For nginx auth_request: 204 with a valid session; the middleware answers 401 (or 403 cross-origin) otherwise."""
return Response(status_code=204)


# One route per method, each with its own operation id: FastAPI would name a many-method route from an unordered set,
# so openapi.json would change from one generation to the next.
for _method in ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"):
router.add_api_route(
"/check", check, methods=[_method], status_code=204, operation_id=f"auth_check_{_method.lower()}"
)


@router.post("/logout")
def logout(request: Request) -> JSONResponse:
"""Clear the session cookie. In OIDC mode also returns the provider's sign-out URL for the browser to visit."""
Expand Down
14 changes: 10 additions & 4 deletions api/app/batches.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,11 +310,15 @@ def leaderboard(rows: list[dict], ranking: str) -> list[dict]:

def running_batch(conn, *, dataset_id: int | None = None, experiment_id: int | None = None) -> dict | None:
"""A batch with runs still to finish on the dataset or in the experiment (the oldest), or None. Its waiting runs hold off
whatever would pull the data or the experiment from under them, so the refusal can say which batch to cancel."""
whatever would pull the data or the experiment from under them, so the refusal can say which batch to cancel. A
tuning study counts while its job is live (queued, running, stopping or paused): its trials read the export. A
study whose job ended without ending it doesn't hold anything (its Stop ends it)."""
return conn.execute(
"""SELECT b.id, b.name FROM meta.run_batches b JOIN meta.exports e ON e.id = b.export_id
WHERE b.status = 'running' AND (%(ds)s::bigint IS NULL OR e.dataset_id = %(ds)s)
AND EXISTS (SELECT 1 FROM meta.model_runs r WHERE r.batch_id = b.id AND r.status IN ('pending', 'running'))
AND ((b.kind = 'tune' AND EXISTS (SELECT 1 FROM meta.jobs j WHERE j.id = b.job_id
AND j.status IN ('queued', 'running', 'stopping', 'paused')))
OR EXISTS (SELECT 1 FROM meta.model_runs r WHERE r.batch_id = b.id AND r.status IN ('pending', 'running')))
AND (%(x)s::bigint IS NULL OR b.experiment_id = %(x)s)
ORDER BY b.id LIMIT 1""",
{"ds": dataset_id, "x": experiment_id},
Expand All @@ -339,11 +343,12 @@ def _feed(conn, batch_id: int) -> None:
from . import jobkinds

batch = conn.execute(
"""SELECT b.status, b.created_by, e.output_dir, e.dataset_id FROM meta.run_batches b
"""SELECT b.kind, b.status, b.created_by, e.output_dir, e.dataset_id FROM meta.run_batches b
JOIN meta.exports e ON e.id = b.export_id WHERE b.id = %s FOR UPDATE OF b""",
(batch_id,),
).fetchone()
if batch is None or batch["status"] != "running":
# A tuning study is one job: its runs (the refit) don't feed it, and its job writes its end.
if batch is None or batch["status"] != "running" or batch["kind"] == "tune":
return
if conn.execute(
"""SELECT 1 FROM meta.model_runs r LEFT JOIN meta.jobs j ON j.id = r.job_id
Expand Down Expand Up @@ -377,6 +382,7 @@ def _feed(conn, batch_id: int) -> None:
dataset_id=batch["dataset_id"],
created_by=run["created_by"],
preemptible=True,
needs=jobkinds.needs_for(run["config"].get("device", "cpu")),
)


Expand Down
116 changes: 116 additions & 0 deletions api/app/bulk_apply.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
"""The bulk decisions write: what `POST /datasets/{id}/decisions/bulk` does once the route has resolved the source and
checked that it may be written. Derive-baseline calls it too, with run provenance."""

from __future__ import annotations

import uuid

from psycopg import sql

from . import recipes
from .assessment import lock_decisions, write_decision, write_settings
from .db import jsonb
from .field_work import (
_jsonb_or_null,
_merged_many,
_provenance,
_snapshot,
_sync_targets,
_write_history,
plan_bulk,
summarize_bulk,
)
from .routers.query import fields_where
from .schemas import BulkDecisionIn


def apply(conn, src, body: BulkDecisionIn, user_id: int) -> dict:
dataset_id = src.recipe["dataset_id"]
fam = src.family
if body.filters is not None:
target = sql.SQL("d.field_id IN (SELECT field_id FROM {} WHERE {})").format(
src.fields_view(), fields_where(dataset_id, body.filters)
)
else:
target = sql.SQL("d.field_id = ANY({})").format(sql.Literal(body.field_ids))
# Fields carried over into this version are locked; bulk edits pass over them.
if src.kind in ("team", "assessment") and (open_ := recipes.open_field_ids(conn, src.recipe)) is not None:
target = sql.SQL("{} AND d.field_id = ANY({})").format(target, sql.Literal(open_))
# Resolve and lock in one statement so the set can't drift between preview and write. An assessor's rows
# are only theirs to change, so they are locked right after.
rows = conn.execute(
sql.SQL("""SELECT d.field_id, f.name, d.action, d.semantic_type, d.reviewed, d.notes, f.suggestion
FROM {} d JOIN meta.fields f ON f.id = d.field_id
WHERE d.dataset_id = {} AND {} AND NOT d.field_id = ANY({})
ORDER BY f.ordinal {}""").format(
src.decisions(),
sql.Literal(dataset_id),
target,
sql.Literal(body.excluded_ids),
sql.SQL("FOR UPDATE OF d" if src.team else ""),
),
).fetchall()
ids = [r["field_id"] for r in rows]
if not src.team:
lock_decisions(conn, src, ids)
settings: dict[int, dict[str, dict]] = {}
for srow in conn.execute(
sql.SQL("SELECT * FROM {} s WHERE field_id = ANY(%s)").format(src.settings()), (ids,)
).fetchall():
settings.setdefault(srow["field_id"], {})[srow["family"]] = srow
if src.kind == "experiment":
# An experiment's copy holds its baseline's model type only; other types' suggestions don't apply.
rows = [
{
**r,
"suggestion": {src.family: (r["suggestion"] or {}).get(src.family)}
if (r["suggestion"] or {}).get(src.family)
else {},
}
for r in rows
]
planned = plan_bulk(rows, settings, body)
result = summarize_bulk(len(rows), planned)
batch_prov, field_prov = _provenance(conn, dataset_id, body, {r["field_id"]: r["name"] for r in rows})
if body.dry_run or not planned:
conn.rollback()
return {"batch_id": None, "dry_run": body.dry_run, **result}

batch_id = uuid.uuid4()
changed_ids = [c["field_id"] for c in planned]
if batch_prov is not None:
batch_prov["ranks"] = {str(i): field_prov[i]["rank"] for i in changed_ids}
snapshot = _snapshot(conn, src, changed_ids)
prev = _merged_many(conn, changed_ids, fam, src)
notes = {r["field_id"]: r["notes"] for r in rows}
for c in planned:
decided = any(c["after"][k] != c["before"][k] for k in ("action", "semantic_type"))
write_decision(conn, src, c["field_id"], {**c["after"], "notes": notes[c["field_id"]]}, decided)
for family, sugg in c["settings"].items():
write_settings(conn, src, dataset_id, c["field_id"], family, sugg, "suggested")
summary = jsonb(
{**{k: v for k, v in result.items() if k != "changes"}, "request": body.model_dump(exclude={"dry_run"})}
)
if src.personal:
conn.execute(
"""INSERT INTO meta.assessment_batches (id, recipe_id, user_id, summary, snapshot, provenance)
VALUES (%s, %s, %s, %s, %s, %s)""",
(batch_id, src.recipe["id"], src.user_id, summary, jsonb(snapshot), _jsonb_or_null(batch_prov)),
)
return {"batch_id": str(batch_id), "dry_run": False, **result}
if src.kind == "experiment":
conn.execute(
"""INSERT INTO meta.experiment_batches (id, experiment_id, user_id, summary, snapshot, provenance)
VALUES (%s, %s, %s, %s, %s, %s)""",
(batch_id, src.experiment["id"], user_id, summary, jsonb(snapshot), _jsonb_or_null(batch_prov)),
)
return {"batch_id": str(batch_id), "dry_run": False, **result}
_write_history(conn, dataset_id, fam, batch_id, prev, _merged_many(conn, changed_ids, fam), user_id, field_prov)
conn.execute(
"""INSERT INTO meta.bulk_batches (id, dataset_id, summary, snapshot, user_id, provenance)
VALUES (%s, %s, %s, %s, %s, %s)""",
(batch_id, dataset_id, summary, jsonb(snapshot), user_id, _jsonb_or_null(batch_prov)),
)
_sync_targets(conn, dataset_id, user_id=user_id)

return {"batch_id": str(batch_id), "dry_run": False, **result}
5 changes: 5 additions & 0 deletions api/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ class Settings(BaseSettings):
ovr_min_positives: int = 10
ovr_weak_pr_auc: float = 0.5
batch_max_runs: int = 500
# Tuning studies: the most trials one may ask for on the CPU (its time ceiling is tuning.MAX_HOURS_CPU), and the
# most trials and hours on the GPU, so a study can't hold the only GPU all day.
tune_max_trials: int = 1000
tune_max_trials_gpu: int = 200
tune_max_hours_gpu: int = 4
# Serving (phase 4): the default records per predict call of a new endpoint.
predict_max_records: int = 1000
# The serve process: its port, where the API reaches it for its health (empty: not configured), and the
Expand Down
Loading
Loading