diff --git a/.env.example b/.env.example index ba234a46..05365f73 100644 --- a/.env.example +++ b/.env.example @@ -83,6 +83,9 @@ RAINFOREST_SECRET_KEY= # OVR_MIN_POSITIVES=10 # OVR_WEAK_PR_AUC=0.5 # BATCH_MAX_RUNS=500 +# TUNE_MAX_TRIALS=1000 +# TUNE_MAX_TRIALS_GPU=200 +# TUNE_MAX_HOURS_GPU=4 # PREDICT_MAX_RECORDS=1000 # SERVE_PORT=8010 # SERVE_URL=http://serve:8010 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02c7b9a8..1384cdcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: api: name: API tests runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 45 services: postgres: image: postgres:16 diff --git a/api/app/analysis/compute.py b/api/app/analysis/compute.py index 96123df9..9f976b10 100644 --- a/api/app/analysis/compute.py +++ b/api/app/analysis/compute.py @@ -134,6 +134,7 @@ def run_map( "hover_fields": [c.name for c in hover], "hover_values": [list(row) for row in zip(*values)] if values else [[] for _ in range(p.n)], "skipped": p.skipped, + **({"notes": p.notes} if p.notes else {}), } @@ -202,6 +203,7 @@ def run_tests(p: Prepared, check: Check = _noop) -> dict: "n": n, "rows": out, "skipped": skipped, + **({"notes": p.notes} if p.notes else {}), } @@ -383,6 +385,7 @@ def run_importance(p: Prepared, check: Check = _noop) -> dict: "components": components, }, "skipped": p.skipped, + **({"notes": p.notes} if p.notes else {}), } diff --git a/api/app/analysis/prep.py b/api/app/analysis/prep.py index a91758ea..efb73961 100644 --- a/api/app/analysis/prep.py +++ b/api/app/analysis/prep.py @@ -50,6 +50,7 @@ class Prepared: target_kind: str | None = None # categorical | numeric y: np.ndarray | None = None # int class codes or float values, aligned with column rows classes: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) # what a result should say about how a field was read @property def n(self) -> int: @@ -196,16 +197,26 @@ def prepare_export(export_dir: Path, manifest: dict, target: str | None, sample: c for c in pl.read_parquet_schema(path) if c not in (ROW_ID, SPLIT_COL, ORIGIN_DATASET, ORIGIN_ROW, *targets) ] df = read_rows(path, names, train[idx]) - columns, skipped = [], [] + columns, skipped, notes = [], [], [] + levels = {k: {v: float(i) for i, v in enumerate(lv)} for k, lv in (manifest.get("category_levels") or {}).items()} + source = manifest.get("feature_source_field") or {} for name in names: - col = numeric_column(name, None, df[name].cast(pl.Float64).fill_nan(None).to_numpy()) + if name in levels: + # A native category column: its codes (the export's own), missing and unseen as NaN. + raw = df[name].cast(pl.String).replace_strict(levels[name], default=None, return_dtype=pl.Float64) + notes.append( + f"{source.get(name, name)} is passed through as a category; this study coded it as ordinal codes." + ) + else: + raw = df[name].cast(pl.Float64) + col = numeric_column(name, None, raw.fill_nan(None).to_numpy()) if col is None: skipped.append({"name": name, "reason": "constant in sample"}) else: columns.append(col) classes = [str(c) for c in (manifest.get("target_classes") or {}).get(target, [])] if target else [] y = None if y_full is None else y_full[idx] - return Prepared(columns, skipped, target, kind, y, classes) + return Prepared(columns, skipped, target, kind, y, classes, list(dict.fromkeys(notes))) def design_matrix(columns: list[Column]) -> tuple[np.ndarray, list[str], list[int]]: diff --git a/api/app/analysis/studies.py b/api/app/analysis/studies.py index 10a15ee4..82d7006d 100644 --- a/api/app/analysis/studies.py +++ b/api/app/analysis/studies.py @@ -65,6 +65,7 @@ def run_study(p: Prepared, params: dict, check: Check = _noop) -> dict: "clusters": None, **body, "skipped": p.skipped, + **({"notes": p.notes} if p.notes else {}), } diff --git a/api/app/auth.py b/api/app/auth.py index c8a7f87f..0d0effe6 100644 --- a/api/app/auth.py +++ b/api/app/auth.py @@ -565,12 +565,19 @@ def revoke_token(token_id: int, user: CurrentUser) -> dict: return row -@router.api_route("/check", methods=["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"], status_code=204) def check() -> Response: """For nginx auth_request: 204 with a valid session; the middleware answers 401 (or 403 cross-origin) otherwise.""" return Response(status_code=204) +# One route per method, each with its own operation id: FastAPI would name a many-method route from an unordered set, +# so openapi.json would change from one generation to the next. +for _method in ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE"): + router.add_api_route( + "/check", check, methods=[_method], status_code=204, operation_id=f"auth_check_{_method.lower()}" + ) + + @router.post("/logout") def logout(request: Request) -> JSONResponse: """Clear the session cookie. In OIDC mode also returns the provider's sign-out URL for the browser to visit.""" diff --git a/api/app/batches.py b/api/app/batches.py index 8e34e5d1..0f4b3841 100644 --- a/api/app/batches.py +++ b/api/app/batches.py @@ -310,11 +310,15 @@ def leaderboard(rows: list[dict], ranking: str) -> list[dict]: def running_batch(conn, *, dataset_id: int | None = None, experiment_id: int | None = None) -> dict | None: """A batch with runs still to finish on the dataset or in the experiment (the oldest), or None. Its waiting runs hold off - whatever would pull the data or the experiment from under them, so the refusal can say which batch to cancel.""" + whatever would pull the data or the experiment from under them, so the refusal can say which batch to cancel. A + tuning study counts while its job is live (queued, running, stopping or paused): its trials read the export. A + study whose job ended without ending it doesn't hold anything (its Stop ends it).""" return conn.execute( """SELECT b.id, b.name FROM meta.run_batches b JOIN meta.exports e ON e.id = b.export_id WHERE b.status = 'running' AND (%(ds)s::bigint IS NULL OR e.dataset_id = %(ds)s) - AND EXISTS (SELECT 1 FROM meta.model_runs r WHERE r.batch_id = b.id AND r.status IN ('pending', 'running')) + AND ((b.kind = 'tune' AND EXISTS (SELECT 1 FROM meta.jobs j WHERE j.id = b.job_id + AND j.status IN ('queued', 'running', 'stopping', 'paused'))) + OR EXISTS (SELECT 1 FROM meta.model_runs r WHERE r.batch_id = b.id AND r.status IN ('pending', 'running'))) AND (%(x)s::bigint IS NULL OR b.experiment_id = %(x)s) ORDER BY b.id LIMIT 1""", {"ds": dataset_id, "x": experiment_id}, @@ -339,11 +343,12 @@ def _feed(conn, batch_id: int) -> None: from . import jobkinds batch = conn.execute( - """SELECT b.status, b.created_by, e.output_dir, e.dataset_id FROM meta.run_batches b + """SELECT b.kind, b.status, b.created_by, e.output_dir, e.dataset_id FROM meta.run_batches b JOIN meta.exports e ON e.id = b.export_id WHERE b.id = %s FOR UPDATE OF b""", (batch_id,), ).fetchone() - if batch is None or batch["status"] != "running": + # A tuning study is one job: its runs (the refit) don't feed it, and its job writes its end. + if batch is None or batch["status"] != "running" or batch["kind"] == "tune": return if conn.execute( """SELECT 1 FROM meta.model_runs r LEFT JOIN meta.jobs j ON j.id = r.job_id @@ -377,6 +382,7 @@ def _feed(conn, batch_id: int) -> None: dataset_id=batch["dataset_id"], created_by=run["created_by"], preemptible=True, + needs=jobkinds.needs_for(run["config"].get("device", "cpu")), ) diff --git a/api/app/bulk_apply.py b/api/app/bulk_apply.py new file mode 100644 index 00000000..d5ed0259 --- /dev/null +++ b/api/app/bulk_apply.py @@ -0,0 +1,116 @@ +"""The bulk decisions write: what `POST /datasets/{id}/decisions/bulk` does once the route has resolved the source and +checked that it may be written. Derive-baseline calls it too, with run provenance.""" + +from __future__ import annotations + +import uuid + +from psycopg import sql + +from . import recipes +from .assessment import lock_decisions, write_decision, write_settings +from .db import jsonb +from .field_work import ( + _jsonb_or_null, + _merged_many, + _provenance, + _snapshot, + _sync_targets, + _write_history, + plan_bulk, + summarize_bulk, +) +from .routers.query import fields_where +from .schemas import BulkDecisionIn + + +def apply(conn, src, body: BulkDecisionIn, user_id: int) -> dict: + dataset_id = src.recipe["dataset_id"] + fam = src.family + if body.filters is not None: + target = sql.SQL("d.field_id IN (SELECT field_id FROM {} WHERE {})").format( + src.fields_view(), fields_where(dataset_id, body.filters) + ) + else: + target = sql.SQL("d.field_id = ANY({})").format(sql.Literal(body.field_ids)) + # Fields carried over into this version are locked; bulk edits pass over them. + if src.kind in ("team", "assessment") and (open_ := recipes.open_field_ids(conn, src.recipe)) is not None: + target = sql.SQL("{} AND d.field_id = ANY({})").format(target, sql.Literal(open_)) + # Resolve and lock in one statement so the set can't drift between preview and write. An assessor's rows + # are only theirs to change, so they are locked right after. + rows = conn.execute( + sql.SQL("""SELECT d.field_id, f.name, d.action, d.semantic_type, d.reviewed, d.notes, f.suggestion + FROM {} d JOIN meta.fields f ON f.id = d.field_id + WHERE d.dataset_id = {} AND {} AND NOT d.field_id = ANY({}) + ORDER BY f.ordinal {}""").format( + src.decisions(), + sql.Literal(dataset_id), + target, + sql.Literal(body.excluded_ids), + sql.SQL("FOR UPDATE OF d" if src.team else ""), + ), + ).fetchall() + ids = [r["field_id"] for r in rows] + if not src.team: + lock_decisions(conn, src, ids) + settings: dict[int, dict[str, dict]] = {} + for srow in conn.execute( + sql.SQL("SELECT * FROM {} s WHERE field_id = ANY(%s)").format(src.settings()), (ids,) + ).fetchall(): + settings.setdefault(srow["field_id"], {})[srow["family"]] = srow + if src.kind == "experiment": + # An experiment's copy holds its baseline's model type only; other types' suggestions don't apply. + rows = [ + { + **r, + "suggestion": {src.family: (r["suggestion"] or {}).get(src.family)} + if (r["suggestion"] or {}).get(src.family) + else {}, + } + for r in rows + ] + planned = plan_bulk(rows, settings, body) + result = summarize_bulk(len(rows), planned) + batch_prov, field_prov = _provenance(conn, dataset_id, body, {r["field_id"]: r["name"] for r in rows}) + if body.dry_run or not planned: + conn.rollback() + return {"batch_id": None, "dry_run": body.dry_run, **result} + + batch_id = uuid.uuid4() + changed_ids = [c["field_id"] for c in planned] + if batch_prov is not None: + batch_prov["ranks"] = {str(i): field_prov[i]["rank"] for i in changed_ids} + snapshot = _snapshot(conn, src, changed_ids) + prev = _merged_many(conn, changed_ids, fam, src) + notes = {r["field_id"]: r["notes"] for r in rows} + for c in planned: + decided = any(c["after"][k] != c["before"][k] for k in ("action", "semantic_type")) + write_decision(conn, src, c["field_id"], {**c["after"], "notes": notes[c["field_id"]]}, decided) + for family, sugg in c["settings"].items(): + write_settings(conn, src, dataset_id, c["field_id"], family, sugg, "suggested") + summary = jsonb( + {**{k: v for k, v in result.items() if k != "changes"}, "request": body.model_dump(exclude={"dry_run"})} + ) + if src.personal: + conn.execute( + """INSERT INTO meta.assessment_batches (id, recipe_id, user_id, summary, snapshot, provenance) + VALUES (%s, %s, %s, %s, %s, %s)""", + (batch_id, src.recipe["id"], src.user_id, summary, jsonb(snapshot), _jsonb_or_null(batch_prov)), + ) + return {"batch_id": str(batch_id), "dry_run": False, **result} + if src.kind == "experiment": + conn.execute( + """INSERT INTO meta.experiment_batches (id, experiment_id, user_id, summary, snapshot, provenance) + VALUES (%s, %s, %s, %s, %s, %s)""", + (batch_id, src.experiment["id"], user_id, summary, jsonb(snapshot), _jsonb_or_null(batch_prov)), + ) + return {"batch_id": str(batch_id), "dry_run": False, **result} + _write_history(conn, dataset_id, fam, batch_id, prev, _merged_many(conn, changed_ids, fam), user_id, field_prov) + conn.execute( + """INSERT INTO meta.bulk_batches (id, dataset_id, summary, snapshot, user_id, provenance) + VALUES (%s, %s, %s, %s, %s, %s)""", + (batch_id, dataset_id, summary, jsonb(snapshot), user_id, _jsonb_or_null(batch_prov)), + ) + _sync_targets(conn, dataset_id, user_id=user_id) + + return {"batch_id": str(batch_id), "dry_run": False, **result} diff --git a/api/app/config.py b/api/app/config.py index b486d252..23fd8cf9 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -95,6 +95,11 @@ class Settings(BaseSettings): ovr_min_positives: int = 10 ovr_weak_pr_auc: float = 0.5 batch_max_runs: int = 500 + # Tuning studies: the most trials one may ask for on the CPU (its time ceiling is tuning.MAX_HOURS_CPU), and the + # most trials and hours on the GPU, so a study can't hold the only GPU all day. + tune_max_trials: int = 1000 + tune_max_trials_gpu: int = 200 + tune_max_hours_gpu: int = 4 # Serving (phase 4): the default records per predict call of a new endpoint. predict_max_records: int = 1000 # The serve process: its port, where the API reaches it for its health (empty: not configured), and the diff --git a/api/app/derive.py b/api/app/derive.py new file mode 100644 index 00000000..e20ae3ef --- /dev/null +++ b/api/app/derive.py @@ -0,0 +1,866 @@ +"""Derive a baseline version from a run: the run's weak fields dropped in the next version of its baseline. + +`derive` starts version N+1 at once when the run's version is finalized and nothing is in progress; when the run's own +version is still in progress, it records the drops as *pending* (`meta.pending_derivations`), and finalizing that +version (`after_finalize`) creates N+1 from them. A derived version is `scope = 'reopened'` with its drops as the +reopened fields, so the locking everything else uses applies unchanged; `derived_from_run_id` tells it apart. Its +creator may finalize it while it differs from its parent only by drops (`drops_only`): `self_finalize`. + +`meta.pending_derivations` rows have two meanings. An open row (`done_recipe_id` and `cancelled_at` NULL) is drops waiting +for its version to be finalized. A row with `done_recipe_id` set is the record of a derived version (what it was +asked for: target, rule, fields, who), and an immediate derivation writes one with `recipe_id = done_recipe_id`. Every +query for waiting drops filters on `done_recipe_id IS NULL`. + +The audit payloads these functions return are written by the route after its transaction has committed +(`write_audit`), as the baseline export and import routes do. +""" + +from __future__ import annotations + +import copy +import json +import logging +from contextlib import contextmanager +from pathlib import Path + +from fastapi import HTTPException + +from . import access, bulk_apply, experiments, projects, recipes, users +from .assessment import Source +from .dataset_work import queue_target_work +from .db import connection, jsonb +from .schemas import BatchRequest, BulkDecisionIn, DeriveIn, ExportRequest, RerunIn, RunProvenance, RunRequest + +log = logging.getLogger(__name__) + +NOTHING_LEFT = "Nothing left to drop" +WAITING_FOR_EXPORT = "Waiting for its export" +# A retried export that then succeeds doesn't train the run its failure ended. +RETRY_HINT = " (retrying the export won't train this run: re-run it from the run's page)" + + +# ----------------------------------------------------------------------------------------------------- classifying +def classify(snapshot: dict, names: list[str]) -> tuple[list[str], list[dict]]: + """Which of `names` are still kept in `snapshot` (to drop), and why the rest are left out.""" + applied: list[str] = [] + skipped: list[dict] = [] + for name in names: + f = snapshot["fields"].get(name) + if f is None: + skipped.append({"name": name, "reason": "no longer in the data"}) + elif f["decision"]["action"] == "keep": + applied.append(name) + else: + skipped.append({"name": name, "reason": "already dropped"}) + return applied, skipped + + +def _fields(conn, dataset_id: int, field_ids: list[int]) -> list[dict]: + rows = conn.execute( + "SELECT id, name, is_target FROM meta.fields WHERE dataset_id = %s AND id = ANY(%s) ORDER BY ordinal", + (dataset_id, field_ids), + ).fetchall() + known = {r["id"] for r in rows} + if unknown := [i for i in dict.fromkeys(field_ids) if i not in known]: + raise HTTPException(422, {"message": "Unknown field(s)", "errors": [str(i) for i in unknown]}) + if targets := [r["name"] for r in rows if r["is_target"]]: + raise HTTPException(422, f"{', '.join(targets)}: a target can't be dropped") + return rows + + +def _in_progress(cur: dict) -> HTTPException: + """The 409 for a derivation while another version is in progress; `pending` names that version for the UI: + `{version, id}`, `id` its recipe id (as in the 409 for drops already waiting on a version).""" + return HTTPException( + 409, + { + "message": f"{recipes.label(cur)} is in progress on this dataset; drop these fields in the team decision " + "instead.", + "pending": {"version": cur["version"], "id": cur["id"]}, + }, + ) + + +def _version(recipe: dict) -> dict: + return {"recipe_id": recipe["id"], "version": recipe["version"], "phase": recipe["phase"]} + + +# ----------------------------------------------------------------------------------------------------------- derive +def derive(conn, run: dict, user: dict, body: DeriveIn) -> tuple[dict, dict | None]: + """The result (the Contract's `DeriveResult`) and the audit payload of a self-finalize (None otherwise). The caller + holds the dataset's row lock and has checked access, readiness and that the run is done. A dry run changes + nothing.""" + dataset_id = run["dataset_id"] + made = conn.execute("SELECT recipe_id FROM meta.exports WHERE id = %s", (run["export_id"],)).fetchone() + run_recipe = recipes.get(conn, made["recipe_id"]) if made and made["recipe_id"] is not None else None + if run_recipe is None: + raise HTTPException(409, f"Run #{run['id']} has no baseline version to drop fields from.") + family = run_recipe["family"] + names = [f["name"] for f in _fields(conn, dataset_id, body.field_ids)] + cur = recipes.current(conn, dataset_id, lock="update") + + if cur["phase"] != "finalized": + if cur["id"] != run_recipe["id"]: + raise _in_progress(cur) + return _pending(conn, run, user, body, cur, names), None + + parent = recipes.latest_finalized(conn, dataset_id, family) + applied, skipped = classify(recipes.snapshot_of(conn, parent), names) + if not applied: + raise HTTPException(422, f"None of these fields is still kept in v{parent['version']}.") + result = { + "mode": "immediate", + "parent": {"recipe_id": parent["id"], "version": parent["version"]}, + "version": None, + "pending_id": None, + "applied": applied, + "skipped": skipped, + "finalized": body.finalize_now, # a dry run says what the real call would do + "message": None, + } + if body.dry_run: + conn.rollback() + return result, None + rule = {"target": body.target, "rule": body.rule} + try: + new = _create(conn, dataset_id, family, user["id"], run["id"], rule, applied) + except HTTPException as e: + # Lost a race for the one version in progress (start_version's refusal): name that version. Any other + # refusal (nothing could be dropped) stands as it is. + now = recipes.current(conn, dataset_id) + if e.status_code == 409 and now["phase"] != "finalized": + raise _in_progress(now) from None + raise + conn.execute( + """INSERT INTO meta.pending_derivations (recipe_id, run_id, field_names, rule, finalize_now, created_by, + done_recipe_id) + VALUES (%s, %s, %s, %s, %s, %s, %s)""", + (new["id"], run["id"], applied, jsonb(rule), body.finalize_now, user["id"], new["id"]), + ) + payload = None + if body.finalize_now: + payload = self_finalize(conn, new, user["id"], run["id"], applied) + new = recipes.get(conn, new["id"]) + return {**result, "version": _version(new), "finalized": new["phase"] == "finalized"}, payload + + +def _pending(conn, run: dict, user: dict, body: DeriveIn, cur: dict, names: list[str]) -> dict: + """The run's own version is in progress: record the drops to make once it is finalized.""" + n = cur["version"] + applied, skipped = classify(recipes.snapshot_of(conn, cur), names) + if not applied: + raise HTTPException(422, f"None of these fields is still kept in v{n}.") + waiting = conn.execute( + """SELECT id, run_id FROM meta.pending_derivations + WHERE recipe_id = %s AND done_recipe_id IS NULL AND cancelled_at IS NULL""", + (cur["id"],), + ).fetchone() + if waiting: + raise HTTPException( + 409, + { + "message": f"v{n} already has drops waiting from run #{waiting['run_id']}.", + "pending": {"version": n, "id": cur["id"]}, # the version's recipe id, as in _in_progress + }, + ) + state = "in review" if cur["phase"] == "review" else "being assessed" + result = { + "mode": "pending", + "parent": {"recipe_id": cur["id"], "version": n}, + "version": None, + "pending_id": None, + "applied": applied, + "skipped": skipped, + "finalized": False, + "message": f"v{n} is still {state}. The re-run starts now as a draft; v{n + 1} with these drops is created " + f"when v{n} is finalized", + } + if body.dry_run: + # The drops are classified again against the snapshot finalizing freezes, which the team may still change. + conn.rollback() + return { + **result, + "message": f"{result['message']}. Which of these fields are still kept is worked out again then, so " + "this preview can change", + } + # Every name asked for is kept: what is still kept is worked out again when the version is finalized. + pid = conn.execute( + """INSERT INTO meta.pending_derivations (recipe_id, run_id, field_names, rule, finalize_now, created_by) + VALUES (%s, %s, %s, %s, %s, %s) RETURNING id""", + (cur["id"], run["id"], names, jsonb({"target": body.target, "rule": body.rule}), body.finalize_now, user["id"]), + ).fetchone()["id"] + return {**result, "pending_id": pid} + + +def _create(conn, dataset_id: int, family: str, creator_id: int, run_id: int, rule: dict, applied: list[str]) -> dict: + """Version N+1 of `family`, in review, with `applied` dropped by the run (the caller checked they are kept in the + version it copies) and everything else carried over, locked.""" + recipe = recipes.start_version(conn, dataset_id, creator_id, family, phase="review", scope="reopened") + uncorrelated, added = recipe.pop("_target_work") + recipe.pop("_own") + conn.execute("UPDATE meta.recipes SET derived_from_run_id = %s WHERE id = %s", (run_id, recipe["id"])) + ids = [ + r["id"] + for r in conn.execute( + "SELECT id FROM meta.fields WHERE dataset_id = %s AND name = ANY(%s) ORDER BY ordinal", + (dataset_id, applied), + ).fetchall() + ] + conn.execute( + "INSERT INTO meta.recipe_reopened (recipe_id, field_id) SELECT %s, unnest(%s::bigint[])", (recipe["id"], ids) + ) + recipe = recipes.get(conn, recipe["id"]) + done = bulk_apply.apply( + conn, + Source(recipe), + BulkDecisionIn( + field_ids=ids, + action="drop", + reviewed=True, + provenance=RunProvenance(kind="run", run_id=run_id, target=rule["target"], rule=rule["rule"]), + ), + creator_id, + ) + # An empty plan makes bulk_apply roll the transaction back, which would silently undo the version just started. + if done["batch_id"] is None: + raise HTTPException(409, "None of these fields could be dropped.") + queue_target_work(conn, dataset_id, uncorrelated, added, user_id=creator_id) + return recipes.get(conn, recipe["id"]) + + +# ----------------------------------------------------------------------------------------------------- finalizing +def drops_only(conn, recipe: dict) -> bool: + """Whether the version differs from its parent only by fields moving from keep to drop, with the same targets: the + team decision as it stands while in review (what finalize would freeze), the snapshot once finalized.""" + if recipe["parent_id"] is None: + return False + parent = recipes.get(conn, recipe["parent_id"]) + if parent is None or parent["dataset_id"] != recipe["dataset_id"]: + return False + family = recipe["family"] + working = ( + recipes.snapshot_of(conn, recipe) + if recipe["phase"] == "finalized" + else recipes.finalized_snapshot(conn, recipe["dataset_id"], family) + ) + d = recipes.diff(recipes.snapshot_of(conn, parent), working, family) + if d["targets"] is not None: + return False + only_drops = all( + f["change"] == "changed" + and all(c["key"] == "action" and c["before"] == "keep" and c["after"] == "drop" for c in f["changes"]) + for f in d["fields"] + ) + # An un-drop is a change too: it is no longer the drops the version was derived with. + dropped = {f["name"] for f in d["fields"]} + reopened = { + r["name"] + for r in conn.execute( + "SELECT f.name FROM meta.recipe_reopened o JOIN meta.fields f ON f.id = o.field_id WHERE o.recipe_id = %s", + (recipe["id"],), + ).fetchall() + } + return only_drops and dropped == reopened + + +def self_finalize(conn, recipe: dict, user_id: int, run_id: int, fields: list[str]) -> dict: + """Finalize a derived version as its creator. The caller has checked who may and that it is drops-only. Returns the + audit payload (`baseline.self_finalize`); the payloads of what finalizing set off are under `then`.""" + from .routers.team import _require_phase, _review_summary, review_record + + recipe = recipes.get(conn, recipe["id"]) + _require_phase(recipe, "review", doing="finalizing") + status = conn.execute("SELECT status FROM meta.datasets WHERE id = %s", (recipe["dataset_id"],)).fetchone() + if status["status"] != "ready": + raise HTTPException(409, "The dataset is still processing") + done = recipes.finalize( + conn, + recipe, + user_id, + review_record(_review_summary(conn, recipe)) | {"derived_from_run": run_id, "self_finalized": True}, + ) + return { + "action": "baseline.self_finalize", + "actor_id": user_id, + "object_id": done["id"], + "detail": { + "dataset_id": done["dataset_id"], + "recipe_id": done["id"], + "version": done["version"], + "run_id": run_id, + "fields": fields, + }, + "then": after_finalize(conn, done, {"id": user_id}), + } + + +def after_finalize(conn, recipe: dict, actor: dict) -> list[dict]: + """Called with every finalize, in its transaction: the version's open pending derivation, if any, becomes version + N+1, created by whoever asked for it; then the re-run drafts of the dataset that now match the finalized version + rebase onto it. Returns the audit payloads to write after the commit.""" + out = _next_version(conn, recipe, actor) + _auto_rebase(conn, recipe) + return out + + +def _next_version(conn, recipe: dict, actor: dict) -> list[dict]: + p = conn.execute( + """SELECT * FROM meta.pending_derivations + WHERE recipe_id = %s AND done_recipe_id IS NULL AND cancelled_at IS NULL FOR UPDATE""", + (recipe["id"],), + ).fetchone() + if p is None: + return [] + applied, _ = classify(recipes.snapshot_of(conn, recipe), p["field_names"]) + if not applied or p["run_id"] is None: + note = NOTHING_LEFT if not applied else "The run is gone" + conn.execute( + "UPDATE meta.pending_derivations SET cancelled_at = now(), note = %s WHERE id = %s", (note, p["id"]) + ) + return [] + # The person who asked made the version. If they are gone (created_by NULL), the version is attributed to the + # person finalizing, and is never finalized for them. + creator = p["created_by"] or actor["id"] + new = _create(conn, recipe["dataset_id"], recipe["family"], creator, p["run_id"], p["rule"], applied) + note = None + finalize = p["finalize_now"] + if finalize and not _may_finalize(conn, recipe, p["created_by"]): + # The role is checked now: they may have been demoted or removed since they asked. + finalize, note = False, "Not finalized: its creator can no longer finalize it" + conn.execute( + "UPDATE meta.pending_derivations SET done_recipe_id = %s, field_names = %s, note = %s WHERE id = %s", + (new["id"], applied, note, p["id"]), + ) + return [self_finalize(conn, new, creator, p["run_id"], applied)] if finalize else [] + + +def _may_finalize(conn, recipe: dict, user_id: int | None) -> bool: + """Whether `user_id` is, right now, an active contributor or above in the dataset's own project.""" + if user_id is None: + return False + who = users.get(user_id) + if not who or not who.get("active"): + return False + row = conn.execute("SELECT project_id FROM meta.datasets WHERE id = %s", (recipe["dataset_id"],)).fetchone() + return row is not None and access.can(who, row["project_id"], "write") + + +def _auto_rebase(conn, recipe: dict) -> None: + """Re-run drafts (of this dataset and family) whose base, with the drops they wait for applied, is what the + finalized version turned out to be (notes aside) move onto it, and their exports stop being drafts. Any other + change in review leaves the draft to the usual rebase, and its exports draft. A draft still waiting for its + derived version is left alone.""" + drafts = conn.execute( + f"""SELECT {experiments.COLS}, x.seed_snapshot FROM {experiments.FROM} + WHERE x.dataset_id = %s AND x.draft AND x.rerun_of_run_id IS NOT NULL AND xr.family = %s + ORDER BY x.id FOR UPDATE OF x""", + (recipe["dataset_id"], recipe["family"]), + ).fetchall() + if not drafts: + return + final = recipes.snapshot_of(conn, recipe) + for x in drafts: + if x["seed_snapshot"] is None or x["recipe_version"] > recipe["version"]: + continue + if experiments.waiting_for(conn, x) is not None: + continue + expected = copy.deepcopy(x["seed_snapshot"]) + if x["pending_derivation_id"] is not None: + p = conn.execute( + """SELECT field_names FROM meta.pending_derivations + WHERE id = %s AND (cancelled_at IS NULL OR note = %s)""", + (x["pending_derivation_id"], NOTHING_LEFT), + ).fetchone() + for name in (p or {}).get("field_names", []): + if name in expected["fields"]: + expected["fields"][name]["decision"]["action"] = "drop" + d = recipes.diff(expected, final, recipe["family"]) + if d["targets"] is not None or any( + f["change"] != "changed" or any(c["key"] != "notes" for c in f["changes"]) for f in d["fields"] + ): + continue + experiments.rebase(conn, x, recipe["id"], base=expected) + conn.execute("UPDATE meta.exports SET draft = false WHERE experiment_id = %s AND draft", (x["id"],)) + + +def write_audit(row: dict, payloads: list[dict]) -> None: + """The audit rows for `payloads` (and what they set off), after the transaction has committed. `row` is the + dataset as an `access.load_*` loader returned it.""" + for p in payloads: + projects.audit_event( + {"id": p["actor_id"]}, + p["action"], + row, + object_kind="recipe", + object_id=p["object_id"], + detail=p["detail"], + ) + write_audit(row, p["then"]) + + +# ------------------------------------------------------------------------------------------------------ team state +def state(conn, recipe: dict) -> dict | None: + """The Contract's `Derived` for a version made from a run, else None.""" + run_id = recipe.get("derived_from_run_id") + if run_id is None: + return None + from . import field_about + + p = conn.execute( + "SELECT rule, field_names FROM meta.pending_derivations WHERE done_recipe_id = %s", (recipe["id"],) + ).fetchone() + stored = conn.execute("SELECT review_summary FROM meta.recipes WHERE id = %s", (recipe["id"],)).fetchone() + return { + "run_id": run_id, + "target": (p or {}).get("rule", {}).get("target"), + "rule": (p or {}).get("rule", {}).get("rule"), + "fields": len(p["field_names"]) if p else len(recipes.open_field_ids(conn, recipe) or []), + "created_by": field_about.user_ref(conn, recipe["created_by"]), + "drops_only": drops_only(conn, recipe), + "self_finalized": bool(((stored or {}).get("review_summary") or {}).get("self_finalized")), + } + + +# ------------------------------------------------------------------------------------------------------------ re-run +def _text(exc: HTTPException) -> str: + d = exc.detail + if isinstance(d, dict): + return "; ".join([str(d.get("message", "")), *map(str, d.get("errors") or [])]).strip("; ") + return str(d) + + +def _source(conn, run: dict, body: RerunIn) -> tuple[dict, dict | None, list[str]]: + """The version the re-run's experiment is built from, the pending derivation it waits on (None when the version + exists), and the fields the derivation dropped.""" + if (body.recipe_id is None) == (body.pending_id is None): + raise HTTPException(422, "Give the derived version (recipe_id) or the pending drops (pending_id).") + if body.pending_id is not None: + p = conn.execute( + "SELECT * FROM meta.pending_derivations WHERE id = %s AND run_id = %s", (body.pending_id, run["id"]) + ).fetchone() + if p is None: + raise HTTPException(404, "Pending drops not found") + if p["cancelled_at"] is not None: + raise HTTPException(409, f"These drops were cancelled: {p['note'] or 'nothing to drop'}.") + if p["done_recipe_id"] is None: + return recipes.get(conn, p["recipe_id"]), p, list(p["field_names"]) + body.recipe_id = p["done_recipe_id"] # the version exists by now + recipe = recipes.get(conn, body.recipe_id) + if recipe is None or recipe["dataset_id"] != run["dataset_id"]: + raise HTTPException(404, "Baseline version not found") + if recipe["derived_from_run_id"] != run["id"]: + raise HTTPException(409, f"{recipes.label(recipe)} wasn't derived from run #{run['id']}.") + dropped = [ + r["name"] + for r in conn.execute( + """SELECT f.name FROM meta.recipe_reopened o JOIN meta.fields f ON f.id = o.field_id + WHERE o.recipe_id = %s ORDER BY f.ordinal""", + (recipe["id"],), + ).fetchall() + ] + return recipe, None, dropped + + +def _copy_adjustments(conn, old: dict, new: dict, skip: set[str]) -> None: + """The old experiment's adjustments, minus those on `skip` (the dropped fields), written over the new one's copy.""" + adj = experiments.adjustments(conn, old) + names = [f["name"] for f in adj["fields"] if f["name"] not in skip] + if names: + ids = [ + r["id"] + for r in conn.execute( + "SELECT id FROM meta.fields WHERE dataset_id = %s AND name = ANY(%s)", (new["dataset_id"], names) + ).fetchall() + ] + conn.execute( + """UPDATE meta.experiment_decisions n SET action = o.action, semantic_type = o.semantic_type, + notes = o.notes, target_task = o.target_task, target_bins = o.target_bins, + target_classes = o.target_classes, sentinels = o.sentinels, reviewed = o.reviewed, + source = o.source, updated_at = now() + FROM meta.experiment_decisions o + WHERE o.experiment_id = %s AND n.experiment_id = %s AND n.field_id = o.field_id + AND n.field_id = ANY(%s)""", + (old["id"], new["id"], ids), + ) + conn.execute( + """UPDATE meta.experiment_settings n SET imputation = o.imputation, impute_value = o.impute_value, + add_missing_indicator = o.add_missing_indicator, transform = o.transform, scaling = o.scaling, + encoding = o.encoding, params = o.params, source = o.source, updated_at = now() + FROM meta.experiment_settings o + WHERE o.experiment_id = %s AND n.experiment_id = %s AND n.field_id = o.field_id + AND n.family = o.family AND n.field_id = ANY(%s)""", + (old["id"], new["id"], ids), + ) + if adj["targets_changed"] and old["target_order"]: + conn.execute("UPDATE meta.experiments SET target_order = %s WHERE id = %s", (old["target_order"], new["id"])) + + +def _run_request(run: dict) -> RunRequest: + """The run's own request, as its config stored it.""" + keys = set(RunRequest.model_fields) - {"experiment_id", "label"} + return RunRequest(**{k: v for k, v in (run["config"] or {}).items() if k in keys}) + + +def _study_request(conn, run: dict) -> BatchRequest: + batch = ( + conn.execute( + "SELECT kind, spec, best_run_id FROM meta.run_batches WHERE id = %s", (run["batch_id"],) + ).fetchone() + if run.get("batch_id") is not None and run.get("trial_number") is not None + else None + ) + if batch is None or batch["kind"] != "tune" or batch["best_run_id"] != run["id"]: + raise HTTPException(422, f"Run #{run['id']} isn't the refit of a tuning study.") + return study_request(batch["spec"]) + + +def study_request(spec: dict, **budget: int | None) -> BatchRequest: + """The request that repeats the tune batch whose stored spec is `spec` (`routers.batches._start_study` writes it); + `budget` replaces its `trials` or `seconds`. The one mapping: a re-run study and the study's Run again use it.""" + ev = spec["evaluation"] + budget = {**spec["budget"], **budget} + ovr = spec.get("positive_class") is not None + return BatchRequest.model_validate( + { + "kind": "tune", + "model": spec["model"], + "device": spec["device"], + "params": spec.get("params") or {}, + "seed": spec.get("run_seed", 42), + "cv": spec.get("cv", 5), + "cv_folds": spec.get("cv_folds", 5), + "space": spec["space"], + "ranking": spec["metric"], + "order": None if ovr else spec.get("order"), + "balance": bool(spec.get("balance")), + "tune": { + "trials": budget["trials"], + "seconds": budget["seconds"], + "patience": budget.get("patience"), + "evaluation": ev["mode"], + "folds": ev.get("folds") or 5, + "pruner": spec.get("pruner", "median"), + "sampler": spec.get("sampler", "tpe"), + "seed": spec.get("seed", 0), + "target": spec.get("target"), + "positive_class": spec.get("positive_class"), + }, + } + ) + + +def rerun(conn, run: dict, user: dict, body: RerunIn) -> dict: + """The Contract's `RerunResult`. A new experiment on the derived version (a draft while it is in progress, with the + drops written as its adjustments when the version doesn't exist yet), carrying the old experiment's other + adjustments and its split plan, and an export of it; once the export is done, `as_was` trains the run again, + `study` tunes again and `customise` leaves it to the user. The caller holds the dataset's row lock.""" + from .routers.exports import start_export_for + from .routers.runs import Checked, check_device, insert_run + + study = _study_request(conn, run) if body.mode == "study" else None + recipe, pending, dropped = _source(conn, run, body) + old = experiments.get(conn, run["experiment_id"]) + old_export = conn.execute( + "SELECT model_family, config, split_plan_id FROM meta.exports WHERE id = %s", (run["export_id"],) + ).fetchone() + req = _run_request(run) if body.mode == "as_was" else None + if req is not None: + from .modeling.registry import MODELS + + spec = MODELS.get(req.model) + if spec is None: + raise HTTPException(422, f"Unknown model {req.model!r}") + check_device(spec, req.device) + + n = len(dropped) + version = recipe["version"] + (1 if pending is not None else 0) + base_name = old["name"] if old else f"Run #{run['id']}" + x = experiments.create( + conn, + run["dataset_id"], + f"{base_name[:150]} · v{version} without {n} fields", + experiments.source_key(recipe["id"], None), + user, + notes=f"Re-run of run #{run['id']} from {base_name}", + project_id=run["project_id"], + ) + conn.execute( + "UPDATE meta.experiments SET rerun_of_run_id = %s, pending_derivation_id = %s WHERE id = %s", + (run["id"], None if pending is None else pending["id"], x["id"]), + ) + x = experiments.get(conn, x["id"]) + if pending is not None: + kept, _ = classify(experiments.base_snapshot(conn, x), dropped) + ids = [ + r["id"] + for r in conn.execute( + "SELECT id FROM meta.fields WHERE dataset_id = %s AND name = ANY(%s) ORDER BY ordinal", + (x["dataset_id"], kept), + ).fetchall() + ] + if ids: + rule = pending["rule"] + done = bulk_apply.apply( + conn, + Source(recipe, experiment=x), + BulkDecisionIn( + field_ids=ids, + action="drop", + reviewed=True, + provenance=RunProvenance(kind="run", run_id=run["id"], target=rule["target"], rule=rule["rule"]), + ), + user["id"], + ) + # As in _create: an empty plan rolls the transaction back, which would undo the experiment just made. + if done["batch_id"] is None: + raise HTTPException(409, "None of the waiting drops could be written to the new experiment.") + if old is not None: + _copy_adjustments(conn, old, x, set(dropped)) + + plan_id = None + if old_export["split_plan_id"] is not None: + plan = conn.execute( + "SELECT name, spec FROM meta.split_plans WHERE id = %s", (old_export["split_plan_id"],) + ).fetchone() + if plan is not None: + plan_id = conn.execute( + "INSERT INTO meta.split_plans (experiment_id, name, spec, created_by) VALUES (%s, %s, %s, %s) " + "RETURNING id", + (x["id"], plan["name"], jsonb(plan["spec"]), user["id"]), + ).fetchone()["id"] + keys = set(ExportRequest.model_fields) - {"source", "recipe_id", "experiment_id", "model_family", "split_plan_id"} + export_req = ExportRequest( + **{k: v for k, v in (old_export["config"] or {}).items() if k in keys}, + model_family=old_export["model_family"], + experiment_id=x["id"], + split_plan_id=plan_id, + ) + then_run = {"run_id": None} if req is not None else {"tune": study.model_dump(mode="json")} if study else None + exp = start_export_for(run["dataset_id"], export_req, user, None, then_run=then_run, conn=conn) + + run_id = None + if req is not None: + label = (run["report"] or {}).get("model_label") + row = insert_run( + conn, + { + "id": exp["id"], + "project_id": run["project_id"], + "dataset_id": run["dataset_id"], + "experiment_id": x["id"], + }, + Checked(MODELS[req.model], [], [], label or MODELS[req.model].label), + req, + user["id"], + ) + run_id = row["id"] + conn.execute("UPDATE meta.model_runs SET status_message = %s WHERE id = %s", (WAITING_FOR_EXPORT, run_id)) + conn.execute("UPDATE meta.exports SET then_run = %s WHERE id = %s", (jsonb({"run_id": run_id}), exp["id"])) + x = experiments.get(conn, x["id"]) + return { + "experiment": {"id": x["id"], "name": x["name"], "draft": bool(x["draft"])}, + "export_id": exp["id"], + "run_id": run_id, + "batch_pending": study is not None, + } + + +@contextmanager +def _own(conn): + if conn is not None: + yield conn + else: + with connection() as c: + yield c + + +def export_failed(export_id: int, error: str | None, conn=None) -> None: + """The export a re-run waits for failed or was stopped: its waiting run fails with the reason.""" + first = ((error or "").strip().splitlines() or ["Failed"])[0] + with _own(conn) as c: + c.execute( + """UPDATE meta.model_runs r SET status = 'failed', status_message = 'Failed', error = %s, + finished_at = now(), updated_at = now() + FROM meta.exports e + WHERE e.id = %s AND e.then_run ? 'run_id' AND r.id = (e.then_run ->> 'run_id')::bigint AND r.export_id = e.id + AND r.status = 'pending'""", + (f"The export failed: {first}{RETRY_HINT}", export_id), + ) + + +def _claim(export_id: int, extra: dict | None = None, conn=None) -> bool: + """Mark the export's `then_run` done, once.""" + with _own(conn) as c: + return ( + c.execute( + """UPDATE meta.exports SET then_run = then_run || %s WHERE id = %s AND then_run IS NOT NULL + AND NOT coalesce((then_run ->> 'done')::boolean, false) RETURNING id""", + (jsonb({"done": True, **(extra or {})}), export_id), + ).fetchone() + is not None + ) + + +def _fail_run(export_id: int, run_id: int, text: str) -> None: + """End the waiting run failed and the hook done, in one transaction.""" + with connection() as conn: + if _claim(export_id, conn=conn): + conn.execute( + """UPDATE meta.model_runs SET status = 'failed', status_message = 'Failed', error = %s, + finished_at = now(), updated_at = now() WHERE id = %s AND status = 'pending'""", + (text, run_id), + ) + + +def export_done(export_id: int) -> None: + """What a re-run's export does once it is built: train the run it waits with, or start the study, as the user who + asked. A problem ends the run with its text (a study's is kept in `then_run.error`); never fails the export.""" + try: + _export_done(export_id) + except Exception as exc: # noqa: BLE001 + log.exception("then_run of export %s failed", export_id) + try: + with connection() as conn: + then = conn.execute("SELECT then_run FROM meta.exports WHERE id = %s", (export_id,)).fetchone() + then = (then or {}).get("then_run") or {} + if not then.get("done"): + text = f"The re-run could not start: {type(exc).__name__}" + if then.get("run_id") is not None: + _fail_run(export_id, then["run_id"], text) + else: + with connection() as conn: + started = _study_made(conn, export_id) + _claim(export_id, None if started else {"error": text}) + except Exception: # noqa: BLE001 + log.exception("could not record the failure of then_run of export %s", export_id) + + +def _may_write(user_id: int | None, project_id: int) -> bool: + who = users.get(user_id) if user_id is not None else None + return bool(who and who.get("active") and access.can(who, project_id, "write")) + + +def _export_done(export_id: int) -> None: + from . import jobkinds + + with connection() as conn: + exp = conn.execute( + """SELECT id, project_id, dataset_id, experiment_id, model_family, output_dir, then_run, created_by + FROM meta.exports WHERE id = %s""", + (export_id,), + ).fetchone() + then = (exp or {}).get("then_run") + if not then or then.get("done"): + return + if "tune" in then: + _start_study(exp, then) + return + run_id = then.get("run_id") + with connection() as conn: + run = conn.execute("SELECT * FROM meta.model_runs WHERE id = %s", (run_id,)).fetchone() + if run is None or run["status"] != "pending" or run["export_id"] != export_id: + _claim(export_id) + return + from .routers.runs import check_model, check_params + + problem = None + req = None + try: + if not _may_write(run["created_by"], exp["project_id"]): + raise HTTPException(403, "Not started: whoever asked can no longer train here") + manifest = json.loads((Path(exp["output_dir"]) / "manifest.json").read_text()) + req = _run_request(run) + checked = check_model(exp, manifest, req) + check_params(checked, manifest, req) + except HTTPException as exc: + problem = _text(exc) + except (OSError, ValueError): + problem = "The export's files were removed or unreadable" + except Exception as exc: # noqa: BLE001 + log.exception("checking the re-run of export %s", export_id) + problem = f"The re-run could not start: {type(exc).__name__}" + if problem is not None: + _fail_run(export_id, run_id, problem) + return + try: + with connection() as conn: + if not _claim(export_id, conn=conn): + return + conn.execute("UPDATE meta.model_runs SET status_message = 'Queued' WHERE id = %s", (run_id,)) + jobkinds.enqueue( + conn, + "run.train", + run_id, + project_id=run["project_id"], + args={"export_dir": exp["output_dir"], "request": req.model_dump()}, + dataset_id=run["dataset_id"], + created_by=run["created_by"], + needs=jobkinds.needs_for(req.device), + ) + except Exception as exc: # noqa: BLE001 (the claim rolled back with it) + log.exception("queueing the re-run of export %s", export_id) + _fail_run(export_id, run_id, f"The re-run could not start: {type(exc).__name__}") + + +STUDY_CLAIM_S = 600 # a claimed start older than this, with no study made, was a crash: it may be tried again + + +def _study_made(conn, export_id: int) -> bool: + return ( + conn.execute( + "SELECT 1 FROM meta.run_batches WHERE export_id = %s AND kind = 'tune' LIMIT 1", (export_id,) + ).fetchone() + is not None + ) + + +def _start_study(exp: dict, then: dict) -> None: + """Start the study as the user who asked: claim, commit, start, record. The claim (`then_run.claimed_at`, under + the export row's lock, committed before the study starts) keeps a second caller from starting one too; nothing is + held while start_batch runs on its own connection. The export's one study is the record: whoever finds it marks + the hook done, and a claim left by a crash (older than STUDY_CLAIM_S, no study) is tried again.""" + from . import users as _users + from .routers.batches import start_batch + + export_id = exp["id"] + with connection() as conn: # claim + row = conn.execute("SELECT then_run FROM meta.exports WHERE id = %s FOR NO KEY UPDATE", (export_id,)).fetchone() + current = row["then_run"] if row else None + if not current or current.get("done"): + return + if _study_made(conn, export_id): + conn.execute( + "UPDATE meta.exports SET then_run = then_run || %s WHERE id = %s", (jsonb({"done": True}), export_id) + ) + return + claimed = conn.execute( + """UPDATE meta.exports SET then_run = then_run || jsonb_build_object('claimed_at', now()) + WHERE id = %s AND (NOT then_run ? 'claimed_at' + OR (then_run ->> 'claimed_at')::timestamptz < now() - make_interval(secs => %s)) + RETURNING id""", + (export_id, STUDY_CLAIM_S), + ).fetchone() + if claimed is None: + return # another caller is starting it + error = None + try: # start, holding nothing + user = _users.get(exp["created_by"]) if exp["created_by"] is not None else None + if user is None: + raise HTTPException(409, "The user who asked for this study is gone.") + if not _may_write(exp["created_by"], exp["project_id"]): + raise HTTPException(403, "Not started: whoever asked can no longer train here") + start_batch(export_id, BatchRequest.model_validate(then["tune"]), user) + except HTTPException as exc: + error = _text(exc) + except Exception as exc: # noqa: BLE001 + log.exception("starting the re-run study of export %s", export_id) + error = f"The study could not start: {type(exc).__name__}" + with connection() as conn: # record + if error is not None and _study_made(conn, export_id): + # start_batch made the study, then something after it raised: the study runs; say nothing failed. + log.warning("then_run of export %s: the study started, then: %s", export_id, error) + error = None + conn.execute( + "UPDATE meta.exports SET then_run = (then_run - 'claimed_at') || %s WHERE id = %s", + (jsonb({"done": True, **({"error": error} if error else {})}), export_id), + ) diff --git a/api/app/exchange.py b/api/app/exchange.py index 5ef15608..efaf1ec4 100644 --- a/api/app/exchange.py +++ b/api/app/exchange.py @@ -33,7 +33,8 @@ settings_of, ) -FORMAT = "rainforest-baseline/1" +FORMAT = "rainforest-baseline/2" +FORMAT_V1 = "rainforest-baseline/1" MEDIA_JSON = "application/vnd.rainforest.baseline+json" MEDIA_CSV = "text/csv; charset=utf-8" MAX_BYTES = 5 * 1024 * 1024 @@ -54,14 +55,18 @@ "scaling", "encoding", "params", - "notes", + "decision_notes", + "description", + "field_notes", ) ROLE = {"keep": "feature", "drop": "drop", "target": "target"} _ACTIONS = ("keep", "drop", "target") _STR_KEYS = ("semantic_type", "notes", "target_task", "imputation", "impute_value", "transform", "scaling", "encoding") # CSV column of a decision key whose name differs. -_COLUMN = {"target_classes": "class_policy", "add_missing_indicator": "missing_indicator"} -_BAD_FILE = "This isn't a rainforest-baseline/1 file." +_COLUMN = {"target_classes": "class_policy", "add_missing_indicator": "missing_indicator", "notes": "decision_notes"} +ABOUT_KEYS = ("description", "field_notes") +_BAD_FILE = "This isn't a rainforest-baseline/2 file." +_V1_FILE = "This is a rainforest-baseline/1 file; export it again from a current Rainforest." _BAD_CHARS = re.compile("[\x00\ud800-\udfff]") _PLAIN_NUMBER = re.compile(r"-\d+(\.\d+)?") _MAX_INT = 2**53 @@ -155,6 +160,13 @@ def to_json(conn, recipe: dict, user: dict, instance: str | None) -> dict: for r in conn.execute("SELECT name, ordinal FROM meta.fields WHERE dataset_id = %s", (recipe["dataset_id"],)) } names = sorted(snap["fields"], key=lambda n: (n not in order, order.get(n, 0))) + # Descriptions and notes belong to the field, not to the version: read live, never from the snapshot. + about = { + r["name"]: r + for r in conn.execute( + "SELECT name, description, field_notes FROM meta.fields WHERE dataset_id = %s", (recipe["dataset_id"],) + ) + } for name in names: e = snap["fields"][name] dec = e["decision"] @@ -168,6 +180,8 @@ def to_json(conn, recipe: dict, user: dict, instance: str | None) -> dict: "role": ROLE[dec["action"]], "decision": decision, "stats": {"null_pct": e.get("null_pct"), "n_unique": e.get("n_unique")}, + "description": (about.get(name) or {}).get("description"), + "field_notes": (about.get(name) or {}).get("field_notes"), } ) return { @@ -237,6 +251,8 @@ def to_csv(doc: dict) -> str: d.get("encoding") or "", "" if params is None else _compact(params), _text_cell(d.get("notes")), + _text_cell(f.get("description")), + _text_cell(f.get("field_notes")), ] ) return "" + out.getvalue() @@ -262,6 +278,8 @@ class Parsed: undecided: list[str] = field(default_factory=list) unreadable: list[dict] = field(default_factory=list) warnings: list[str] = field(default_factory=list) + # Description and notes of every named field in the file that has some (also of an undecided one). + about: dict[str, dict] = field(default_factory=dict) def _is_num(v: Any) -> bool: @@ -351,6 +369,8 @@ def _parse_json(text: str, file: str) -> Parsed: raise HTTPException(422, _BAD_FILE) from None if _has_bad_chars(doc): raise HTTPException(422, _BAD_CHARS_MESSAGE) + if isinstance(doc, dict) and doc.get("format") == FORMAT_V1: + raise HTTPException(422, _V1_FILE) if ( not isinstance(doc, dict) or doc.get("format") != FORMAT @@ -391,6 +411,8 @@ def _parse_json(text: str, file: str) -> Parsed: p.warnings.append(f"Skipped {nameless} field{'s' if nameless > 1 else ''} without a name.") for name, e in entries: p.order.append(name) + if about := _about_of(e.get): + p.about[name] = about dec, typ = e.get("decision"), e.get("type") if not isinstance(dec, dict): p.unreadable.append({"name": name, "problem": "decision must be an object"}) @@ -431,7 +453,12 @@ def _parse_json(text: str, file: str) -> Parsed: stats = {"null_pct": st.get("null_pct"), "n_unique": st.get("n_unique")} else: bad_stats.append(name) - p.fields[name] = {"type": typ, "decision": decision, "stats": stats} + p.fields[name] = { + "type": typ, + "decision": decision, + "stats": stats, + "about": {k: p.about.get(name, {}).get(k) for k in ABOUT_KEYS}, + } if bad_stats: p.warnings.append(f"Ignored stats of: {', '.join(bad_stats)}.") listed = [ @@ -442,6 +469,16 @@ def _parse_json(text: str, file: str) -> Parsed: return p +def _about_of(get) -> dict: + """The description and notes a file entry carries: text only (anything else, and an empty text, is no text).""" + out = {} + for k in ABOUT_KEYS: + v = get(k) + if isinstance(v, str) and v not in ("", '""'): + out[k] = v + return out + + def _sniff(first_line: str) -> str: counts = {d: first_line.count(d) for d in (",", ";", "\t")} best = max(counts.values()) @@ -481,6 +518,11 @@ def _parse_csv(text: str, file: str, family: str | None) -> Parsed: comment = None if i < len(lines) and lines[i].strip().startswith(("#", '"#')): comment, i = lines[i], i + 1 + token = re.match(r"""[\s"]*#\s*([^\s",;]+)""", comment) + if token and token.group(1) == FORMAT_V1: + raise HTTPException(422, _V1_FILE) + if token and token.group(1).startswith("rainforest-baseline/") and token.group(1) != FORMAT: + raise HTTPException(422, _BAD_FILE) body = lines[i:] first = next((ln for ln in body if ln.strip()), "") delim = _sniff(first) @@ -500,7 +542,7 @@ def _parse_csv(text: str, file: str, family: str | None) -> Parsed: if fam is None: if family is None: raise HTTPException( - 422, "This CSV has lost its first line (# rainforest-baseline/1 …); choose the model type." + 422, "This CSV has lost its first line (# rainforest-baseline/2 …); choose the model type." ) fam = _check_family(family) warnings.append("The model type came from the dialog: the file's first line is missing.") @@ -550,13 +592,20 @@ def cell(r: list[str], c: str) -> str: for name, r in parsed_rows: p.order.append(name) decision, problem, undecided, pos = _csv_row(lambda c, r=r: cell(r, c)) + if about := _about_of(lambda k, r=r: cell(r, k)): + p.about[name] = about if undecided: p.undecided.append(name) elif problem: p.unreadable.append({"name": name, "problem": problem}) else: typ = cell(r, "type").strip() or None - p.fields[name] = {"type": typ, "decision": decision, "stats": None} + p.fields[name] = { + "type": typ, + "decision": decision, + "stats": None, + "about": {k: p.about.get(name, {}).get(k) for k in ABOUT_KEYS}, + } if decision["action"] == "target": positions[name] = pos numbered = sorted((n for n, k in positions.items() if k is not None), key=lambda n: positions[n]) @@ -591,12 +640,12 @@ def _csv_row(get) -> tuple[dict, str | None, bool, int | None]: ): if v := get(col).strip(): d[key] = v - for col in ("impute_value", "notes"): + for col, key in (("impute_value", "impute_value"), ("decision_notes", "notes")): v = get(col) if v == '""': - d[col] = "" + d[key] = "" elif v != "": - d[col] = v + d[key] = v for col, key in ( ("target_bins", "target_bins"), ("class_policy", "target_classes"), diff --git a/api/app/experiments.py b/api/app/experiments.py index 7d507b56..119a2b03 100644 --- a/api/app/experiments.py +++ b/api/app/experiments.py @@ -32,7 +32,8 @@ TEAM_LOCK = 7_311_208 COLS = """x.id, x.project_id, x.dataset_id, x.name, x.owner, x.recipe_id, x.assessor_id, x.source, x.notes, x.created_at, - x.archived_at, x.is_default, x.target_order, x.draft, x.rebase, xr.family AS recipe_family, xr.version AS recipe_version, + x.archived_at, x.is_default, x.target_order, x.draft, x.rebase, x.rerun_of_run_id, x.pending_derivation_id, + xr.family AS recipe_family, xr.version AS recipe_version, xr.phase AS recipe_phase, meta.baseline_name(xr.dataset_id, xr.family) AS recipe_name, (x.assessor_id IS NOT NULL AND coalesce(xr.phase, 'assessing') = 'assessing') AS private, (SELECT name FROM meta.users u WHERE u.id = coalesce(x.owner, x.assessor_id)) AS owner_name, @@ -43,7 +44,7 @@ FROM = "meta.experiments x LEFT JOIN meta.recipes xr ON xr.id = x.recipe_id" EXPORT_COLS = """e.id, e.project_id, e.dataset_id, e.status, e.progress, e.status_message, e.error, e.model_family, e.config, e.output_dir, e.n_rows, e.n_features, e.summary, e.created_at, e.updated_at, e.finished_at, - e.job_id, e.created_by, e.recipe_id, e.experiment_id, e.pull_id, e.draft, r.family AS recipe_family, + e.job_id, e.created_by, e.recipe_id, e.experiment_id, e.pull_id, e.draft, e.then_run, r.family AS recipe_family, r.version AS recipe_version, meta.baseline_name(r.dataset_id, r.family) AS recipe_name, x.name AS experiment_name""" EXPORT_SELECT = ( @@ -211,6 +212,12 @@ def can_adjust(x: dict, user: dict) -> bool: return adjust_denied(x, user) is None +def adjust_denied_reason(x: dict, user: dict) -> str | None: + """The payload's `adjust_denied`: why `user` can't adjust `x` (adjust_denied's text), or None when they can.""" + denied = adjust_denied(x, user) + return denied[1] if denied else None + + def require_editable(x: dict, user: dict) -> None: """Renaming, notes, archiving: the owner (the assessor, for a personal one) with the `write` role, or the project's owner.""" @@ -423,25 +430,56 @@ def adjustments(conn, x: dict, snapshots: dict[int, dict] | None = None) -> dict } +def waiting_for(conn, x: dict) -> int | None: + """The version a re-run draft waits for: N+1, while the pending derivation it was made for hasn't produced a + finalized version (not yet created, or still in review). None for any other experiment, and once the drops were + cancelled or the version is finalized.""" + if x.get("pending_derivation_id") is None: + return None + p = conn.execute( + """SELECT r.version AS parent_version, d.version AS done_version, d.phase AS done_phase, p.done_recipe_id, + p.cancelled_at + FROM meta.pending_derivations p JOIN meta.recipes r ON r.id = p.recipe_id + LEFT JOIN meta.recipes d ON d.id = p.done_recipe_id WHERE p.id = %s""", + (x["pending_derivation_id"],), + ).fetchone() + if p is None or p["cancelled_at"] is not None: + return None + if p["done_recipe_id"] is None: + return p["parent_version"] + 1 + return None if p["done_phase"] == "finalized" else p["done_version"] + + def rebase_available(conn, x: dict) -> dict | None: """For a draft, the finalized version it can rebase on: its family's latest finalized one, when that is the - draft's own version or newer. None otherwise.""" + draft's own version or newer. None otherwise, and while a re-run draft waits for its derived version (a manual + rebase onto the version before it would defeat the automatic one).""" if not x.get("draft"): return None + if waiting_for(conn, x) is not None: + return None latest = recipes.latest_finalized(conn, x["dataset_id"], x["recipe_family"]) if latest is None or latest["version"] < x["recipe_version"]: return None return {"recipe_id": latest["id"], "version": latest["version"]} -def rebase(conn, x: dict, recipe_id: int) -> dict: +def rebase(conn, x: dict, recipe_id: int, *, base: dict | None = None) -> dict: """Move a draft onto a finalized version (`recipe_id`, of its dataset and family, its own version or newer). Per field: where the new version decides it as the draft's base did (`seed_snapshot`; notes aside), the draft's copy stays as it is, adjustment and all; where it doesn't, the copy takes the new version's decision and settings, and an adjustment there is reverted and named. The draft's exports and runs keep their own snapshots. `x` is locked by - the caller. Answers {kept, reverted: [{field_id, name}]}.""" + the caller. `base` stands in for the draft's own base (its `seed_snapshot`) when given: the automatic rebase of a + re-run passes the base with its derived drops applied, so those drops aren't reverted. Answers + {kept, reverted: [{field_id, name}]}.""" if not x.get("draft"): raise HTTPException(409, f"Experiment {x['name']!r} isn't a draft; only a draft rebases.") + if (waiting := waiting_for(conn, x)) is not None: + raise HTTPException( + 409, + f"Experiment {x['name']!r} waits for v{waiting}, created when v{waiting - 1} is finalized; " + "it rebases onto it then.", + ) target = recipes.get(conn, recipe_id) if target is None or target["dataset_id"] != x["dataset_id"]: raise HTTPException(404, "Recipe not found") @@ -457,7 +495,7 @@ def rebase(conn, x: dict, recipe_id: int) -> dict: raise HTTPException( 409, f"{recipes.label(target)} is older than the version this draft was built from; rebase on it or newer." ) - seed = base_snapshot(conn, x) + seed = base if base is not None else base_snapshot(conn, x) new = recipes.snapshot_of(conn, target) adjusted = {f["name"] for f in adjustments(conn, x)["fields"]} moved = { @@ -645,12 +683,24 @@ def list_for(conn, dataset_id: int, user: dict, project: str | None = None) -> l "adjusted_count": adj["count"], "targets_changed": adj["targets_changed"], "can_adjust": can_adjust(r, user), + "adjust_denied": adjust_denied_reason(r, user), "rebase_available": None if blind(r) else rebase_available(conn, r), } ) return out +def exports_stay_draft(conn, x: dict) -> bool: + """Whether `x` was rebased but still has exports built from the version in progress: they stay draft (their + models can't be registered) until the experiment is exported again.""" + if x.get("draft") or not x.get("rebase"): + return False + return ( + conn.execute("SELECT 1 FROM meta.exports WHERE experiment_id = %s AND draft LIMIT 1", (x["id"],)).fetchone() + is not None + ) + + def shaped(conn, x: dict, user: dict) -> dict: """One experiment row (`get`'s) as the API shows it: `project`, `my_role`, `borrowed`, `source_shared`, `source_unshared` and `source_dataset`. A row already through `with_source` keeps its answers.""" @@ -661,6 +711,9 @@ def shaped(conn, x: dict, user: dict) -> dict: "project": access.project_refs([x["project_id"]])[x["project_id"]], "my_role": access.scope(user).role(x["project_id"]), "borrowed": False, + # A re-run draft waits for the version its drops will make; the exports of a rebased draft stay draft. + "rebase_waiting": None if (n := waiting_for(conn, x)) is None else {"version": n}, + "exports_stay_draft": exports_stay_draft(conn, x), } diff --git a/api/app/export_code/gather.py b/api/app/export_code/gather.py index 03084c7d..a525769b 100644 --- a/api/app/export_code/gather.py +++ b/api/app/export_code/gather.py @@ -75,7 +75,8 @@ def info_for(export_id: int) -> ExportInfo: from_db = None if manifest is not None else export_spec(conn, ds["id"], snap, e["model_family"]) tasks = target_tasks(conn, ds["id"], snap) fields_rows = conn.execute( - """SELECT name, derived FROM meta.fields WHERE dataset_id = %s ORDER BY ordinal""", (ds["id"],) + """SELECT name, derived, description FROM meta.fields WHERE dataset_id = %s ORDER BY ordinal""", + (ds["id"],), ).fetchall() config0 = e["config"] or {} plan = config0.get("plan") @@ -198,6 +199,7 @@ def info_for(export_id: int) -> ExportInfo: **fit_settings(config, manifest, specs), source=source, derived=[{"name": d["name"], "kind": d["kind"], "spec": d["spec"]} for d in used_derived], + descriptions={r["name"]: r["description"] for r in fields_rows if r["description"]}, notes={n: f["decision"].get("notes") for n, f in snap["fields"].items() if f["decision"].get("notes")}, adjusted={f["name"]: _change_text(f) for f in adjustments.get("detail") or []}, targets_changed=( diff --git a/api/app/export_code/render.py b/api/app/export_code/render.py index d3874980..3b474386 100644 --- a/api/app/export_code/render.py +++ b/api/app/export_code/render.py @@ -68,6 +68,7 @@ class ExportInfo: chunk_mb: int | None = None # the transform's batches' memory budget (None: chunk_rows as given) derived: list[dict[str, Any]] = field(default_factory=list) notes: dict[str, str] = field(default_factory=dict) # field -> the team's notes + descriptions: dict[str, str] = field(default_factory=dict) # field -> its description now adjusted: dict[str, str] = field(default_factory=dict) # field -> what the experiment changed targets_changed: str | None = None left_out: list[tuple[str, str, str]] = field(default_factory=list) # (field, type, why) @@ -349,6 +350,8 @@ def _decisions(info: ExportInfo) -> str: out.append(f"FAMILY = {info.family!r} # the model family the features are prepared for") calls = [] for spec in info.specs: + if info.descriptions.get(spec.name): + calls.append(_comment(info.descriptions[spec.name], indent=8, wrap=True)) for label, text in ( ("adjusted in this experiment", info.adjusted.get(spec.name)), ("notes", info.notes.get(spec.name)), @@ -372,6 +375,8 @@ def _decisions(info: ExportInfo) -> str: if info.targets_changed: tlines.append(_comment(f"targets changed in this experiment: {info.targets_changed}", indent=8, wrap=True)) for t, task in info.targets.items(): + if info.descriptions.get(t): + tlines.append(_comment(info.descriptions[t], indent=8, wrap=True)) for label, text in (("adjusted in this experiment", info.adjusted.get(t)), ("notes", info.notes.get(t))): if text: tlines.append(_comment(f"{label}: {text}", indent=8, wrap=True)) diff --git a/api/app/export_code/script.py b/api/app/export_code/script.py index 59826fa9..913b5188 100644 --- a/api/app/export_code/script.py +++ b/api/app/export_code/script.py @@ -28,6 +28,7 @@ import pyarrow.parquet as pq from ..derived import compute +from ..feature_frame import FeatureFrame from ..ingest import _sink from ..parquet_rows import read_rows from ..sample_library.build import build @@ -43,6 +44,7 @@ build_preprocessor, coercion_exprs, estimated_width, + native_levels, plan_order, pooled_input, prepare_targets, @@ -508,9 +510,16 @@ def prepare(path=None, check: bool = True) -> Prepared: size = min(size, rows_for_budget(FIT_MB, estimated_width(specs))) fit_idx = np.sort(np.random.default_rng(seed).choice(train_idx, size=size, replace=False)) cols = [f.name for f in specs] - pre = build_preprocessor(specs, seed) + native = native_levels( + raw, [f for f in specs if f.semantic_type == "categorical" and f.encoding == "native"], train_idx + ) + pre = build_preprocessor(specs, seed, native=native) pre.fit(to_pandas_frame(read_rows(raw, cols, fit_idx), specs, coercion_exprs(specs), consume=True)) feat_names = safe_feature_names(readable_feature_names(pre), reserved=[*RESERVED, *names]) + feature_source = _feature_sources(pre, feat_names) + features = FeatureFrame( + feat_names, {n: native[feature_source[n]][0] for n in feat_names if feature_source.get(n) in native} + ) coercers = coercion_exprs(specs) chunk_rows = CHUNK_ROWS if CHUNK_MB is None else min(CHUNK_ROWS, rows_for_budget(CHUNK_MB, len(feat_names))) blocks, kept_rows, offset = [], [], 0 @@ -537,7 +546,8 @@ def prepare(path=None, check: bool = True) -> Prepared: rows = np.concatenate(kept_rows) if kept_rows else np.zeros(0, dtype=np.int64) X = np.vstack(blocks) if blocks else np.zeros((0, len(feat_names)), dtype=np.float32) data: dict[str, Any] = {ROW_ID: rows.astype(np.int64), SPLIT_COL: _parts(plan)[split[rows]], **origin} - data.update({n: X[:, k] for k, n in enumerate(feat_names)}) + columns = features.from_codes(X) # a native feature as a category column, the rest as float32 + data.update({n: columns[n] if features.native else X[:, k] for k, n in enumerate(feat_names)}) data.update( {t: y_enc[t].to_numpy()[rows].astype(np.float64 if tasks[t] == "regression" else np.int32) for t in names} ) @@ -545,7 +555,7 @@ def prepare(path=None, check: bool = True) -> Prepared: pd.DataFrame(data), pre, feat_names, - _feature_sources(pre, feat_names), + feature_source, {t: classes[t] for t in names if tasks[t] != "regression"}, split_info, ) @@ -642,10 +652,9 @@ def input_summary(s: pl.Series) -> dict[str, Any]: def output_summary(df: pl.DataFrame, features: list[str]) -> dict[str, Any]: - feats = set(features) return { "rows": df.height, - "columns": {c: column_summary(df[c], c in feats or df[c].dtype.is_float()) for c in df.columns}, + "columns": {c: column_summary(df[c], df[c].dtype.is_float()) for c in df.columns}, } @@ -704,6 +713,9 @@ def compare_files(data: pd.DataFrame, path: str | Path, rtol: float = COMPARE_RT a, b = mine[c], theirs[c] if a.dtype != b.dtype: diffs.append(f"{c}: {a.dtype} here, {b.dtype} there") + elif a.dtype == pl.Categorical: # a native feature: by its text, not the codes of each side's own dictionary + if not a.cast(pl.String).equals(b.cast(pl.String)): + diffs.append(f"{c}: {int((a.cast(pl.String) != b.cast(pl.String)).sum()):,} value(s) differ") elif a.dtype.is_float(): tol = rtol if a.dtype == pl.Float32 else INPUT_RTOL # features are float32; number targets float64 x, y = a.to_numpy(), b.to_numpy() @@ -735,7 +747,14 @@ def write_model_ready(data: pd.DataFrame, path: str | Path, regression: set[str] else pa.float32() for c in data.columns[2:] ] - arrays = [pa.array(data[c].to_numpy(), t) for c, t in zip(data.columns, types)] + dictionary = pa.dictionary(pa.int32(), pa.string()) # a native feature, as the app writes it + arrays = [] + for k, (c, t) in enumerate(zip(data.columns, types)): + if isinstance(data[c].dtype, pd.CategoricalDtype): + arrays.append(pa.Array.from_pandas(data[c]).cast(dictionary)) + types[k] = dictionary + else: + arrays.append(pa.array(data[c].to_numpy(), t)) pq.write_table( pa.Table.from_arrays(arrays, schema=pa.schema(list(zip(data.columns, types)))), path, compression="zstd" ) diff --git a/api/app/feature_frame.py b/api/app/feature_frame.py new file mode 100644 index 00000000..b1ad801d --- /dev/null +++ b/api/app/feature_frame.py @@ -0,0 +1,70 @@ +"""FeatureFrame: the one place a native categorical feature becomes a category column. + +The preprocessor writes a native categorical as a float code (the level's index, NaN for missing or unseen) so +`transform_into` keeps its shape. The export stores those columns dictionary-encoded; training, scoring, the bundle +and the code script turn codes (or text) back into `pd.Categorical` here. With no native feature every method is +the float path as it was. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa + + +@dataclass +class FeatureFrame: + names: list[str] + levels: dict[str, list[str]] = field(default_factory=dict) # native feature -> its categories, in code order + + @classmethod + def from_manifest(cls, m: dict) -> "FeatureFrame": + return cls(list(m["feature_names"]), {k: list(v) for k, v in (m.get("category_levels") or {}).items()}) + + @property + def native(self) -> bool: + return bool(self.levels) + + def from_codes(self, X: np.ndarray) -> np.ndarray | pd.DataFrame: + """The preprocessor's float codes (columns in `names` order) as model input.""" + if not self.native: + return X + cols: dict[str, object] = {} + for k, n in enumerate(self.names): + col = X[:, k] + if n in self.levels: + codes = np.where(np.isnan(col), -1, col).astype(np.int64) + cols[n] = pd.Categorical.from_codes(codes, categories=self.levels[n]) + else: + cols[n] = np.asarray(col, dtype=np.float32) + return pd.DataFrame(cols) + + def from_polars(self, df: pl.DataFrame) -> np.ndarray | pd.DataFrame: + """A model-ready frame (as read from model_ready.parquet) as model input. A native column is mapped by its + text, never by polars' own category codes.""" + if not self.native: + return df.select(self.names).to_numpy().astype(np.float32) + cols: dict[str, object] = {} + for n in self.names: + if n in self.levels: + cols[n] = pd.Categorical(df[n].cast(pl.String).to_numpy(), categories=self.levels[n]) + else: + cols[n] = df[n].to_numpy().astype(np.float32) + return pd.DataFrame(cols) + + def arrow_field(self, name: str) -> pa.Field: + if name in self.levels: + return pa.field(name, pa.dictionary(pa.int32(), pa.string())) + return pa.field(name, pa.float32()) + + def arrow_column(self, name: str, values: np.ndarray) -> pa.Array: + """One feature's chunk for the export file: float32, or dictionary-encoded codes (null where NaN).""" + if name not in self.levels: + return pa.array(values) + missing = np.isnan(values) + codes = pa.array(np.where(missing, 0, values).astype(np.int32), pa.int32(), mask=missing) + return pa.DictionaryArray.from_arrays(codes, pa.array(self.levels[name], pa.string())) diff --git a/api/app/field_about.py b/api/app/field_about.py new file mode 100644 index 00000000..532c0a82 --- /dev/null +++ b/api/app/field_about.py @@ -0,0 +1,216 @@ +"""A field's description (one paragraph) and notes (longer), who wrote them, and the history of both. The text lives +on `meta.fields`, so a refresh that keeps the field keeps it; the history is keyed by dataset and field name, so it +outlives a field a refresh removes and returns for one of that name that comes back.""" + +from __future__ import annotations + +from fastapi import HTTPException + +MAX_DESCRIPTION = 500 +MAX_NOTES = 10_000 +KEYS = ("description", "field_notes") + + +# Unicode's own line and paragraph breaks (and NEL): a description shows them as a space, keeping it one paragraph. +PARAGRAPH_BREAKS = str.maketrans({"\u2028": " ", "\u2029": " ", "\x85": " "}) + + +def _clean(key: str, value: str | None) -> str | None: + """The text as stored: stripped, empty as None; 422 when it breaks a limit or holds a NUL (Postgres text can't).""" + if value and "\x00" in value: + raise HTTPException(422, "The text holds a NUL character; remove it and save again.") + if key == "description" and value: + value = value.translate(PARAGRAPH_BREAKS) + text = (value or "").strip() or None + if text is None: + return None + if key == "description": + if "\n" in text or "\r" in text: + raise HTTPException(422, "A description is one paragraph, without line breaks.") + if len(text) > MAX_DESCRIPTION: + raise HTTPException(422, f"A description is at most {MAX_DESCRIPTION} characters.") + elif len(text) > MAX_NOTES: + raise HTTPException(422, "Notes are at most 10,000 characters.") + return text + + +def user_ref(conn, user_id: int | None) -> dict | None: + """Who wrote a field's text, as `{id, name}`.""" + if user_id is None: + return None + u = conn.execute("SELECT id, name FROM meta.users WHERE id = %s", (user_id,)).fetchone() + return {"id": u["id"], "name": u["name"]} if u else None + + +def _about(conn, row: dict) -> dict: + return { + "description": row["description"], + "field_notes": row["field_notes"], + "about_by": user_ref(conn, row["about_by"]), + "about_at": row["about_at"], + } + + +def _write(conn, row: dict, user_id: int, new: dict) -> bool: + """Store `new` (already clean, only the keys to change) on the locked field `row` with one history entry; False + when nothing differs.""" + previous = {k: row[k] for k in KEYS} + current = {k: new.get(k, previous[k]) for k in KEYS} + if current == previous: + return False + conn.execute( + "UPDATE meta.fields SET description = %s, field_notes = %s, about_by = %s, about_at = now() WHERE id = %s", + (current["description"], current["field_notes"], user_id, row["id"]), + ) + from .db import jsonb + + conn.execute( + """INSERT INTO meta.field_about_history (dataset_id, field_name, user_id, previous, current) + VALUES (%s, %s, %s, %s, %s)""", + (row["dataset_id"], row["name"], user_id, jsonb(previous), jsonb(current)), + ) + return True + + +def _locked(conn, field_id: int) -> dict: + row = conn.execute( + "SELECT id, dataset_id, name, description, field_notes, about_by, about_at FROM meta.fields WHERE id = %s " + "FOR UPDATE", + (field_id,), + ).fetchone() + if row is None: + raise HTTPException(404, "Field not found") + return row + + +def update(conn, field: dict, user_id: int, changes: dict) -> dict: + """Set the description and/or notes (the keys of `changes`); returns the field's `FieldAbout`.""" + new = {k: _clean(k, changes[k]) for k in KEYS if k in changes} + row = _locked(conn, field["id"]) + _write(conn, row, user_id, new) + return _about(conn, _locked(conn, field["id"])) + + +def history(conn, dataset_id: int, field_name: str) -> list[dict]: + """The about entries of a field, newest first, as the field's history lists them.""" + return conn.execute( + """SELECT 'about' AS kind, h.at AS changed_at, h.user_id, u.name AS user_name, h.previous, h.current + FROM meta.field_about_history h LEFT JOIN meta.users u ON u.id = h.user_id + WHERE h.dataset_id = %s AND h.field_name = %s ORDER BY h.at DESC, h.id DESC LIMIT 50""", + (dataset_id, field_name), + ).fetchall() + + +def _skip(name: str, reason: str) -> dict: + return {"name": name, "reason": reason} + + +def plan_fill(conn, dataset_id: int, texts: dict[str, dict]) -> tuple[list[tuple[int, dict]], dict]: + """What fill_missing would do, with no writes: ([(field id, clean texts to store)], {filled, skipped}). `filled` + lists names; each `skipped` entry is {name, reason}: unknown_field, already_set (the field has the text), + too_long, line_break (a description), nul_character (Postgres text can't hold one) or empty (nothing to give).""" + updates: list[tuple[int, dict]] = [] + filled: list[str] = [] + skipped: list[dict] = [] + for name, given in texts.items(): + row = conn.execute( + "SELECT id, description, field_notes FROM meta.fields WHERE dataset_id = %s AND name = %s", + (dataset_id, name), + ).fetchone() + if row is None: + skipped.append(_skip(name, "unknown_field")) + continue + new: dict = {} + why: list[str] = [] + for k in KEYS: + if k not in given: + continue + if row[k] is not None: + why.append("already_set") + continue + try: + text = _clean(k, given[k]) + except HTTPException as exc: + why.append( + "nul_character" + if "NUL" in exc.detail + else "line_break" + if "line breaks" in exc.detail + else "too_long" + ) + continue + if text is not None: + new[k] = text + if new: + updates.append((row["id"], new)) + filled.append(name) + else: + skipped.append(_skip(name, why[0] if why else "empty")) + return updates, {"filled": filled, "skipped": skipped} + + +def fill_missing(conn, dataset_id: int, texts: dict[str, dict], user_id: int) -> dict: + """Give each named field the description and notes in `texts[name]` it has none of, under the row locks. Plans + with plan_fill, then writes what is still empty once the row is locked.""" + updates, report = plan_fill(conn, dataset_id, texts) + for field_id, new in updates: + row = _locked(conn, field_id) + new = {k: v for k, v in new.items() if row[k] is None} + if not new or not _write(conn, row, user_id, new): + name = row["name"] + report["filled"].remove(name) + report["skipped"].append(_skip(name, "already_set")) + return report + + +def copy_from_composition(conn, dataset_id: int) -> int: + """A promoted composition's dataset starts with its sources' text: each field that maps to exactly one source + column, of a dataset source, and has no text takes that column's description and notes. Returns the fields filled.""" + ds = conn.execute("SELECT composition_id, created_by FROM meta.datasets WHERE id = %s", (dataset_id,)).fetchone() + if ds is None or ds["composition_id"] is None: + return 0 + comp = conn.execute("SELECT spec FROM meta.compositions WHERE id = %s", (ds["composition_id"],)).fetchone() + if comp is None: + return 0 + sources = (comp["spec"] or {}).get("sources") or [] + names = [ + r["name"] + for r in conn.execute( + "SELECT name FROM meta.fields WHERE dataset_id = %s AND description IS NULL AND field_notes IS NULL", + (dataset_id,), + ).fetchall() + ] + # The columns each source offers under the composition's names: (kind, id, its own column name). + offered: dict[str, list[tuple[int | None, str]]] = {} + for s in sources: + mapping = s.get("mapping") or {} + if s.get("dataset_id") is not None: + own = [ + r["name"] + for r in conn.execute( + "SELECT name FROM meta.fields WHERE dataset_id = %s", (s["dataset_id"],) + ).fetchall() + ] + else: + own = None # a composition source: its columns aren't known here, so it can only count as a rival + if own is None: + sub = conn.execute( + "SELECT shape FROM meta.compositions WHERE id = %s", (s.get("composition_id"),) + ).fetchone() + own = list((sub and sub["shape"]) or {}) + for col in own: + offered.setdefault(mapping.get(col, col), []).append((s.get("dataset_id"), col)) + texts: dict[str, dict] = {} + for name in names: + hits = offered.get(name) or [] + if len(hits) != 1 or hits[0][0] is None: + continue + src = conn.execute( + "SELECT description, field_notes FROM meta.fields WHERE dataset_id = %s AND name = %s", + (hits[0][0], hits[0][1]), + ).fetchone() + if src and (src["description"] or src["field_notes"]): + texts[name] = {k: src[k] for k in KEYS if src[k]} + if not texts: + return 0 + return len(fill_missing(conn, dataset_id, texts, ds["created_by"])["filled"]) diff --git a/api/app/gpu.py b/api/app/gpu.py index e52dae04..2b264e8d 100644 --- a/api/app/gpu.py +++ b/api/app/gpu.py @@ -9,6 +9,7 @@ import json import logging +import os import shutil import subprocess import sys @@ -23,7 +24,8 @@ _probed = False -def _nvidia_smi() -> dict[str, Any] | None: +def _smi_lines() -> list[str] | None: + """The `nvidia-smi` rows as text lines, None without the tool or when it fails.""" exe = shutil.which("nvidia-smi") if exe is None: return None @@ -38,15 +40,63 @@ def _nvidia_smi() -> dict[str, Any] | None: except (OSError, subprocess.SubprocessError) as exc: log.info("nvidia-smi present but failed: %s", exc) return None - line = next((ln for ln in out.splitlines() if ln.strip()), None) - if line is None: - return None + return [ln for ln in out.splitlines() if ln.strip()] + + +def _row(line: str) -> dict[str, Any]: name, mem, driver = (p.strip() for p in (line.split(",") + ["", ""])[:3]) try: memory_mb = int(float(mem)) except ValueError: memory_mb = None - return {"name": name, "memory_mb": memory_mb, "driver": driver or None, "count": len(out.strip().splitlines())} + return {"name": name, "memory_mb": memory_mb, "driver": driver or None} + + +def nvidia_rows() -> list[dict[str, Any]]: + """Every `nvidia-smi` row: name, memory_mb, driver (empty without a driver).""" + return [_row(ln) for ln in _smi_lines() or []] + + +def _nvidia_smi() -> dict[str, Any] | None: + lines = _smi_lines() + if not lines: + return None + return {**_row(lines[0]), "count": len(lines)} + + +def hardware(device_asked: str, device_used: str, worker: str | None) -> dict[str, Any]: + """The hardware record of a run: what was asked, what XGBoost used, and the GPU (only when it used one).""" + from . import jobs + + on_gpu = device_used.startswith("cuda") + slot_env = os.environ.get("CUDA_VISIBLE_DEVICES", "").strip() + slot = int(slot_env) if slot_env.isdigit() else None + row = None + if on_gpu: + rows = nvidia_rows() + row = rows[slot if slot is not None and slot < len(rows) else 0] if rows else None + cuda = version = None + try: + import xgboost + + version = xgboost.__version__ + raw = xgboost.build_info().get("CUDA_VERSION") + cuda = ".".join(str(n) for n in raw) if raw else None + except Exception: # noqa: BLE001 - not installed + pass + return { + "device_asked": device_asked, + "device_used": device_used, + "gpu": row["name"] if row else None, + "memory_mb": row["memory_mb"] if row else None, + "driver": row["driver"] if row else None, + "cuda": cuda, + "xgboost": version, + "xgboost_cuda": xgboost_cuda_build(), + "worker": worker, + "slot": slot if on_gpu else None, + "threads": jobs.cpu_threads(), + } def xgboost_cuda_build() -> bool: @@ -98,7 +148,11 @@ def _probe_xgboost() -> str | None: tail = (r.stderr.strip().splitlines() or ["no output"])[-1] return f"XGBoost CUDA test fit failed: {tail}" device = r.stdout.strip().splitlines()[-1] if r.stdout.strip() else "unknown" - return None if device.startswith("cuda") else f"XGBoost fell back to {device} (CUDA 13 builds need driver 580+)" + return ( + None + if device.startswith("cuda") + else f"XGBoost fell back to {device}: use the cu12 worker image (worker.gpu.cuda: cu12) for drivers below 580, or a newer driver" + ) def detect(refresh: bool = False) -> dict[str, Any] | None: diff --git a/api/app/jobkinds.py b/api/app/jobkinds.py index 883c5747..74ce56d6 100644 --- a/api/app/jobkinds.py +++ b/api/app/jobkinds.py @@ -43,6 +43,7 @@ "model_versions", "evaluations", "cleanups", + "run_batches", } ) @@ -136,9 +137,16 @@ def enqueue( "model_versions": "SELECT m.project_id FROM meta.model_versions v JOIN meta.registered_models m ON m.id = v.model_id WHERE v.id = %s", "evaluations": """SELECT m.project_id FROM meta.evaluations e JOIN meta.model_versions v ON v.id = e.version_id JOIN meta.registered_models m ON m.id = v.model_id WHERE e.id = %s""", + "run_batches": """SELECT x.project_id FROM meta.run_batches b JOIN meta.experiments x ON x.id = b.experiment_id + WHERE b.id = %s""", } # fmt: skip +def needs_for(device: str) -> dict | None: + """The queue slot a run on `device` waits in: a `cuda` job only ever runs in the gpu slot.""" + return {"slot": "gpu"} if device == "cuda" else None + + def project_of(conn, ref_table: str, ref_id: int) -> int: """The project of the domain row `ref_table` / `ref_id`, for `enqueue`'s `project_id` (an evaluation's is its model's, a pull's its dataset's).""" @@ -181,6 +189,9 @@ def _within(conn) -> Iterator: JOIN meta.model_versions v ON v.id = e.version_id JOIN meta.registered_models m ON m.id = v.model_id WHERE e.id = ANY(%s)""", "cleanups": "SELECT id, dry_run FROM meta.cleanups WHERE id = ANY(%s)", + "run_batches": """SELECT id, experiment_id, export_id, name, spec ->> 'model' AS model, + (SELECT dataset_id FROM meta.exports e WHERE e.id = export_id) AS dataset_id + FROM meta.run_batches WHERE id = ANY(%s)""", } # table -> {id: row}, while `prefetched` is open; a table that isn't here is read row by row. _prefetched: ContextVar[dict[str, dict[int, dict]] | None] = ContextVar("jobkinds_prefetched", default=None) @@ -376,6 +387,8 @@ def _export_label(row: dict) -> tuple[str, str | None]: def _abort_export(job: Job, status: str, error: str | None, conn=None) -> None: """Failed, as the export's own failure path writes it (an export has no cancelled state).""" + from . import derive + with _within(conn) as conn: conn.execute( """UPDATE meta.exports SET status = 'failed', error = %s, status_message = %s, finished_at = now(), @@ -386,14 +399,22 @@ def _abort_export(job: Job, status: str, error: str | None, conn=None) -> None: job.ref_id, ), ) + # A re-run waiting for this export can't start. + derive.export_failed(job.ref_id, error if status == "failed" and error else STOPPED, conn) @register("export.build", ref_table="exports", describe=_export_label, abort=_abort_export) def _export(job: Job) -> None: + from . import derive from .routers.exports import _run _run(job.ref_id, job=job) - _failed_unless_ok("exports", job.ref_id) + try: + _failed_unless_ok("exports", job.ref_id) + except jobs.Failed as exc: + derive.export_failed(job.ref_id, str(exc)) + raise + derive.export_done(job.ref_id) def _run_label(row: dict) -> tuple[str, str | None]: @@ -444,6 +465,33 @@ def _train(job: Job) -> None: _failed_unless_ok("model_runs", job.ref_id) +# ---- tuning studies ------------------------------------------------------------------------------------------------- + + +def _study_label(row: dict) -> tuple[str, str | None]: + from .modeling.registry import MODELS + + b = _row("run_batches", row["ref_id"]) + label = MODELS[b["model"]].label if b["model"] in MODELS else b["model"] + return f"Tuning study · {label}", at( + row, f"/d/{b['dataset_id']}/experiments/{b['experiment_id']}/batches/{b['id']}" + ) + + +def _abort_study(job: Job, status: str, error: str | None, conn=None) -> None: + from .tuning.study import abort + + with _within(conn) as conn: + abort(job, status, error, conn) + + +@register("run.tune", ref_table="run_batches", preemptible=True, describe=_study_label, abort=_abort_study) +def _tune(job: Job) -> None: + from .tuning.study import run + + run(job.ref_id, job) + + # ---- compositions --------------------------------------------------------------------------------------------------- diff --git a/api/app/jobs.py b/api/app/jobs.py index 77f5e0bd..e7f820e2 100644 --- a/api/app/jobs.py +++ b/api/app/jobs.py @@ -181,6 +181,47 @@ def _lock(conn: psycopg.Connection, job_id: int) -> dict: return row +def locked(conn: psycopg.Connection, job_id: int) -> dict: + """The job's row, locked until the transaction ends.""" + return _lock(conn, job_id) + + +def queued_slots(conn: psycopg.Connection, job_ids: list[int]) -> dict[int, str | None]: + """The queue slot of each of `job_ids` that is queued (one query).""" + return { + j["id"]: j["slot"] + for j in conn.execute( + "SELECT id, needs ->> 'slot' AS slot FROM meta.jobs WHERE id = ANY(%s) AND status = 'queued'", (job_ids,) + ) + } + + +def move_to_cpu(conn: psycopg.Connection, row: dict, by: int | None) -> dict: + """Queue a GPU `run.train` or `run.tune` job on the CPU instead: its slot, its request's device and the run's (a + study's batch's) device change; nothing else does.""" + message = "Queued (moved to the CPU)" + if row["kind"] == "run.tune": + path = None # a study's job args hold no spec: the batch's spec is the request + conn.execute( + "UPDATE meta.run_batches SET spec = jsonb_set(spec, '{device}', '\"cpu\"') WHERE id = %s", (row["ref_id"],) + ) + else: + path = "{request,device}" + conn.execute( + """UPDATE meta.model_runs SET config = jsonb_set(config, '{device}', '"cpu"'), status_message = %s, + updated_at = now() WHERE id = %s""", + (message, row["ref_id"]), + ) + done = conn.execute( + """UPDATE meta.jobs SET needs = '{"slot":"cpu"}', + args = CASE WHEN %s::text[] IS NULL THEN args ELSE jsonb_set(args, %s::text[], '"cpu"') END, + status_message = %s WHERE id = %s RETURNING *""", + (path, path, message, row["id"]), + ).fetchone() + event(conn, row["id"], "moved_to_cpu", by=by) + return {**row, **done} + + def _queued(conn: psycopg.Connection, job_id: int, *also: int) -> dict: """The job's row, locked, after the order lock(s) of its priority (and of `also`, a priority it is moving to). The advisory locks come first, then the row, the same order every reorder takes them in, so none waits on a lock @@ -412,6 +453,9 @@ def allowed(row: dict) -> dict: "demote": status == "queued", "move": status == "queued", "priority": status == "queued", + "move_to_cpu": status == "queued" + and row["kind"] in ("run.train", "run.tune") + and (row.get("needs") or {}).get("slot") == "gpu", } @@ -424,7 +468,7 @@ def _can(row: dict, user: dict, first_other: int | None, role: str | None) -> di out = dict.fromkeys(can, False) starter = row["created_by"] == user["id"] if starter or role == "owner": - for action in ("stop", "pause", "resume"): + for action in ("stop", "pause", "resume", "move_to_cpu"): out[action] = can[action] if starter: out["demote"] = can["demote"] @@ -517,6 +561,23 @@ def _describe(row: dict) -> tuple[str, str | None]: ) # fmt: skip +def _waits_for_gpu(row: dict) -> bool: + return row["status"] == "queued" and (row.get("needs") or {}).get("slot") == "gpu" + + +GPU_WAITING = "Waiting for a GPU slot; no live worker has a GPU" + + +def gpu_waiting(conn: psycopg.Connection) -> str | None: + """The waiting copy while no live worker has a GPU slot (a `gpu` job queues until one does), else None.""" + live = conn.execute( + """SELECT 1 FROM meta.workers + WHERE coalesce((slots ->> 'gpu')::int, 0) > 0 AND heartbeat_at > now() - make_interval(secs => %s) LIMIT 1""", + (stale_after_s(),), + ).fetchone() + return None if live else GPU_WAITING + + def public_many(conn: psycopg.Connection, rows: list[dict], *, user: dict) -> list[dict]: """Jobs as the API shows them (docs/project/phase-0-foundations/design.md, 5.1), in a constant number of queries for their starters' names and a member's promote checks. Each kind's `describe` reads its own domain row; @@ -528,6 +589,7 @@ def public_many(conn: psycopg.Connection, rows: list[dict], *, user: dict) -> li else {} ) others = _first_others(conn, rows, user) + waiting = gpu_waiting(conn) if any(_waits_for_gpu(r) for r in rows) else None refs = access.project_refs({r["project_id"] for r in rows if r.get("project_id") is not None}) sc = access.scope(user) out = [] @@ -536,6 +598,7 @@ def public_many(conn: psycopg.Connection, rows: list[dict], *, user: dict) -> li if row["kind"] in PRIVATE_LOGS and row["created_by"] != user["id"] and job.get("error"): job["error"] = job["error"].splitlines()[0] # the tail is the starter's own log job["created_by_name"] = names.get(row["created_by"]) + job["waiting"] = waiting if _waits_for_gpu(row) else None project = refs.get(row.get("project_id")) job["project"] = project row["project_slug"] = project["slug"] if project else None @@ -559,22 +622,24 @@ def workers(conn: psycopg.Connection) -> list[dict]: """Every registered worker, live ones first, each with `alive` from its heartbeat's age (spec 5.3).""" return conn.execute( """SELECT id, hostname, started_at, heartbeat_at, heartbeat_at > now() - make_interval(secs => %s) AS alive, - slots, busy, gpu + slots, busy, gpu, threads FROM meta.workers ORDER BY alive DESC, started_at DESC, id""", (stale_after_s(),), ).fetchall() def live_gpu(conn: psycopg.Connection) -> dict | None: - """The GPU a live worker detected (`gpu.detect()`'s shape), one XGBoost can use first; None when no live worker - sees one. The API detects nothing itself: the GPU belongs to the worker.""" + """The GPU a live worker detected (`gpu.detect()`'s shape plus `slots`, its GPU slot count, and `worker`, its id): + a worker with GPU slots first, then one XGBoost can use, then the newest. None when no live worker sees one. The + API detects nothing itself: the GPU belongs to the worker.""" row = conn.execute( - """SELECT gpu FROM meta.workers + """SELECT id, gpu, coalesce((slots ->> 'gpu')::int, 0) AS gpu_slots FROM meta.workers WHERE gpu IS NOT NULL AND heartbeat_at > now() - make_interval(secs => %s) - ORDER BY coalesce((gpu ->> 'available')::boolean, false) DESC, started_at DESC, id LIMIT 1""", + ORDER BY coalesce((slots ->> 'gpu')::int, 0) > 0 DESC, coalesce((gpu ->> 'available')::boolean, false) DESC, + started_at DESC, id LIMIT 1""", (stale_after_s(),), ).fetchone() - return row["gpu"] if row else None + return {**row["gpu"], "slots": row["gpu_slots"], "worker": row["id"]} if row else None def cpu_threads() -> int: diff --git a/api/app/lineage.py b/api/app/lineage.py index 54e63e49..6ba1611f 100644 --- a/api/app/lineage.py +++ b/api/app/lineage.py @@ -13,7 +13,7 @@ from typing import Callable -from . import access, experiments +from . import access, experiments, recipes from .registry import export_summary EDGE_CAP = 50 @@ -30,6 +30,7 @@ "batch", "plan", "connection", + "baseline", ) # A status word to the panel's states. @@ -48,7 +49,7 @@ # The lineage kinds' loaders in `access` (a plan is a split plan there). -ACCESS_KIND = {"plan": "split_plan"} +ACCESS_KIND = {"plan": "split_plan", "baseline": "recipe"} UNSHARED = "Source no longer shared" HIDDEN = {"hidden": True} # a loader's answer for a thing the caller may not see for a reason other than its project @@ -304,6 +305,22 @@ def _load_connection(g: Graph, id_: int) -> dict | None: } +def _load_baseline(g: Graph, id_: int) -> dict | None: + r = g.conn.execute( + "SELECT dataset_id, family, version, meta.baseline_name(dataset_id, family) AS name, project_id " + "FROM meta.recipes WHERE id = %s", + (id_,), + ).fetchone() + return r and _node( + g, + "baseline", + id_, + recipes.short_label(r["family"], r["version"], r["name"]), + f"/d/{r['dataset_id']}/baselines", + r["project_id"], + ) + + LOADERS: dict[str, Callable[[Graph, int], dict | None]] = { "dataset": _load_dataset, "composition": _load_composition, @@ -316,6 +333,7 @@ def _load_connection(g: Graph, id_: int) -> dict | None: "batch": _load_batch, "plan": _load_plan, "connection": _load_connection, + "baseline": _load_baseline, } @@ -481,7 +499,17 @@ def _up_experiment(g: Graph, root: str, id_: int) -> list[str]: row = g.conn.execute("SELECT dataset_id FROM meta.experiments WHERE id = %s", (id_,)).fetchone() k = g.add("dataset", row["dataset_id"]) if row else None g.edge(k, root, "experiment on") - return [k] if k else [] + out = [k] if k else [] + out += g.neighbours( + root, + g.query( + "SELECT rerun_of_run_id AS id FROM meta.experiments WHERE id = %s AND rerun_of_run_id IS NOT NULL", (id_,) + ), + "run", + "re-run of", + up=True, + ) + return out def _down_experiment(g: Graph, root: str, id_: int) -> list[str]: @@ -574,7 +602,7 @@ def _up_run(g: Graph, root: str, id_: int) -> list[str]: def _down_run(g: Graph, root: str, id_: int) -> list[str]: - return g.neighbours( + out = g.neighbours( root, g.query( """SELECT id FROM meta.model_versions v @@ -586,6 +614,21 @@ def _down_run(g: Graph, root: str, id_: int) -> list[str]: "registered as", up=False, ) + out += g.neighbours( + root, + g.query("SELECT id FROM meta.recipes WHERE derived_from_run_id = %s ORDER BY id DESC", (id_,)), + "baseline", + "dropped fields from", + up=False, + ) + out += g.neighbours( + root, + g.query("SELECT id FROM meta.experiments WHERE rerun_of_run_id = %s ORDER BY id DESC", (id_,)), + "experiment", + "re-run of", + up=False, + ) + return out def _up_batch(g: Graph, root: str, id_: int) -> list[str]: @@ -728,6 +771,29 @@ def _down_evaluation(g: Graph, root: str, id_: int) -> list[str]: return [k] if k else [] +def _up_baseline(g: Graph, root: str, id_: int) -> list[str]: + return g.neighbours( + root, + g.query( + "SELECT derived_from_run_id AS id FROM meta.recipes WHERE id = %s AND derived_from_run_id IS NOT NULL", + (id_,), + ), + "run", + "dropped fields from", + up=True, + ) + + +def _down_baseline(g: Graph, root: str, id_: int) -> list[str]: + return g.neighbours( + root, + g.query("SELECT id FROM meta.experiments WHERE recipe_id = %s ORDER BY id DESC", (id_,)), + "experiment", + "experiment on", + up=False, + ) + + def _up_connection(g: Graph, root: str, id_: int) -> list[str]: return [] @@ -754,6 +820,7 @@ def _down_connection(g: Graph, root: str, id_: int) -> list[str]: "batch": _up_batch, "plan": _up_plan, "connection": _up_connection, + "baseline": _up_baseline, } DOWN = { "dataset": _down_dataset, @@ -767,6 +834,7 @@ def _down_connection(g: Graph, root: str, id_: int) -> list[str]: "batch": _down_batch, "plan": _down_plan, "connection": _down_connection, + "baseline": _down_baseline, } diff --git a/api/app/main.py b/api/app/main.py index a44f3eb1..68e454a1 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -19,6 +19,7 @@ from .routers import batches as batches_router from .routers import compositions as compositions_router from .routers import connections as connections_router +from .routers import derive as derive_router from .routers import derived as derived_router from .routers import endpoints as endpoints_router from .routers import experiments as experiments_router @@ -122,6 +123,7 @@ async def validation_error(_: Request, exc: RequestValidationError) -> JSONRespo app.include_router(team.router) app.include_router(recipes.router) app.include_router(baselines.router) +app.include_router(derive_router.router) app.include_router(derived_router.router) app.include_router(experiments_router.router) app.include_router(jobs_router.router) diff --git a/api/app/migrations/043_modeling.sql b/api/app/migrations/043_modeling.sql new file mode 100644 index 00000000..25abc62d --- /dev/null +++ b/api/app/migrations/043_modeling.sql @@ -0,0 +1,119 @@ +-- Migration 043 (phase 9): modeling upgrades. Field descriptions and notes (with their history), derived baseline +-- versions and pending derivations, re-runs, tuning studies (a batch of kind 'tune' with one row per trial), and the +-- worker's job-thread count. No backward compatibility. Replay-safe. + +-- 1. Field descriptions and notes. +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS description TEXT; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS field_notes TEXT; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS about_by BIGINT REFERENCES meta.users ON DELETE SET NULL; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS about_at TIMESTAMPTZ; +ALTER TABLE meta.fields DROP CONSTRAINT IF EXISTS fields_description_check; +ALTER TABLE meta.fields ADD CONSTRAINT fields_description_check CHECK (char_length(description) <= 500); +ALTER TABLE meta.fields DROP CONSTRAINT IF EXISTS fields_field_notes_check; +ALTER TABLE meta.fields ADD CONSTRAINT fields_field_notes_check CHECK (char_length(field_notes) <= 10000); + +CREATE TABLE IF NOT EXISTS meta.field_about_history ( + id BIGSERIAL PRIMARY KEY, + dataset_id BIGINT NOT NULL REFERENCES meta.datasets ON DELETE CASCADE, + field_name TEXT NOT NULL, + user_id BIGINT REFERENCES meta.users ON DELETE SET NULL, + at TIMESTAMPTZ NOT NULL DEFAULT now(), + previous JSONB NOT NULL, + current JSONB NOT NULL +); +CREATE INDEX IF NOT EXISTS field_about_history_idx ON meta.field_about_history (dataset_id, field_name, at DESC); + +-- 2. Runs, versions and re-runs. A derived baseline version remembers the run it came from; a re-run's experiment +-- remembers its run and, while its baseline version isn't made yet, the pending derivation it waits for. +ALTER TABLE meta.model_runs ADD COLUMN IF NOT EXISTS trial_number INT; +ALTER TABLE meta.recipes ADD COLUMN IF NOT EXISTS derived_from_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; + +CREATE TABLE IF NOT EXISTS meta.pending_derivations ( + id BIGSERIAL PRIMARY KEY, + recipe_id BIGINT NOT NULL REFERENCES meta.recipes ON DELETE CASCADE, + run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL, + field_names TEXT[] NOT NULL, + rule JSONB NOT NULL, + finalize_now BOOLEAN NOT NULL DEFAULT false, + created_by BIGINT REFERENCES meta.users ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + done_recipe_id BIGINT REFERENCES meta.recipes ON DELETE SET NULL, + cancelled_at TIMESTAMPTZ, + note TEXT +); +CREATE UNIQUE INDEX IF NOT EXISTS pending_derivations_open_idx ON meta.pending_derivations (recipe_id) + WHERE done_recipe_id IS NULL AND cancelled_at IS NULL; +-- derive.state (every team state read) looks rows up by the version they made; _source and lineage by the run. +CREATE INDEX IF NOT EXISTS pending_derivations_done_idx ON meta.pending_derivations (done_recipe_id); +CREATE INDEX IF NOT EXISTS pending_derivations_run_idx ON meta.pending_derivations (run_id); + +ALTER TABLE meta.experiments ADD COLUMN IF NOT EXISTS rerun_of_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; +ALTER TABLE meta.experiments ADD COLUMN IF NOT EXISTS pending_derivation_id BIGINT REFERENCES meta.pending_derivations ON DELETE SET NULL; +ALTER TABLE meta.exports ADD COLUMN IF NOT EXISTS then_run JSONB; + +-- 3. Tuning studies: a batch of kind 'tune' (one job; a study stops through its job, so no 'stopping' status), one +-- row per trial. +ALTER TABLE meta.run_batches DROP CONSTRAINT IF EXISTS run_batches_kind_check; +ALTER TABLE meta.run_batches ADD CONSTRAINT run_batches_kind_check CHECK (kind IN ('one_vs_rest', 'search', 'tune')); +ALTER TABLE meta.run_batches DROP CONSTRAINT IF EXISTS run_batches_status_check; +ALTER TABLE meta.run_batches ADD CONSTRAINT run_batches_status_check + CHECK (status IN ('running', 'done', 'failed', 'cancelled')); +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS best_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS job_id BIGINT; +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS result JSONB; + +CREATE TABLE IF NOT EXISTS meta.tuning_trials ( + batch_id BIGINT NOT NULL REFERENCES meta.run_batches ON DELETE CASCADE, + number INT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('running', 'complete', 'pruned', 'failed')), + value DOUBLE PRECISION, + params JSONB NOT NULL DEFAULT '{}', + steps JSONB NOT NULL DEFAULT '[]', + best_iterations JSONB, + pruned_at INT, + error TEXT, + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL, + PRIMARY KEY (batch_id, number) +); +-- A running trial's last stop check: a killed process's trial counts its time up to here against the budget. +ALTER TABLE meta.tuning_trials ADD COLUMN IF NOT EXISTS last_seen TIMESTAMPTZ; + +-- 4. The worker's job-thread count (the live worker's JOB threads, shown beside CPU runs). +ALTER TABLE meta.workers ADD COLUMN IF NOT EXISTS threads INT; + +-- 5. Row-level security, in 041's style. Every policy is dropped before it is made, so a replay changes nothing. +DO $$ +DECLARE + s CONSTANT TEXT := '(SELECT meta.rf_system())'; + ids CONSTANT TEXT := '(SELECT meta.rf_ids(''projects''))::bigint[]'; + t TEXT; + rd TEXT; + wr TEXT; + rules TEXT[][] := ARRAY[ + ARRAY['field_about_history', + s || ' OR dataset_id IN (SELECT id FROM meta.datasets)', + s || ' OR dataset_id IN (SELECT id FROM meta.datasets WHERE project_id = ANY(' || ids || '))'], + ARRAY['pending_derivations', + s || ' OR recipe_id IN (SELECT id FROM meta.recipes WHERE project_id = ANY(' || ids || '))', + s || ' OR recipe_id IN (SELECT id FROM meta.recipes WHERE project_id = ANY(' || ids || '))'], + ARRAY['tuning_trials', + s || ' OR batch_id IN (SELECT id FROM meta.run_batches)', + s || ' OR batch_id IN (SELECT b.id FROM meta.run_batches b JOIN meta.experiments x ON x.id = b.experiment_id' + || ' WHERE x.project_id = ANY(' || ids || '))'] + ]; +BEGIN + FOR i IN 1 .. array_length(rules, 1) LOOP + t := rules[i][1]; rd := rules[i][2]; wr := rules[i][3]; + EXECUTE format('ALTER TABLE meta.%I ENABLE ROW LEVEL SECURITY', t); + EXECUTE format('DROP POLICY IF EXISTS rf_read ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_write ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_change ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_delete ON meta.%I', t); + EXECUTE format('CREATE POLICY rf_read ON meta.%I AS PERMISSIVE FOR SELECT TO PUBLIC USING (%s)', t, rd); + EXECUTE format('CREATE POLICY rf_write ON meta.%I AS PERMISSIVE FOR INSERT TO PUBLIC WITH CHECK (%s)', t, wr); + EXECUTE format('CREATE POLICY rf_change ON meta.%I AS PERMISSIVE FOR UPDATE TO PUBLIC USING (%s) WITH CHECK (%s)', t, wr, wr); + EXECUTE format('CREATE POLICY rf_delete ON meta.%I AS PERMISSIVE FOR DELETE TO PUBLIC USING (%s)', t, wr); + END LOOP; +END $$; diff --git a/api/app/modeling/metrics.py b/api/app/modeling/metrics.py new file mode 100644 index 00000000..bd21a405 --- /dev/null +++ b/api/app/modeling/metrics.py @@ -0,0 +1,169 @@ +"""Classification arithmetic over a confusion matrix (rows actual, columns predicted), in one place for training runs +and evaluations. `ui/src/metrics.ts` computes the same numbers; both are held to `tests/data/confusion_cases.json`. + +Nothing here reads a model or a file: the input is a matrix (or, for the fragile-call helpers, scores), so a run's +numbers and an evaluation's cannot drift apart.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from .registry import other_class + +FRAGILE_DELTA = 0.05 # a correct call whose probability is within this of the cut is fragile + + +def _div(a: float, b: float) -> float | None: + return float(a) / float(b) if b > 0 else None + + +def _zdiv(a: float, b: float) -> float: + return float(a) / float(b) if b > 0 else 0.0 + + +def _f1(p: float, r: float) -> float: + return 2 * p * r / (p + r) if p + r > 0 else 0.0 + + +def per_class(cm: np.ndarray) -> list[dict[str, Any]]: + """Per class: tp, fp, fn, tn, support, precision, recall, f1 (0 when undefined, as scikit-learn) and the false + alarm rate FP / (FP + TN) (None when the class has no negatives).""" + cm = np.asarray(cm, dtype=np.int64) + total = int(cm.sum()) + out = [] + for i in range(len(cm)): + tp = int(cm[i, i]) + support = int(cm[i].sum()) + fp = int(cm[:, i].sum()) - tp + fn = support - tp + tn = total - tp - fp - fn + p, r = _zdiv(tp, tp + fp), _zdiv(tp, tp + fn) + out.append( + { + "tp": tp, + "fp": fp, + "fn": fn, + "tn": tn, + "support": support, + "precision": p, + "recall": r, + "f1": _f1(p, r), + "far": _div(fp, fp + tn), + } + ) + return out + + +def _mean(xs: list[float]) -> float | None: + return float(sum(xs) / len(xs)) if xs else None + + +def _weighted(pairs: list[tuple[float, int]]) -> float | None: + w = sum(s for _, s in pairs) + return float(sum(v * s for v, s in pairs) / w) if w > 0 else None + + +def scores(cm: np.ndarray, *, include: list[int] | None = None, positive: int | None = None) -> dict[str, Any]: + """The scores table. `include` (default all) are the classes macro and weighted average over and micro pools + over. `positive` fills the plain precision, recall, F1 and false alarm rate (otherwise None). Plain accuracy is + the included classes' correct calls over every row, so a row whose label has no model counts as wrong.""" + cm = np.asarray(cm, dtype=np.int64) + rows = per_class(cm) + include = list(range(len(cm))) if include is None else list(include) + used = [rows[i] for i in include] + total = int(cm.sum()) + tp, fp = sum(r["tp"] for r in used), sum(r["fp"] for r in used) + fn, tn = sum(r["fn"] for r in used), sum(r["tn"] for r in used) + accuracy = _div(tp, total) + micro_p, micro_r = _zdiv(tp, tp + fp), _zdiv(tp, tp + fn) + with_far = [r for r in used if r["far"] is not None] + pos = rows[positive] if positive is not None else None + + def by_support(key: str) -> float | None: + return _weighted([(r[key], r["support"]) for r in used]) + + return { + "plain": { + "accuracy": accuracy, + "precision": pos["precision"] if pos else None, + "recall": pos["recall"] if pos else None, + "f1": pos["f1"] if pos else None, + "far": pos["far"] if pos else None, + }, + "macro": { + "accuracy": _mean([r["recall"] for r in used]), + "precision": _mean([r["precision"] for r in used]), + "recall": _mean([r["recall"] for r in used]), + "f1": _mean([r["f1"] for r in used]), + "far": _mean([r["far"] for r in with_far]), + }, + "micro": { + "accuracy": accuracy, + "precision": micro_p, + "recall": micro_r, + "f1": _f1(micro_p, micro_r), + "far": _div(fp, fp + tn), + }, + "weighted": { + "precision": by_support("precision"), + "recall": by_support("recall"), + "f1": by_support("f1"), + "far": _weighted([(r["far"], r["support"]) for r in with_far]), + }, + } + + +def types(cm: np.ndarray, reference: int, *, k: int | None = None) -> dict[str, Any]: + """Type I: the reference class's rows called something else. Type II: another class called the reference. + Type III: every other wrong call (None for fewer than three real labels `k`; the matrix may hold an extra + "(no match)" column). `errors` is every off-diagonal cell.""" + cm = np.asarray(cm, dtype=np.int64) + k = len(cm) if k is None else k + off = cm - np.diag(np.diag(cm)) + type_i = int(off[reference].sum()) + type_ii = int(off[:, reference].sum()) + rest = int(off.sum()) - type_i - type_ii + int(off[reference, reference]) + return { + "reference": reference, + "type_i": type_i, + "type_ii": type_ii, + "type_iii": rest if k >= 3 else None, + "errors": type_i + type_ii + rest, + } + + +def plain_far(cm: np.ndarray, reference: int) -> float | None: + """The reference class's rows predicted as another class, over the reference class's rows.""" + cm = np.asarray(cm, dtype=np.int64) + n = int(cm[reference].sum()) + return _div(n - int(cm[reference, reference]), n) + + +def reference_for(manifest: dict, target: str, labels: list[str]) -> int | None: + """The index of the reference class: the class a collapsed policy merged the rest into, else the first class of + a binary target, else None (no most-frequent fallback).""" + other = other_class(manifest, target) + if other is not None: + return labels.index(other) if other in labels else None + return 0 if len(labels) == 2 else None + + +def margins(proba: np.ndarray) -> np.ndarray: + """Top-1 minus top-2 probability per row.""" + proba = np.asarray(proba, dtype=float) + if proba.shape[1] < 2: + return np.ones(len(proba)) + top = np.partition(proba, -2, axis=1) + return top[:, -1] - top[:, -2] + + +def threshold_margins(score: np.ndarray, threshold: float) -> np.ndarray: + """A binary call's margin: 2 |score - threshold|, on the same scale as `margins`.""" + return 2 * np.abs(np.asarray(score, dtype=float) - threshold) + + +def fragile_correct(correct: np.ndarray, margin: np.ndarray, delta: float = FRAGILE_DELTA) -> int: + """Correct calls with a margin under 2 delta (for a binary call: within delta of the cut).""" + return int((np.asarray(correct, dtype=bool) & (np.asarray(margin) < 2 * delta)).sum()) diff --git a/api/app/modeling/params.py b/api/app/modeling/params.py index ab6843e3..39f401c5 100644 --- a/api/app/modeling/params.py +++ b/api/app/modeling/params.py @@ -27,11 +27,15 @@ class Setting: when: dict[str, tuple] | None = None none_value: float | None = None none_label: str | None = None + # A tuning study's default search over this setting: a range ({low, high, log?, integer?, none?}) or a tuple of + # choices; None: not tuned by default. + tune: dict | tuple | None = None def to_json(self) -> dict: d = asdict(self) d["options"] = list(self.options) if self.options else None d["when"] = {k: list(v) for k, v in self.when.items()} if self.when else None + d["tune"] = list(self.tune) if isinstance(self.tune, tuple) else (dict(self.tune) if self.tune else None) return d @@ -39,6 +43,24 @@ def _opt(value, label, help=None) -> dict: return {"value": value, "label": label, **({"help": help} if help else {})} +def _range(low: float, high: float, *, log: bool = False, integer: bool = False, none: bool = False) -> dict: + """A tuning range, in the batch `space` format; `none` lets the search also pick the setting's none value.""" + return { + "low": low, + "high": high, + **({"log": True} if log else {}), + **({"integer": True} if integer else {}), + **({"none": True} if none else {}), + } + + +# Ranges the boosters share (spec 3). +LEARNING_RATE = _range(0.01, 0.3, log=True) +SUBSAMPLE = _range(0.5, 1) +COLSAMPLE = _range(0.4, 1) +PENALTY = _range(1e-3, 10, log=True) + + RUN = [ Setting( "seed", @@ -103,6 +125,7 @@ def _opt(value, label, help=None) -> dict: 63, 2, 4096, + tune=_range(15, 255, log=True, integer=True), ), Setting( "max_depth", @@ -125,6 +148,7 @@ def _opt(value, label, help=None) -> dict: 40, 1, 10_000, + tune=_range(5, 200, log=True, integer=True), ), Setting( "learning_rate", @@ -136,6 +160,7 @@ def _opt(value, label, help=None) -> dict: 0.001, 1.0, 0.005, + tune=LEARNING_RATE, ), Setting( "subsample", @@ -148,6 +173,7 @@ def _opt(value, label, help=None) -> dict: 1.0, 0.05, when=BAGGING, + tune=SUBSAMPLE, ), Setting( "subsample_freq", @@ -170,6 +196,7 @@ def _opt(value, label, help=None) -> dict: 0.1, 1.0, 0.05, + tune=COLSAMPLE, ), Setting( "reg_alpha", @@ -181,6 +208,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.1, + tune=PENALTY, ), Setting( "reg_lambda", @@ -192,6 +220,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.1, + tune=PENALTY, ), Setting( "min_split_gain", @@ -459,6 +488,7 @@ def _opt(value, label, help=None) -> dict: 64, none_value=0, none_label="No limit", + tune=_range(3, 10, integer=True), ), Setting( "grow_policy", @@ -492,6 +522,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 1000.0, 0.5, + tune=_range(1, 20, log=True), ), Setting( "learning_rate", @@ -503,8 +534,20 @@ def _opt(value, label, help=None) -> dict: 0.001, 1.0, 0.005, + tune=LEARNING_RATE, + ), + Setting( + "subsample", + "Row sample", + "Share of rows each tree sees.", + "sampling", + "float", + 0.8, + 0.1, + 1.0, + 0.05, + tune=SUBSAMPLE, ), - Setting("subsample", "Row sample", "Share of rows each tree sees.", "sampling", "float", 0.8, 0.1, 1.0, 0.05), Setting( "colsample_bytree", "Feature sample", @@ -515,6 +558,7 @@ def _opt(value, label, help=None) -> dict: 0.1, 1.0, 0.05, + tune=COLSAMPLE, ), Setting( "reg_alpha", @@ -526,6 +570,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.1, + tune=PENALTY, ), Setting( "reg_lambda", @@ -537,6 +582,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.1, + tune=PENALTY, ), Setting( "gamma", @@ -548,6 +594,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.01, + tune=_range(0, 5), ), Setting( "scale_pos_weight", @@ -758,6 +805,7 @@ def _opt(value, label, help=None) -> dict: 10, 5000, 50, + tune=_range(100, 800, integer=True), ), Setting( "max_depth", @@ -770,6 +818,7 @@ def _opt(value, label, help=None) -> dict: 100, none_value=None, none_label="No limit", + tune=_range(4, 30, integer=True, none=True), ), Setting( "min_samples_leaf", @@ -780,6 +829,7 @@ def _opt(value, label, help=None) -> dict: 3, 1, 1000, + tune=_range(1, 20, integer=True), ), Setting( "min_samples_split", @@ -801,10 +851,13 @@ def _opt(value, label, help=None) -> dict: options=( _opt("sqrt", "Square root of all"), _opt("log2", "Log2 of all"), + _opt(0.8, "Four fifths"), _opt(0.5, "Half"), _opt(0.33, "A third"), + _opt(0.3, "Three tenths"), _opt(1.0, "All"), ), + tune=("sqrt", "log2", 0.3, 0.5, 0.8), ), Setting("bootstrap", "Bootstrap", "Each tree sees a resample of the rows.", "sampling", "bool", True), Setting( @@ -891,6 +944,7 @@ def _opt(value, label, help=None) -> dict: 4096, none_value=None, none_label="No limit", + tune=_range(15, 255, integer=True), ), Setting( "max_depth", @@ -913,6 +967,7 @@ def _opt(value, label, help=None) -> dict: 20, 1, 10_000, + tune=_range(5, 200, integer=True), ), Setting( "learning_rate", @@ -924,6 +979,7 @@ def _opt(value, label, help=None) -> dict: 0.001, 1.0, 0.005, + tune=LEARNING_RATE, ), Setting( "max_features", @@ -946,6 +1002,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 100.0, 0.1, + tune=PENALTY, ), Setting( "class_weight", @@ -1009,6 +1066,7 @@ def _opt(value, label, help=None) -> dict: "choice", "128,64", options=tuple(_opt(v, " × ".join(v.split(",")) + (" units" if "," not in v else "")) for v in MLP_LAYERS), + tune=("64", "128", "128,64", "256,128"), ), Setting( "activation", @@ -1029,6 +1087,7 @@ def _opt(value, label, help=None) -> dict: 0.00001, 1.0, 0.0005, + tune=_range(1e-4, 1e-2, log=True), ), Setting( "solver", @@ -1050,6 +1109,7 @@ def _opt(value, label, help=None) -> dict: 0.0, 10.0, 0.0001, + tune=_range(1e-6, 1e-2, log=True), ), Setting( "early_stopping", @@ -1086,6 +1146,7 @@ def _opt(value, label, help=None) -> dict: 10_000.0, 0.1, applies="classes", + tune=_range(1e-3, 100, log=True), ), Setting( "penalty", @@ -1159,6 +1220,7 @@ def _opt(value, label, help=None) -> dict: 10_000.0, 0.1, applies="numbers", + tune=_range(1e-3, 100, log=True), ), ] diff --git a/api/app/modeling/registry.py b/api/app/modeling/registry.py index 25d5f541..b8b7a1e8 100644 --- a/api/app/modeling/registry.py +++ b/api/app/modeling/registry.py @@ -36,6 +36,7 @@ class ModelSpec: regressor_label: str | None = None regressor_description: str | None = None mixed_label: str | None = None # a chain of class and number targets + native_categorical: bool = False # splits on a category column directly (no encoding needed) @property def devices(self) -> list[str]: @@ -142,6 +143,7 @@ def _mlp_reg(params: dict[str, Any], seed: int): importance="booster_gain", description="Leaf-wise gradient boosting. The default.", build_regressor=_lightgbm_reg, + native_categorical=True, ), ModelSpec( "xgboost", @@ -163,6 +165,7 @@ def _mlp_reg(params: dict[str, Any], seed: int): description="Depth-wise gradient boosting; trains on an NVIDIA GPU in the GPU image.", cuda_params={"device": "cuda", "tree_method": "hist"}, build_regressor=_xgboost_reg, + native_categorical=True, ), ModelSpec( "hist_gradient_boosting", @@ -180,6 +183,7 @@ def _mlp_reg(params: dict[str, Any], seed: int): importance="none", description="scikit-learn's LightGBM-style booster; no extra dependency.", build_regressor=_hgb_reg, + native_categorical=True, ), ModelSpec( "random_forest", @@ -438,7 +442,52 @@ def regression_problem(spec: ModelSpec, kinds: list[str]) -> str | None: return None -def compatibility(spec: ModelSpec, export_family: str, contains_nan: bool) -> str | None: +HGB_MAX_LEVELS = 255 + + +def native_params(spec: ModelSpec) -> dict[str, Any]: + """The setting that makes a model read category columns (LightGBM reads the dtype with its default "auto").""" + if not spec.native_categorical: + return {} + if spec.key == "xgboost": + return {"enable_categorical": True} + if spec.key == "hist_gradient_boosting": + return {"categorical_features": "from_dtype"} + return {} + + +def _source_fields(manifest: dict, category_levels: dict[str, list[str]]) -> list[str]: + """The source fields behind the native features, once each, in feature order.""" + src = manifest.get("feature_source_field") or {} + out: list[str] = [] + for f in category_levels: + name = src.get(f, f) + if name not in out: + out.append(name) + return out + + +def _names_text(names: list[str]) -> str: + return ", ".join(names[:5]) + (f", and {len(names) - 5} more" if len(names) > 5 else "") + + +def encode_fields(spec: ModelSpec, manifest: dict) -> list[str] | None: + """The source fields a refusal for categories names (None when the model isn't refused for them).""" + levels = manifest.get("category_levels") or {} + if compatibility(spec, manifest.get("model_family", "tree"), bool(manifest.get("contains_nan"))) is not None: + return None # refused for its family or for NaN first; the categories aren't the reason + if levels and not spec.native_categorical: + return _source_fields(manifest, levels) + return None + + +def compatibility( + spec: ModelSpec, + export_family: str, + contains_nan: bool, + category_levels: dict[str, list[str]] | None = None, + source_fields: dict[str, str] | None = None, +) -> str | None: """None if the model can train on an export, else the reason it can't.""" if contains_nan and not spec.handles_nan: return f"{spec.label} can't handle missing values and this export contains NaN" @@ -447,6 +496,21 @@ def compatibility(spec: ModelSpec, export_family: str, contains_nan: bool) -> st f"{spec.label} needs a {FAMILIES[spec.family]['label'].lower()} export " f"(this one was prepared for {FAMILIES[export_family]['label'].lower()})" ) + if category_levels: + src = source_fields or {} + if not spec.native_categorical: + names = _source_fields({"feature_source_field": src}, category_levels) + return ( + f"{spec.label} can't take categories as-is ({_names_text(names)}); encode them for this experiment, " + "or choose LightGBM, XGBoost or histogram GBM" + ) + if spec.key == "hist_gradient_boosting": + for f, lv in category_levels.items(): + if len(lv) > HGB_MAX_LEVELS: + return ( + f"Histogram GBM takes at most {HGB_MAX_LEVELS} levels per field; {src.get(f, f)} has " + f"{len(lv)} levels. Encode it for this experiment, or choose LightGBM or XGBoost" + ) return None @@ -505,3 +569,9 @@ def order_problem(order: list[str], targets: list[str]) -> str | None: if unknown := [t for t in order if t not in targets]: return f"Not targets of this export: {', '.join(unknown)} (its targets: {', '.join(targets)})" return None + + +def other_class(manifest: dict, target: str | None) -> str | None: + """The label the target's class policy gave the classes it merged, when it merged any.""" + p = (manifest.get("target_policy") or {}).get(target or "") or {} + return p.get("other") if p.get("collapsed") else None diff --git a/api/app/modeling/train.py b/api/app/modeling/train.py index fb845ff2..401f143c 100644 --- a/api/app/modeling/train.py +++ b/api/app/modeling/train.py @@ -37,6 +37,7 @@ import joblib import numpy as np +import pandas as pd import polars as pl from sklearn import config_context from sklearn.metrics import ( @@ -53,8 +54,9 @@ from sklearn.multioutput import ClassifierChain from .. import bands +from ..feature_frame import FeatureFrame from ..gpu import effective_device -from . import crossval +from . import crossval, metrics from .mlflow_log import MODEL_FILE from .registry import ( DEFAULT_MODEL, @@ -67,6 +69,7 @@ get_model, importances, label_for, + native_params, no_valid_problem, one_vs_rest_target, order_problem, @@ -84,6 +87,30 @@ warnings.filterwarnings("ignore", message="X does not have valid feature names", category=UserWarning) +def _rows(X, rows): + """`X` (an array or a model-ready frame) at the row positions `rows`.""" + return X.iloc[rows] if isinstance(X, pd.DataFrame) else X[rows] + + +def _with_chain(X, outputs: np.ndarray): + """`X` with the earlier targets' outputs as extra columns: floats named `__chain_` beside a frame's columns, + stacked beside an array's.""" + if outputs.shape[1] == 0: + return X + if isinstance(X, pd.DataFrame): + return X.assign(**{f"__chain_{k}": outputs[:, k] for k in range(outputs.shape[1])}) + return np.hstack((X, outputs)) + + +def split_xy( + manifest: dict, df: pl.DataFrame, split: str, targets: list[str], frame: FeatureFrame +) -> tuple[np.ndarray | pd.DataFrame, np.ndarray]: + """The model's features (as `frame` makes them: float32, or a frame with category columns) and the targets + (float64) of one part of the export.""" + part = df.filter(pl.col(manifest["split_column"]) == split) + return frame.from_polars(part), part.select(targets).to_numpy().astype(np.float64) + + class NaNTolerantClassifierChain(ClassifierChain): """ClassifierChain that lets NaN through to models that route missing values natively. @@ -138,10 +165,10 @@ def _scaled(self, t: int, values: np.ndarray) -> np.ndarray: return (values - mean) / std @staticmethod - def _output(est, X: np.ndarray, kind: str) -> np.ndarray: + def _output(est, X, kind: str) -> np.ndarray: return est.predict_proba(X)[:, 1] if kind == "binary" else est.predict(X) - def fit(self, X: np.ndarray, Y: np.ndarray) -> "TargetChain": + def fit(self, X, Y: np.ndarray) -> "TargetChain": from sklearn.base import clone from sklearn.model_selection import cross_val_predict @@ -156,7 +183,7 @@ def fit(self, X: np.ndarray, Y: np.ndarray) -> "TargetChain": for pos, t in enumerate(self.order): kind = self.kinds[t] y = Y[:, t] if kind == "regression" else Y[:, t].astype(np.int64) - X_aug = np.hstack((X, feats[:, :pos])) + X_aug = _with_chain(X, feats[:, :pos]) base = self.regressor if kind == "regression" else self.classifier self.estimators_.append(clone(base).fit(X_aug, y)) if pos == len(self.order) - 1: @@ -176,14 +203,14 @@ def fit(self, X: np.ndarray, Y: np.ndarray) -> "TargetChain": self.order_ = list(self.order) return self - def predict_all(self, X: np.ndarray, n_classes: list[int]) -> list[np.ndarray]: + def predict_all(self, X, n_classes: list[int]) -> list[np.ndarray]: """Per target, in target order: the class-probability matrix (column j = encoded class j) or the values.""" feats = np.zeros((X.shape[0], len(self.order_))) out: list[np.ndarray | None] = [None] * len(self.order_) with config_context(assume_finite=self.handles_nan): for pos, est in enumerate(self.estimators_): t = self.order_[pos] - X_aug = np.hstack((X, feats[:, :pos])) + X_aug = _with_chain(X, feats[:, :pos]) if self.kinds[t] == "regression": out[t] = est.predict(X_aug).astype(float) feats[:, pos] = self._scaled(t, out[t]) @@ -278,12 +305,13 @@ def roc_curves(y_true: np.ndarray, proba: np.ndarray, names: list[str]) -> list[ def evaluate( - y_true: np.ndarray, proba: np.ndarray, labels: list[str] | None = None + y_true: np.ndarray, proba: np.ndarray, labels: list[str] | None = None, reference: int | None = None ) -> tuple[dict[str, float], dict[str, Any]]: """Score one target. Predictions are the argmax class (i.e. a 0.5 threshold for binary targets). Returns (scalars, details): scalars are single numbers (stored in `metrics`, queryable via - meta.v_model_metrics); details hold the confusion matrix, per-class table and ROC curves. + meta.v_model_metrics); details hold the confusion matrix, per-class table and ROC curves. `reference` (a class + index, None when the target has none) adds the error types I and II, and III for three or more classes. """ k = proba.shape[1] classes = list(range(k)) @@ -298,9 +326,29 @@ def evaluate( p, r, f, _ = precision_recall_fscore_support(y_true, pred, labels=classes, average=avg, zero_division=0) out[f"precision_{avg}"], out[f"recall_{avg}"], out[f"f1_{avg}"] = float(p), float(r), float(f) cm = confusion_matrix(y_true, pred, labels=classes) + table = metrics.scores(cm, positive=1 if k == 2 else None) + out["precision_micro"], out["recall_micro"] = table["micro"]["precision"], table["micro"]["recall"] + out["f1_micro"] = table["micro"]["f1"] + for key, cell in (("far_macro", table["macro"]["far"]), ("far_weighted", table["weighted"]["far"])): + if cell is not None: + out[key] = cell + out["fragile_correct"] = metrics.fragile_correct( + pred == y_true, metrics.threshold_margins(proba[:, 1], 0.5) if k == 2 else metrics.margins(proba) + ) + # Binary: the plain precision, recall, F1 and false alarm rate always describe the positive class (index 1), + # while the error types describe the reference. A collapsed policy's other class can be index 1 (it sorts after + # the kept class): then Type I is the positive class's misses (= fn) and Type II its false + # alarms (= fp), the plain numbers unchanged. The UI says so when reference = positive. + if reference is not None: + kinds = metrics.types(cm, reference) + out["type_i"], out["type_ii"] = kinds["type_i"], kinds["type_ii"] + if kinds["type_iii"] is not None: + out["type_iii"] = kinds["type_iii"] if k == 2: # Positive class = encoded label 1 (see manifest target_classes). tn, fp, fn, tp = (int(v) for v in cm.ravel()) + if table["plain"]["far"] is not None: + out["far"] = table["plain"]["far"] out["precision"] = tp / (tp + fp) if tp + fp else 0.0 out["recall"] = tp / (tp + fn) if tp + fn else 0.0 out["f1"] = 2 * out["precision"] * out["recall"] / (out["precision"] + out["recall"]) if tp else 0.0 @@ -320,9 +368,11 @@ def evaluate( except ValueError: # a class absent from this split pass p, r, f, support = precision_recall_fscore_support(y_true, pred, labels=classes, zero_division=0) + cells = metrics.per_class(cm) details = { "labels": names, "positive_label": names[1] if k == 2 else None, + "reference": names[reference] if reference is not None else None, "confusion_matrix": cm.astype(int).tolist(), # rows = actual, columns = predicted "per_class": [ { @@ -331,6 +381,7 @@ def evaluate( "recall": float(r[c]), "f1": float(f[c]), "support": int(support[c]), + **{key: cells[c][key] for key in ("tp", "fp", "fn", "tn", "far")}, } for c in classes ], @@ -341,6 +392,12 @@ def evaluate( return out, details +def _evaluate_target(manifest: dict, target: str, y_true: np.ndarray, proba: np.ndarray): + labels = manifest["target_classes"].get(target) + ref = metrics.reference_for(manifest, target, labels) if labels else (0 if proba.shape[1] == 2 else None) + return evaluate(y_true, proba, labels, ref) + + TABLE_THRESHOLDS = np.round(np.arange(0.05, 0.951, 0.05), 2) CURVE_THRESHOLDS = np.round(np.linspace(0, 1, 101), 2) @@ -352,6 +409,8 @@ def threshold_table(y_true: np.ndarray, p1: np.ndarray, thresholds: np.ndarray = for t in thresholds: flagged = p1 >= t tp = int((flagged & (y_true == 1)).sum()) + fp = int(flagged.sum()) - tp + negatives = int((y_true != 1).sum()) prec = tp / int(flagged.sum()) if flagged.any() else 0.0 rec = tp / int((y_true == 1).sum()) if (y_true == 1).any() else 0.0 out.append( @@ -361,6 +420,10 @@ def threshold_table(y_true: np.ndarray, p1: np.ndarray, thresholds: np.ndarray = "recall": rec, "f1": 2 * prec * rec / (prec + rec) if prec + rec else 0.0, "flagged": float(flagged.mean()), + "far": fp / negatives if negatives else None, + "fragile_correct": metrics.fragile_correct( + flagged == (y_true == 1), metrics.threshold_margins(p1, float(t)) + ), } ) return out @@ -406,7 +469,7 @@ def calibration_method(positives: int, negatives: int) -> str: return "isotonic" if positives >= ISOTONIC_MIN and negatives >= ISOTONIC_MIN else "sigmoid" -def calibrate(est, X: np.ndarray, y: np.ndarray, method: str, handles_nan: bool): +def calibrate(est, X, y: np.ndarray, method: str, handles_nan: bool): """`est`, already fitted, with a calibrator fit on (X, y): its predict_proba gives calibrated probabilities.""" from sklearn.calibration import CalibratedClassifierCV from sklearn.frozen import FrozenEstimator @@ -429,11 +492,12 @@ def balance_params(spec: ModelSpec, y: np.ndarray) -> tuple[dict[str, Any], str return {}, f"{spec.label} has no class weighting; balance was ignored" -def scores_of(model, report: dict, X: np.ndarray) -> np.ndarray: +def scores_of(model, report: dict, X: np.ndarray | pd.DataFrame) -> np.ndarray: """A run's scores for rows X (the export's features, in manifest order): calibrated P(positive) for a one-vs-rest run; for a single-target run, the class-probability matrix (column j = encoded class j) or the predicted numbers. Rows are scored as float32, as training scores them, so the same rows give the same numbers.""" - X = np.asarray(X, dtype=np.float32) + if not isinstance(X, pd.DataFrame): + X = np.asarray(X, dtype=np.float32) spec = MODELS.get(report.get("model")) with config_context(assume_finite=bool(spec and spec.handles_nan)): if report.get("chain_kind") == "one_vs_rest": @@ -607,7 +671,13 @@ def train( params = {**(params or {}), **device_params(spec, device)} manifest, df = load_export(export_dir) export_family = manifest.get("model_family", "tree") - problem = compatibility(spec, export_family, bool(manifest.get("contains_nan", False))) + problem = compatibility( + spec, + export_family, + bool(manifest.get("contains_nan", False)), + manifest.get("category_levels"), + manifest.get("feature_source_field"), + ) if problem: raise ValueError(problem) say = progress or (lambda msg: log.info(msg)) @@ -633,7 +703,6 @@ def train( balance, ) feats: list[str] = manifest["feature_names"] - split_col = manifest["split_column"] order_names = order_names or manifest["targets"] if problem := order_problem(order_names, manifest["targets"]): raise ValueError(problem) @@ -645,9 +714,10 @@ def train( if problem := regression_problem(spec, kinds): raise ValueError(problem) - def xy(split: str) -> tuple[np.ndarray, np.ndarray]: - part = df.filter(pl.col(split_col) == split) - return part.select(feats).to_numpy().astype(np.float32), part.select(targets).to_numpy().astype(np.float64) + frame = FeatureFrame.from_manifest(manifest) + + def xy(split: str): + return split_xy(manifest, df, split, targets, frame) X_tr, Y_tr = xy("train") n_classes = [0 if num else len(manifest["target_classes"][t]) for t, num in zip(targets, numeric)] @@ -668,10 +738,12 @@ def xy(split: str) -> tuple[np.ndarray, np.ndarray]: ) def make_chain(): - if not any(numeric): + if not any(numeric) and not frame.native: return build_chain(spec, kinds, order, seed, params, cv) merged = {**spec.default_params, **(params or {})} - regressor = spec.build_regressor({**spec.regressor_defaults, **regressor_params(spec, merged)}, seed) + regressor = spec.build_regressor( + {**spec.regressor_defaults, **regressor_params(spec, merged), **native_params(spec)}, seed + ) if spec.family != "tree": # Gradient-trained and penalized models fit a target on its own scale badly (an MLP stops long before a # price in the hundreds of thousands); trees split on order and don't care. @@ -680,26 +752,32 @@ def make_chain(): regressor = TransformedTargetRegressor(regressor=regressor, transformer=StandardScaler()) return TargetChain( - spec.build(classifier_params(spec, merged), seed), regressor, kinds, order, cv, spec.handles_nan, seed + spec.build({**classifier_params(spec, merged), **native_params(spec)}, seed), + regressor, + kinds, + order, + cv, + spec.handles_nan, + seed, ) - def fit(chain, X: np.ndarray, Y: np.ndarray): + def fit(chain, X, Y: np.ndarray): return chain.fit(X, Y if any(numeric) else Y.astype(np.int64)) chain = make_chain() t0 = time.time() fit(chain, X_tr, Y_tr) fit_seconds = time.time() - t0 + # XGBoost quietly drops to the CPU when it can't reach the GPU; don't report that as a GPU run. + used = effective_device(chain.estimators_[0].get_booster()) if spec.key == "xgboost" else "cpu" if device == "cuda": - # XGBoost quietly drops to the CPU when it can't reach the GPU; don't report that as a GPU run. - used = effective_device(chain.estimators_[0].get_booster()) if not used.startswith("cuda"): raise RuntimeError(f"GPU training was requested but {spec.label} ran on {used}") # Ranges for scoring a number as classes too, cut on the training rows like an export's equal-sized bands. band_edges = [_band_edges(Y_tr[:, i]) if num else None for i, num in enumerate(numeric)] - def score(chain, X: np.ndarray, Y: np.ndarray) -> tuple[dict[str, Any], dict[str, Any]]: + def score(chain, X, Y: np.ndarray) -> tuple[dict[str, Any], dict[str, Any]]: """Each target's metrics and evaluation details on rows (X, Y).""" m, d = {}, {} if isinstance(chain, TargetChain): @@ -708,14 +786,14 @@ def score(chain, X: np.ndarray, Y: np.ndarray) -> tuple[dict[str, Any], dict[str m[t], d[t] = ( evaluate_regression(Y[:, i], outs[i], seed, band_edges[i]) if numeric[i] - else evaluate(Y[:, i].astype(int), outs[i], manifest["target_classes"].get(t)) + else _evaluate_target(manifest, t, Y[:, i].astype(int), outs[i]) ) return m, d with config_context(assume_finite=spec.handles_nan): probas = chain_predict_proba(chain, X, n_classes) predicted = chain.predict(X).astype(int) for i, t in enumerate(targets): - m[t], d[t] = evaluate(Y[:, i].astype(int), probas[i], manifest["target_classes"].get(t)) + m[t], d[t] = _evaluate_target(manifest, t, Y[:, i].astype(int), probas[i]) # Sanity check: the manual walk must agree with sklearn's own chain prediction if not np.array_equal(np.column_stack([p.argmax(1) for p in probas]), predicted): raise RuntimeError("chain_predict_proba diverged from ClassifierChain.predict") @@ -744,8 +822,8 @@ def score(chain, X: np.ndarray, Y: np.ndarray) -> tuple[dict[str, Any], dict[str def fit_score(rows: np.ndarray, held: np.ndarray) -> dict: fold_chain = make_chain() - fit(fold_chain, X_tr[rows], Y_tr[rows]) - return score(fold_chain, X_tr[held], Y_tr[held])[0] + fit(fold_chain, _rows(X_tr, rows), Y_tr[rows]) + return score(fold_chain, _rows(X_tr, held), Y_tr[held])[0] cross_validation = _cross_validate( splits, @@ -785,7 +863,7 @@ def fit_score(rows: np.ndarray, held: np.ndarray) -> dict: "export_family": export_family, "order": order_names, "chain_method": chain.chain_method_, - "chain_kind": "target_chain" if any(numeric) else "classifier_chain", + "chain_kind": "target_chain" if any(numeric) or frame.native else "classifier_chain", "target_tasks": {t: (manifest.get("target_tasks") or {}).get(t, "classes") for t in targets}, "target_kinds": dict(zip(targets, kinds)), "cv": cv, @@ -796,6 +874,7 @@ def fit_score(rows: np.ndarray, held: np.ndarray) -> dict: "n_features": len(feats), "fit_seconds": round(fit_seconds, 1), "device": device, + "effective_device": used, "params": { k: (list(v) if isinstance(v, tuple) else v) for k, v in {**spec.default_params, **(params or {})}.items() }, @@ -827,15 +906,15 @@ def _train_one_vs_rest( """One binary model of `target == positive_class`, calibrated on the validation split; every score it reports is a calibrated one. The export's other targets take no part.""" feats: list[str] = manifest["feature_names"] - split_col = manifest["split_column"] code = manifest["target_classes"][target].index(positive_class) labels = [f"not {positive_class}", positive_class] name = f"{target} = {positive_class}" - def xy(split: str) -> tuple[np.ndarray, np.ndarray]: - part = df.filter(pl.col(split_col) == split) - y = part[target].to_numpy().astype(np.int64) == code - return part.select(feats).to_numpy().astype(np.float32), y.astype(np.int64) + frame = FeatureFrame.from_manifest(manifest) + + def xy(split: str): + X, Y = split_xy(manifest, df, split, [target], frame) + return X, (Y[:, 0].astype(np.int64) == code).astype(np.int64) if problem := no_valid_problem(manifest): raise ValueError(problem) @@ -862,8 +941,8 @@ def xy(split: str) -> tuple[np.ndarray, np.ndarray]: + (" on the GPU" if device == "cuda" else "") ) - def fit(X: np.ndarray, y: np.ndarray): - est = spec.build(classifier_params(spec, {**spec.default_params, **params}), seed) + def fit(X, y: np.ndarray): + est = spec.build({**classifier_params(spec, {**spec.default_params, **params}), **native_params(spec)}, seed) with config_context(assume_finite=spec.handles_nan): est.fit(X, y) return est, calibrate(est, X_va, y_va, method, spec.handles_nan) @@ -871,14 +950,15 @@ def fit(X: np.ndarray, y: np.ndarray): t0 = time.time() est, model = fit(X_tr, y_tr) fit_seconds = time.time() - t0 - if device == "cuda" and not (used := effective_device(est.get_booster())).startswith("cuda"): + used = effective_device(est.get_booster()) if spec.key == "xgboost" else "cpu" + if device == "cuda" and not used.startswith("cuda"): raise RuntimeError(f"GPU training was requested but {spec.label} ran on {used}") info = {"chain_kind": "one_vs_rest", "model": spec.key} - def score(fitted, X: np.ndarray, y: np.ndarray) -> tuple[dict[str, float], dict[str, Any]]: + def score(fitted, X, y: np.ndarray) -> tuple[dict[str, float], dict[str, Any]]: p1 = scores_of(fitted, info, X) - return evaluate(y, np.column_stack([1 - p1, p1]), labels) + return evaluate(y, np.column_stack([1 - p1, p1]), labels, reference=0) say(f"Calibrating ({method}) and evaluating on {', '.join(held)}") with config_context(assume_finite=spec.handles_nan): @@ -891,7 +971,7 @@ def score(fitted, X: np.ndarray, y: np.ndarray) -> tuple[dict[str, float], dict[ if not len(X): continue p1 = scores_of(model, info, X) - m, d = evaluate(y, np.column_stack([1 - p1, p1]), labels) + m, d = evaluate(y, np.column_stack([1 - p1, p1]), labels, reference=0) metrics[split], details[split] = {target: m}, {target: d} curve[split] = threshold_table(y, p1, CURVE_THRESHOLDS) brier[split] = { @@ -924,7 +1004,7 @@ def score(fitted, X: np.ndarray, y: np.ndarray) -> tuple[dict[str, float], dict[ def fit_score(rows: np.ndarray, held: np.ndarray) -> dict: # Each fold's model is calibrated on the validation split too, so its scores are on the run's own scale. - return {target: score(fit(X_tr[rows], y_tr[rows])[1], X_tr[held], y_tr[held])[0]} + return {target: score(fit(_rows(X_tr, rows), y_tr[rows])[1], _rows(X_tr, held), y_tr[held])[0]} cross_validation = _cross_validate( splits, @@ -976,6 +1056,7 @@ def fit_score(rows: np.ndarray, held: np.ndarray) -> dict: "n_features": len(feats), "fit_seconds": round(fit_seconds, 1), "device": device, + "effective_device": used, "params": {k: (list(v) if isinstance(v, tuple) else v) for k, v in {**spec.default_params, **params}.items()}, "metrics": metrics, "evaluation": details, diff --git a/api/app/paths.py b/api/app/paths.py index 6942413c..5c66e6ac 100644 --- a/api/app/paths.py +++ b/api/app/paths.py @@ -12,8 +12,8 @@ log = logging.getLogger(__name__) -Kind = Literal["datasets", "exports", "compositions", "models"] -KINDS: tuple[str, ...] = ("datasets", "exports", "compositions", "models") +Kind = Literal["datasets", "exports", "compositions", "models", "studies"] +KINDS: tuple[str, ...] = ("datasets", "exports", "compositions", "models", "studies") # `notebooks` is the project's shared notebook folder, written by Labs (spawner.LAB_UID), so it is not a mounted kind. PROJECT_FOLDERS = (*KINDS, "notebooks") @@ -43,6 +43,11 @@ def run_dir(project_id: int, run_id: int) -> Path: return project_dir(project_id) / "models" / str(run_id) +def study_dir(project_id: int, batch_id: int) -> Path: + """A tuning study's journal and files: one folder per batch.""" + return project_dir(project_id) / "studies" / str(batch_id) + + def notebook_folder(project_id: int) -> Path: return project_dir(project_id) / "notebooks" diff --git a/api/app/processing.py b/api/app/processing.py index 58827a4d..6ea77c9f 100644 --- a/api/app/processing.py +++ b/api/app/processing.py @@ -156,6 +156,11 @@ def process_dataset(dataset_id: int, upload_path: str | None, fmt: str, job: Job upsert_correlations(conn, dataset_id, [(field_ids[a.i], field_ids[a.j], a) for a in screened]) refresh_target_state(conn, dataset_id) _apply_pending_recipe(dataset_id) + if fmt == "composition": + from . import field_about + + with connection() as conn: + field_about.copy_from_composition(conn, dataset_id) st.set("ready", 1.0, "Ready", force=True) execute("UPDATE meta.datasets SET profiled_at = now() WHERE id = %s", (dataset_id,)) diff --git a/api/app/projects.py b/api/app/projects.py index 3c8596fb..76091f0c 100644 --- a/api/app/projects.py +++ b/api/app/projects.py @@ -183,7 +183,7 @@ def audit( ) -AuditEvent = Literal["baseline.export", "baseline.import"] +AuditEvent = Literal["baseline.export", "baseline.import", "baseline.self_finalize"] def audit_event( @@ -1197,6 +1197,7 @@ def _folders(kind: str, object_id: int, rows: dict[str, list[int]]) -> list[tupl [("datasets", object_id)] + [("exports", i) for i in rows["exports"]] + [("models", i) for i in rows["model_runs"]] + + [("studies", i) for i in rows["run_batches"]] ) return [("compositions", object_id)] if kind == "composition" else [] diff --git a/api/app/recipes.py b/api/app/recipes.py index 558ac32d..fe0244cf 100644 --- a/api/app/recipes.py +++ b/api/app/recipes.py @@ -19,7 +19,7 @@ FAMILY_LABEL = {"tree": "Trees", "linear": "Linear", "neural": "Neural"} NAME_MAX = 60 RECIPE_COLS = """id, dataset_id, family, version, phase, scope, parent_id, created_by, created_at, finalized_by, - finalized_at, meta.baseline_name(dataset_id, family) AS name""" + finalized_at, derived_from_run_id, meta.baseline_name(dataset_id, family) AS name""" # Joins a dataset row `d` to its current (latest started) baseline version as `rc`. CURRENT_JOIN = """LEFT JOIN LATERAL (SELECT id, family, version, phase, scope, meta.baseline_name(dataset_id, family) AS name FROM meta.recipes diff --git a/api/app/registry/bundle.py b/api/app/registry/bundle.py index 55cab8fb..16c7b56d 100644 --- a/api/app/registry/bundle.py +++ b/api/app/registry/bundle.py @@ -1,7 +1,8 @@ """A version's bundle (docs/project/phase-4-serving/design.md §4): everything scoring needs, so the version keeps working after its run, export and dataset are deleted. Nothing in it points at `/data/exports/` or a run's folder. - bundle.json format 1: export_id, n_features, inputs (as scoring.Input), derived (the closure + bundle.json format 1: export_id, n_features, feature_names, category_levels (native + categoricals' levels), inputs (as scoring.Input), derived (the closure scoring._frozen computes), fields (the manifest's FieldSpecs), target_classes, models (as the version's column, each with model_file, report_file and sha256), preprocessor (file, sha256), input_schema, created_at @@ -10,7 +11,8 @@ reference.parquet a seeded sample of at most scoring.TRAINING_SAMPLE raw training rows of the inputs (R10), for PSI and the schema's enums target.json how labels are read without the export or meta.fields: the target's name, kind, dtype, - decimal comma, bins, classes, class policy, and the dataset's identifier fields + decimal comma, bins, classes, class policy, null tokens, and the dataset's identifier + fields schema.json the JSON Schema of one record `scoring.Scorer.from_bundle` reads it back; it is immutable once its version is ready. @@ -79,6 +81,8 @@ def write(conn, version: dict, folder: Path, progress=None) -> dict: "format": scoring.BUNDLE_FORMAT, "export_id": export_id, "n_features": int(manifest["n_features"]), + "feature_names": manifest["feature_names"], + "category_levels": manifest.get("category_levels") or {}, "inputs": [ { "name": i.name, @@ -86,6 +90,7 @@ def write(conn, version: dict, folder: Path, progress=None) -> dict: "datetime_format": i.datetime_format, "decimal_comma": i.decimal_comma, "integer_codes": i.integer_codes, + "null_tokens": list(i.null_tokens), } for i in inputs ], @@ -139,6 +144,7 @@ def target_reading(conn, export_id: int, target: str, kind: str) -> dict: "bins": how["bins"], "classes": how["classes"], "policy": how["policy"], + "null_tokens": list(how["null_tokens"]), "identifiers": [r["name"] for r in identifiers], } ) @@ -180,7 +186,7 @@ def schema_of( prop = _json_type(i) if semantic.get(i.name): prop["x-semantic-type"] = semantic[i.name] - values = scoring.missing_as_null(reference[i.name]) if i.name in reference.columns else None + values = scoring.missing_as_null(reference[i.name], i.null_tokens) if i.name in reference.columns else None if values is not None and semantic.get(i.name) in CATEGORICAL and prop["type"] != "boolean": levels = values.drop_nulls().unique().sort() if 0 < levels.len() <= ENUM_MAX: diff --git a/api/app/reuse.py b/api/app/reuse.py index 2ca61972..9598b8a9 100644 --- a/api/app/reuse.py +++ b/api/app/reuse.py @@ -9,7 +9,7 @@ from fastapi import HTTPException -from . import access, recipes, shape +from . import access, field_about, recipes, shape from .db import jsonb from .semantics import target_problem, validate_decision @@ -144,7 +144,7 @@ def apply( suggestions when `own` is None. The version's parent is `own`, else the source. Call holding the recipe row lock.""" snap = conn.execute("SELECT snapshot FROM meta.recipes WHERE id = %s", (source["id"],)).fetchone()["snapshot"] ds = conn.execute("SELECT name FROM meta.datasets WHERE id = %s", (source["dataset_id"],)).fetchone() - return apply_snapshot( + summary, added, uncorrelated = apply_snapshot( conn, recipe, snap, @@ -161,6 +161,18 @@ def apply( "name": source["name"], }, ) + # Descriptions and notes belong to the fields: fill what this dataset has none of from the source's, by name. + texts = { + r["name"]: {"description": r["description"], "field_notes": r["field_notes"]} + for r in conn.execute( + """SELECT name, description, field_notes FROM meta.fields + WHERE dataset_id = %s AND (description IS NOT NULL OR field_notes IS NOT NULL)""", + (source["dataset_id"],), + ).fetchall() + } + summary["about"] = field_about.fill_missing(conn, recipe["dataset_id"], texts, user_id) + conn.execute("UPDATE meta.recipes SET reuse = %s WHERE id = %s", (jsonb(summary), recipe["id"])) + return summary, added, uncorrelated def apply_snapshot( diff --git a/api/app/routers/baselines.py b/api/app/routers/baselines.py index 8e8b94d4..874a1e1d 100644 --- a/api/app/routers/baselines.py +++ b/api/app/routers/baselines.py @@ -11,7 +11,7 @@ from starlette.concurrency import run_in_threadpool from starlette.datastructures import UploadFile -from .. import access, exchange, projects, recipes, reuse +from .. import access, exchange, field_about, projects, recipes, reuse from ..assessment import require_assessors, seed from ..auth import CurrentUser from ..dataset_work import queue_target_work @@ -37,7 +37,7 @@ def list_baselines(dataset_id: int, user: CurrentUser) -> list[dict]: p.version AS parent_version, p.dataset_id AS parent_dataset_id, r.created_at, r.finalized_at, meta.baseline_name(r.dataset_id, r.family) AS name, meta.baseline_name(p.dataset_id, p.family) AS parent_name, pd.name AS parent_dataset_name, - r.reuse, CASE WHEN r.scope = 'reopened' + r.reuse, r.derived_from_run_id, CASE WHEN r.scope = 'reopened' THEN (SELECT count(*) FROM meta.recipe_reopened o WHERE o.recipe_id = r.id) END AS reopened, cu.name AS created_by_name, fu.name AS finalized_by_name FROM meta.recipes r @@ -224,7 +224,7 @@ def export_baseline( user: CurrentUser, format: Literal["json", "csv"] = "json", ) -> Response: - """Download a finalized baseline version as `rainforest-baseline/1` JSON (lossless) or as a CSV with one row per + """Download a finalized baseline version as `rainforest-baseline/2` JSON (lossless) or as a CSV with one row per field. Needs read access to the version, so a project that borrows the baseline can export it too. A version still being decided is a 409. The export is recorded in the audit log of the baseline's home project.""" with connection() as conn: @@ -279,7 +279,7 @@ def export_baseline( @router.post("/datasets/{dataset_id}/baselines/import", openapi_extra=IMPORT_BODY) async def import_baseline(dataset_id: int, request: Request, user: CurrentUser, dry_run: bool = True) -> dict: - """Read a baseline file (`file`: `rainforest-baseline/1` JSON or a CSV; `family` for a CSV that lost its first + """Read a baseline file (`file`: `rainforest-baseline/2` JSON or a CSV; `family` for a CSV that lost its first line; `assessors`: comma-separated user ids). Needs write access to the dataset, which must be ready. With `dry_run` (the default) it returns the report of what would carry over and what wouldn't, and why, and changes nothing. Otherwise it opens the file's model type's next version (or, on a dataset whose first baseline is still @@ -392,6 +392,7 @@ def plan_and_report(fields: list[dict]) -> tuple[dict, dict]: blocked=blocked, version=version, ) + rep["about"] = field_about.plan_fill(conn, dataset_id, parsed.about)[1] return snap, rep snap, rep = plan_and_report(fields) @@ -426,6 +427,7 @@ def plan_and_report(fields: list[dict]) -> tuple[dict, dict]: ) if not setup: added, uncorrelated = reuse.merge_target_work(conn, dataset_id, first, added, uncorrelated) + rep["about"] = field_about.fill_missing(conn, dataset_id, parsed.about, user["id"]) recipe = recipes.get(conn, recipe["id"]) work = _open(conn, user, recipe, ids, name, uncorrelated, added) state = _team_state(conn, dataset_id, user) diff --git a/api/app/routers/batches.py b/api/app/routers/batches.py index 5b3daedc..1d97042a 100644 --- a/api/app/routers/batches.py +++ b/api/app/routers/batches.py @@ -3,21 +3,24 @@ from __future__ import annotations import itertools +from importlib.metadata import version from pathlib import Path from typing import Any import polars as pl from fastapi import APIRouter, HTTPException -from .. import access, batches, experiments, jobs +from .. import access, batches, experiments, jobs, tuning from ..auth import CurrentUser from ..config import get_settings from ..db import connection, fetch_all, fetch_one, jsonb from ..locks import refuse_while_refreshing from ..modeling import params as model_params -from ..modeling.registry import MODELS, class_target, no_valid_problem +from ..modeling.registry import MODELS, class_target, no_valid_problem, other_class from ..schemas import BatchRequest, RunRequest, Sampler -from .runs import Checked, check_model, check_params, export_manifest, insert_run +from ..tuning import space as tune_space +from ..tuning import study as tune_study +from .runs import Checked, check_device, check_model, check_params, export_manifest, insert_run, run_row router = APIRouter(prefix="/api", tags=["batches"]) @@ -57,12 +60,6 @@ def _class_counts(exp: dict, manifest: dict, target: str) -> batches.ClassCounts return counts -def other_class(manifest: dict, target: str | None) -> str | None: - """The label the target's class policy gave the classes it merged, when it merged any.""" - p = (manifest.get("target_policy") or {}).get(target or "") or {} - return p.get("other") if p.get("collapsed") else None - - def _search_classes(req: BatchRequest, counts: batches.ClassCounts, min_pos: int, other: str | None) -> list[str]: if req.search_classes is None: chosen = batches.search_classes_default(counts, min_pos, other) @@ -93,6 +90,11 @@ def batch_plan(export_id: int, user: CurrentUser, target: str | None = None) -> exp, manifest = export_manifest(export_id, user) settings = get_settings() out: dict[str, Any] = {"min_positives": settings.ovr_min_positives, "max_runs": settings.batch_max_runs} + out["tune"] = { + "max_trials": {"cpu": settings.tune_max_trials, "cuda": settings.tune_max_trials_gpu}, + "max_hours": {"cpu": tuning.MAX_HOURS_CPU, "cuda": settings.tune_max_hours_gpu}, + "train_rows": (manifest.get("split_counts") or {}).get("train", 0), + } out.update(dict.fromkeys(("target", "classes", "valid_rows", "skipped", "search_classes", "other"))) if target is None: return out @@ -141,6 +143,8 @@ def start_batch(export_id: int, req: BatchRequest, user: CurrentUser) -> dict: x = experiments.get_visible(conn, exp["experiment_id"], user) experiments.require_writable(x, user) experiments.require_source_shared(conn, x) + if req.kind == "tune": + return _start_study(exp, manifest, req, user) settings = get_settings() base = { "model": req.model, @@ -281,6 +285,90 @@ def problem_of(combo: dict) -> str | None: } +def _start_study(exp: dict, manifest: dict, req: BatchRequest, user: dict) -> dict: + """A tuning study: its first run request checked as a single run's, its metric, budget and space checked, then the + batch and its one `run.tune` job. No `model_runs` row until the job refits the best trial.""" + t = req.tune + assert t is not None + one_vs_rest = t.positive_class is not None + run_req = RunRequest( + model=req.model, + device=req.device, + seed=req.seed, + cv=req.cv, + cv_folds=req.cv_folds, + order=req.order, + params=req.params, + balance=req.balance, + **({"positive_class": t.positive_class, "target": t.target} if one_vs_rest else {}), + ) + checked = check_model(exp, manifest, run_req) + check_params(checked, manifest, run_req) + model, base = checked.spec.key, run_req.params + if one_vs_rest: + target, kind = run_req.target, "binary" + else: + target = t.target or checked.targets[0] + if target not in checked.targets: + raise HTTPException(422, f"{target} isn't a target this study trains ({', '.join(checked.targets)})") + kind = checked.kinds[checked.targets.index(target)] + metric = req.ranking or batches.default_ranking(kind) + if problem := batches.ranking_problem(metric, kind): + raise HTTPException(422, problem) + if problem := tuning.ceiling_problem(req.device, t.trials, t.seconds): + raise HTTPException(422, problem) + chain = len(checked.targets) > 1 + if req.space is not None: + try: + tune_space.check(model, req.space, base) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + space = req.space + else: + space = tune_space.default_space(model, chain=chain, base=base, kinds=checked.kinds) + early = tuning.early_stopping(model, chain=chain) + pruned = tuning.pruning(model, chain=chain, evaluation=t.evaluation, pruner=t.pruner) + _, hours = tuning.limits(req.device) + spec = { + "model": model, + "device": req.device, + "params": base, + "space": space, + "fixed": tune_space.fixed(model, space, early_stopping=early), + "early_stopping": early, + "metric": metric, + "direction": "minimize" if metric in batches.LOWER_IS_BETTER else "maximize", + "target": target, + "positive_class": t.positive_class, + "balance": req.balance, + "order": None if one_vs_rest else checked.targets, + "cv": req.cv, + "cv_folds": req.cv_folds, + "evaluation": { + "mode": t.evaluation, + "folds": t.folds if t.evaluation == "cv" else None, + # Folds keep the export's groups whole, as a run's cross-validation does. + "groups": t.evaluation == "cv" and bool(manifest["files"].get("groups")), + }, + "pruner": t.pruner if pruned else "none", + "sampler": t.sampler, + "seed": t.seed, + "budget": {"trials": t.trials, "seconds": t.seconds or hours * 3600, "patience": t.patience}, + "threads": None, # the job's, recorded when it starts + "optuna": version("optuna"), + "run_seed": req.seed, + } + name = (req.name or "").strip() or f"{checked.spec.label} · tuning study" + with connection() as conn: + refuse_while_refreshing(conn, exp["dataset_id"]) + tuning.hold_experiment(conn, exp["experiment_id"]) + x = experiments.get_visible(conn, exp["experiment_id"], user) + experiments.require_writable(x, user) + experiments.require_source_shared(conn, x) + batch_id = tuning.queue(conn, exp, name, spec, user["id"]) + return {"batch": _with_counts([_batch_row(batch_id)])[0], "run_ids": [], "skipped": [], "redraws": 0} + + def _batch_row(batch_id: int, user: dict | None = None) -> dict | None: visible = "" if user is None else f" AND b.experiment_id IN ({experiments.visible_ids(user['id'])})" return fetch_one(f"SELECT {BATCH_COLS} FROM {BATCH_FROM} WHERE b.id = %s{visible}", (batch_id,)) @@ -317,8 +405,12 @@ def list_batches(experiment_id: int, user: CurrentUser) -> list[dict]: @router.get("/batches/{batch_id}") def get_batch(batch_id: int, user: CurrentUser) -> dict: - """The batch, its runs' counts by status, a row per run, and for a search its leaderboard.""" + """The batch, its runs' counts by status, a row per run, and for a search its leaderboard; a tuning study's has its + `study` (settings, budget, trial counts, best so far) instead of rows (its trials are GET /batches/{id}/trials).""" batch = _with_counts([_visible_batch(batch_id, user)])[0] + if batch["kind"] == "tune": + with connection() as conn: + return {**batch, "study": tune_study.view(conn, batch)} runs = fetch_all( f"""SELECT id, status, status_message, error, config, metrics, {SUMMARY_REPORT} AS report, finished_at FROM meta.model_runs WHERE batch_id = %s ORDER BY id""", @@ -335,12 +427,20 @@ def get_batch(batch_id: int, user: CurrentUser) -> dict: @router.post("/batches/{batch_id}/cancel") def cancel_batch(batch_id: int, user: CurrentUser) -> dict: - """Cancel the batch's waiting runs; a run already training finishes.""" + """Cancel the batch's waiting runs; a run already training finishes. A tuning study stops as its job's Stop does: + a queued one ends cancelled, a running one ends its trial and refits the best so far.""" access.load_batch(batch_id, user, "write") batch = _visible_batch(batch_id, user) with connection() as conn: refuse_while_refreshing(conn, batch["dataset_id"]) experiments.require_writable(experiments.get_visible(conn, batch["experiment_id"], user), user) + if batch["kind"] == "tune": + try: + tune_study.request_stop(conn, batch_id, user["id"]) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + conn.commit() + return get_batch(batch_id, user) status = conn.execute("SELECT status FROM meta.run_batches WHERE id = %s FOR UPDATE", (batch_id,)).fetchone() if status["status"] != "running": raise HTTPException(409, f"The batch is already {status['status']}") @@ -356,3 +456,61 @@ def cancel_batch(batch_id: int, user: CurrentUser) -> dict: if job and job["status"] in ("queued", "paused"): jobs.request_stop(conn, r["job_id"], "cancel", user["id"]) return get_batch(batch_id, user) + + +def _study(batch_id: int, user: dict) -> dict: + """A tuning study's batch, after the access gate: 409 for a batch of another kind.""" + batch = _visible_batch(batch_id, user) + if batch["kind"] != "tune": + raise HTTPException(409, f"Batch #{batch_id} isn't a tuning study.") + return batch + + +@router.get("/batches/{batch_id}/trials") +def list_trials(batch_id: int, user: CurrentUser) -> list[dict]: + """A tuning study's trials, by number: state, score, the settings tried, the scores reported on the way, tree + counts, the round it was pruned at, error, and the run made from it.""" + _study(batch_id, user) # the access gate, then the kind check + with connection() as conn: + return tune_study.trials(conn, batch_id) + + +@router.post("/batches/{batch_id}/trials/{number}/run", status_code=202) +def run_trial(batch_id: int, number: int, user: CurrentUser) -> dict: + """Make a run from a complete trial (a runner-up worth keeping): its settings refit as the study's best is.""" + access.load_batch(batch_id, user, "write") + batch = _visible_batch(batch_id, user) + if batch["kind"] == "tune": # a refit on the GPU needs one live, as a run does (the device the study is on now) + check_device(MODELS[batch["spec"]["model"]], batch["spec"]["device"]) + with connection() as conn: + refuse_while_refreshing(conn, batch["dataset_id"]) + x = experiments.get_visible(conn, batch["experiment_id"], user) + experiments.require_writable(x, user) + if batch["kind"] != "tune": + raise HTTPException(409, f"Batch #{batch_id} isn't a tuning study.") + experiments.require_source_shared(conn, x) + found = tune_study.trial(conn, batch_id, number) + if found is None: + raise HTTPException(404, f"Trial {number} not found") + if found["state"] != "complete": + raise HTTPException( + 409, f"Trial {number} is {found['state']}; only a complete trial can be made into a run." + ) + run = tune_study.refit(conn, batch, number, user["id"]) + return run_row(run["id"], user["id"]) + + +@router.post("/batches/{batch_id}/again", status_code=202) +def run_again(batch_id: int, user: CurrentUser) -> dict: + """The same tuning study again on the same export: the request rebuilt from its stored spec (the mapping a re-run + study uses), with as many trials as it reached and no time limit but the device's ceiling. For a study that has + ended done and reached at least one trial.""" + from ..derive import study_request + + access.load_batch(batch_id, user, "write") + batch = _study(batch_id, user) + with connection() as conn: + reached = tune_study.view(conn, batch)["trials_reached"] + if batch["status"] != "done" or not reached: + raise HTTPException(409, f"Study #{batch_id} hasn't finished with any trials to repeat.") + return start_batch(batch["export_id"], study_request(batch["spec"], trials=reached, seconds=None), user) diff --git a/api/app/routers/datasets.py b/api/app/routers/datasets.py index 4313e279..0b817c7e 100644 --- a/api/app/routers/datasets.py +++ b/api/app/routers/datasets.py @@ -881,6 +881,9 @@ def list_fields(dataset_id: int, user: CurrentUser, view: str | None = None) -> rows = conn.execute( sql.SQL("""SELECT f.id, f.ordinal, f.name, f.inferred_type, f.inference_confidence, f.is_target, f.null_pct, f.n_unique, f.max_target_assoc, f.leakage_flag AS leakage, + f.description, f.field_notes, f.about_at, + (SELECT jsonb_build_object('id', u.id, 'name', u.name) + FROM meta.users u WHERE u.id = f.about_by) AS about_by, d.action, d.semantic_type, d.reviewed, d.source, NOT (%s OR f.id = ANY(%s)) AS locked FROM meta.fields f JOIN {} d ON d.field_id = f.id diff --git a/api/app/routers/derive.py b/api/app/routers/derive.py new file mode 100644 index 00000000..e6e8946c --- /dev/null +++ b/api/app/routers/derive.py @@ -0,0 +1,65 @@ +"""Drop fields a run ranked low into a new baseline version (`app/derive.py`).""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from .. import access, derive, experiments +from ..auth import CurrentUser +from ..db import connection +from ..locks import refuse_while_refreshing +from ..schemas import DeriveIn, RerunIn + +router = APIRouter(prefix="/api", tags=["derive"]) + + +def _visible(conn, run: dict, user: dict) -> None: + """404, as GET /runs/{id} answers, for a run whose experiment the user may not see (another assessor's personal + one, a draft hidden while its version is assessed): before anything else, so nothing tells it exists.""" + try: + experiments.get_visible(conn, run["experiment_id"], user) + except HTTPException as exc: + if exc.status_code == 404: + raise HTTPException(404, "Run not found") from None + raise + + +@router.post("/runs/{run_id}/derive-baseline") +def derive_baseline(run_id: int, body: DeriveIn, user: CurrentUser) -> dict: + """Start the next version of the run's baseline with `field_ids` dropped, or, while the run's own version is in + progress, record the drops to make when it is finalized. Needs the contributor role: the re-run that follows + creates an experiment, an export and runs. A dry run reports what would happen.""" + run = access.load_run(run_id, user, "write") + with connection() as conn: + _visible(conn, run, user) + if access.load_dataset(run["dataset_id"], user, "read", conn=conn)["borrowed"]: + raise HTTPException( + 409, + f"This dataset is lent to {run['project']['name']}; a borrower can't make baseline versions. " + "Drop the fields in this experiment's adjustments instead.", + ) + ds = access.load_dataset(run["dataset_id"], user, "write", conn=conn, lock="update") + refuse_while_refreshing(conn, ds["id"]) + if ds["status"] != "ready": + raise HTTPException(409, "The dataset is still processing") + if run["status"] != "done": + raise HTTPException(409, f"Run #{run_id} isn't finished.") + result, payload = derive.derive(conn, run, user, body) + if payload is not None: + derive.write_audit(ds, [payload]) + return result + + +@router.post("/runs/{run_id}/rerun", status_code=202) +def rerun(run_id: int, body: RerunIn, user: CurrentUser) -> dict: + """Re-run a run on a version derived from it (or on the drops still waiting for its version): a new experiment, an + export of it, and then, as it was, the run again; or, for *customise*, nothing more (the user trains it); or, for + *study*, the tuning study again. The run starts when the export is done.""" + run = access.load_run(run_id, user, "write") + with connection() as conn: + _visible(conn, run, user) + ds = access.load_dataset(run["dataset_id"], user, "write", conn=conn, lock="update") + refuse_while_refreshing(conn, ds["id"]) + if ds["status"] != "ready": + raise HTTPException(409, "The dataset is still processing") + return derive.rerun(conn, run, user, body) diff --git a/api/app/routers/experiments.py b/api/app/routers/experiments.py index 118656dd..d3f58b5c 100644 --- a/api/app/routers/experiments.py +++ b/api/app/routers/experiments.py @@ -60,6 +60,7 @@ def get_experiment(experiment_id: int, user: CurrentUser) -> dict: "best": best, "adjustments": adjustments, "can_adjust": experiments.can_adjust(x, user) and not blind, + "adjust_denied": experiments.adjust_denied_reason(x, user), "rebase_available": rebase_available, "exports": exports_of(x["dataset_id"], user, x["id"]), "runs": runs_of(x["dataset_id"], user, x["id"]), @@ -84,6 +85,7 @@ def revert_adjustments(experiment_id: int, body: RevertIn, user: CurrentUser) -> **experiments.shaped(conn, x, user), "adjustments": experiments.adjustments(conn, x), "can_adjust": True, + "adjust_denied": None, "rebase_available": experiments.rebase_available(conn, x), } diff --git a/api/app/routers/exports.py b/api/app/routers/exports.py index caacc7f8..aec7f124 100644 --- a/api/app/routers/exports.py +++ b/api/app/routers/exports.py @@ -5,6 +5,7 @@ import logging import time import traceback +from contextlib import nullcontext from functools import lru_cache from pathlib import Path @@ -466,7 +467,25 @@ def start_export(dataset_id: int, req: ExportRequest, user: CurrentUser, request experiment in the `X-Rainforest-Project` project), or a personal export of the viewer's own assessment (into their personal experiment). It lives in its experiment's project and needs the contributor role there. What it is built from is frozen now: the version's snapshot, or the experiment's (with its adjustments).""" - with connection() as conn: + return start_export_for(dataset_id, req, user, request) + + +def start_export_for( + dataset_id: int, + req: ExportRequest, + user: dict, + request: Request | None, + *, + then_run: dict | None = None, + conn=None, +) -> dict: + """A team export of a finalized version (into the experiment given, else the version's default Team + experiment in the `X-Rainforest-Project` project), or a personal export of the viewer's own assessment (into + their personal experiment). It lives in its experiment's project and needs the contributor role there. What it is + built from is frozen now: the version's snapshot, or the experiment's (with its adjustments). `then_run` is what the + export does once it is done (a re-run's run or study, `derive.export_done`), recorded with the export row. With + `conn`, everything is written in the caller's transaction; `request` may then be None (an experiment is given).""" + with nullcontext(conn) if conn is not None else connection() as conn: if req.experiment_id is not None: # The experiment decides the project; its dataset may have stopped being shared into it, which is a 409. early = _experiment(conn, dataset_id, req.experiment_id, user, req.recipe_id) @@ -556,8 +575,9 @@ def start_export(dataset_id: int, req: ExportRequest, user: CurrentUser, request config["pull_id"] = pull["id"] exp = conn.execute( """INSERT INTO meta.exports (project_id, dataset_id, status, status_message, model_family, config, - created_by, recipe_id, experiment_id, snapshot, pull_id, draft, split_plan_id) - VALUES (%s, %s, 'pending', 'Queued', %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""", + created_by, recipe_id, experiment_id, snapshot, pull_id, draft, split_plan_id, + then_run) + VALUES (%s, %s, 'pending', 'Queued', %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id""", ( x["project_id"], dataset_id, @@ -570,6 +590,7 @@ def start_export(dataset_id: int, req: ExportRequest, user: CurrentUser, request None if pull is None else pull["id"], bool(x["draft"]), req.split_plan_id, + None if then_run is None else jsonb(then_run), ), ).fetchone() _record_sources(conn, exp["id"], plan) diff --git a/api/app/routers/fields.py b/api/app/routers/fields.py index c30622a6..99acc37a 100644 --- a/api/app/routers/fields.py +++ b/api/app/routers/fields.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, HTTPException, Query from psycopg import sql -from .. import access, bands, experiments, jobkinds, recipes +from .. import access, bands, bulk_apply, experiments, field_about, jobkinds, recipes from ..assessment import core_key, lock_decisions, mark_decided, source_for, write_decision, write_settings from ..assessment import restore as restore_assessment from ..auth import CurrentUser @@ -19,27 +19,21 @@ _family, _field, _jsonable, - _jsonb_or_null, _merged, _merged_many, - _provenance, _remember_datetime_format, _same_settings, - _snapshot, _sync_targets, _with_stored_task, _write_history, class_levels, has_screen_fields, - plan_bulk, source_targets, - summarize_bulk, ) from ..locks import refuse_while_refreshing from ..profiling import coerce_numeric -from ..schemas import BulkDecisionIn, DecisionIn +from ..schemas import BulkDecisionIn, DecisionIn, FieldAboutIn from ..semantics import _BASE_ALLOWED, FAMILIES, bins_problem, settings_of, suggest, validate_decision -from .query import fields_where router = APIRouter(prefix="/api", tags=["fields"]) @@ -52,6 +46,7 @@ def get_field(field_id: int, user: CurrentUser, family: str | None = None, view: f = _field(field_id) baseline = None with connection() as conn: + f["about_by"] = field_about.user_ref(conn, f["about_by"]) src = source_for(conn, f["dataset_id"], user, view=view) fam = _family(src, family) decision = _merged(conn, field_id, fam, src) @@ -151,17 +146,34 @@ def band_preview( @router.get("/fields/{field_id}/history") def history(field_id: int, user: CurrentUser) -> list[dict]: f = access.load_field(field_id, user) - if f["my_role"] in (None, "viewer"): - with connection() as conn: - if recipes.current(conn, f["dataset_id"])["phase"] == "assessing": - return [] # a version's work in progress, which viewers see only from team review - return fetch_all( - """SELECT h.changed_at, h.family, h.previous, h.current, h.batch_id, h.user_id, u.name AS user_name, - h.provenance - FROM meta.decision_history h LEFT JOIN meta.users u ON u.id = h.user_id - WHERE h.field_id = %s ORDER BY h.changed_at DESC LIMIT 50""", - (field_id,), - ) + with connection() as conn: + # A version's work in progress is what viewers see only from team review; the about entries stay visible. + hide = f["my_role"] in (None, "viewer") and recipes.current(conn, f["dataset_id"])["phase"] == "assessing" + about = field_about.history(conn, f["dataset_id"], f["name"]) + decisions = ( + [] + if hide + else [ + {"kind": "decision", **r} + for r in conn.execute( + """SELECT h.changed_at, h.family, h.previous, h.current, h.batch_id, h.user_id, u.name AS user_name, + h.provenance + FROM meta.decision_history h LEFT JOIN meta.users u ON u.id = h.user_id + WHERE h.field_id = %s ORDER BY h.changed_at DESC LIMIT 50""", + (field_id,), + ).fetchall() + ] + ) + return sorted(decisions + about, key=lambda e: e["changed_at"], reverse=True)[:50] + + +@router.patch("/fields/{field_id}/about") +def set_about(field_id: int, body: FieldAboutIn, user: CurrentUser) -> dict: + """A field's description and notes. Not a decision: no version phase locks it.""" + f = access.load_field(field_id, user, "review") + with connection() as conn: + refuse_while_refreshing(conn, f["dataset_id"]) + return field_about.update(conn, f, user["id"], body.model_dump(exclude_unset=True)) @router.put("/fields/{field_id}/decision") @@ -282,95 +294,7 @@ def bulk_decisions(dataset_id: int, body: BulkDecisionIn, user: CurrentUser, vie refuse_while_refreshing(conn, dataset_id) src = source_for(conn, dataset_id, user, lock="share", view=view) src.require_write() - fam = src.family - if body.filters is not None: - target = sql.SQL("d.field_id IN (SELECT field_id FROM {} WHERE {})").format( - src.fields_view(), fields_where(dataset_id, body.filters) - ) - else: - target = sql.SQL("d.field_id = ANY({})").format(sql.Literal(body.field_ids)) - # Fields carried over into this version are locked; bulk edits pass over them. - if src.kind in ("team", "assessment") and (open_ := recipes.open_field_ids(conn, src.recipe)) is not None: - target = sql.SQL("{} AND d.field_id = ANY({})").format(target, sql.Literal(open_)) - # Resolve and lock in one statement so the set can't drift between preview and write. An assessor's rows - # are only theirs to change, so they are locked right after. - rows = conn.execute( - sql.SQL("""SELECT d.field_id, f.name, d.action, d.semantic_type, d.reviewed, d.notes, f.suggestion - FROM {} d JOIN meta.fields f ON f.id = d.field_id - WHERE d.dataset_id = {} AND {} AND NOT d.field_id = ANY({}) - ORDER BY f.ordinal {}""").format( - src.decisions(), - sql.Literal(dataset_id), - target, - sql.Literal(body.excluded_ids), - sql.SQL("FOR UPDATE OF d" if src.team else ""), - ), - ).fetchall() - ids = [r["field_id"] for r in rows] - if not src.team: - lock_decisions(conn, src, ids) - settings: dict[int, dict[str, dict]] = {} - for srow in conn.execute( - sql.SQL("SELECT * FROM {} s WHERE field_id = ANY(%s)").format(src.settings()), (ids,) - ).fetchall(): - settings.setdefault(srow["field_id"], {})[srow["family"]] = srow - if src.kind == "experiment": - # An experiment's copy holds its baseline's model type only; other types' suggestions don't apply. - rows = [ - { - **r, - "suggestion": {src.family: (r["suggestion"] or {}).get(src.family)} - if (r["suggestion"] or {}).get(src.family) - else {}, - } - for r in rows - ] - planned = plan_bulk(rows, settings, body) - result = summarize_bulk(len(rows), planned) - batch_prov, field_prov = _provenance(conn, dataset_id, body, {r["field_id"]: r["name"] for r in rows}) - if body.dry_run or not planned: - conn.rollback() - return {"batch_id": None, "dry_run": body.dry_run, **result} - - batch_id = uuid.uuid4() - changed_ids = [c["field_id"] for c in planned] - if batch_prov is not None: - batch_prov["ranks"] = {str(i): field_prov[i]["rank"] for i in changed_ids} - snapshot = _snapshot(conn, src, changed_ids) - prev = _merged_many(conn, changed_ids, fam, src) - notes = {r["field_id"]: r["notes"] for r in rows} - for c in planned: - decided = any(c["after"][k] != c["before"][k] for k in ("action", "semantic_type")) - write_decision(conn, src, c["field_id"], {**c["after"], "notes": notes[c["field_id"]]}, decided) - for family, sugg in c["settings"].items(): - write_settings(conn, src, dataset_id, c["field_id"], family, sugg, "suggested") - summary = jsonb( - {**{k: v for k, v in result.items() if k != "changes"}, "request": body.model_dump(exclude={"dry_run"})} - ) - if src.personal: - conn.execute( - """INSERT INTO meta.assessment_batches (id, recipe_id, user_id, summary, snapshot, provenance) - VALUES (%s, %s, %s, %s, %s, %s)""", - (batch_id, src.recipe["id"], src.user_id, summary, jsonb(snapshot), _jsonb_or_null(batch_prov)), - ) - return {"batch_id": str(batch_id), "dry_run": False, **result} - if src.kind == "experiment": - conn.execute( - """INSERT INTO meta.experiment_batches (id, experiment_id, user_id, summary, snapshot, provenance) - VALUES (%s, %s, %s, %s, %s, %s)""", - (batch_id, src.experiment["id"], user["id"], summary, jsonb(snapshot), _jsonb_or_null(batch_prov)), - ) - return {"batch_id": str(batch_id), "dry_run": False, **result} - _write_history( - conn, dataset_id, fam, batch_id, prev, _merged_many(conn, changed_ids, fam), user["id"], field_prov - ) - conn.execute( - """INSERT INTO meta.bulk_batches (id, dataset_id, summary, snapshot, user_id, provenance) - VALUES (%s, %s, %s, %s, %s, %s)""", - (batch_id, dataset_id, summary, jsonb(snapshot), user["id"], _jsonb_or_null(batch_prov)), - ) - _sync_targets(conn, dataset_id, user_id=user["id"]) - return {"batch_id": str(batch_id), "dry_run": False, **result} + return bulk_apply.apply(conn, src, body, user["id"]) @router.post("/datasets/{dataset_id}/decisions/bulk/{batch_id}/undo") diff --git a/api/app/routers/jobs.py b/api/app/routers/jobs.py index 31fe72ea..d22e00a5 100644 --- a/api/app/routers/jobs.py +++ b/api/app/routers/jobs.py @@ -248,14 +248,27 @@ def _locked(conn: psycopg.Connection, job_id: int, user: dict) -> dict: return again -def _act(job_id: int, user: dict, action: str, do: Callable[[psycopg.Connection, dict], dict]) -> dict: - """Check and do one action in one transaction and answer the job's public shape.""" +def _act( + job_id: int, + user: dict, + action: str, + do: Callable[[psycopg.Connection, dict], dict], + refuse: Callable[[dict], None] | None = None, +) -> dict: + """Check and do one action in one transaction and answer the job's public shape. `refuse` reads the row under + its lock before the permission check and raises the HTTPException the row's kind or state calls for.""" with connection() as conn: try: if action == "promote" and user["role"] != "admin": row = _locked(conn, job_id, user) else: row = _row(conn, job_id, user) + if refuse is not None: + try: + row = {**row, **jobs.locked(conn, job_id)} + except ValueError as exc: # the job vanished after the read + raise HTTPException(404, "Job not found") from exc + refuse(row) _check(conn, row, user, action) done = do(conn, row) except ValueError as exc: # the status or priority changed under the request: as for the check @@ -301,6 +314,25 @@ def demote_job(job_id: int, user: CurrentUser) -> dict: return _act(job_id, user, "demote", lambda conn, row: jobs.demote(conn, job_id, user["id"])) +def _cpu_refusal(row: dict) -> None: + if row["kind"] not in ("run.train", "run.tune"): + raise HTTPException(422, "Only a training run or a tuning study can move to the CPU.") + if row["status"] != "queued": + raise HTTPException(409, f"Only a queued job can move to the CPU; this one is {row['status']}.") + if (row.get("needs") or {}).get("slot") != "gpu": + raise HTTPException(409, "This job already runs on the CPU.") + + +def _to_cpu(user: dict) -> Callable[[psycopg.Connection, dict], dict]: + return lambda conn, row: jobs.move_to_cpu(conn, row, user["id"]) + + +@router.post("/jobs/{job_id:int}/move-to-cpu", response_model=Job) +def move_job_to_cpu(job_id: int, user: CurrentUser) -> dict: + """Run a queued GPU job on the CPU instead: its slot and device change; nothing else does.""" + return _act(job_id, user, "move_to_cpu", _to_cpu(user), refuse=_cpu_refusal) + + class MoveIn(BaseModel): before: int | None = Field(None, description="Take this queued job's place") after: int | None = Field(None, description="Take the place just after this queued job") diff --git a/api/app/routers/lineage.py b/api/app/routers/lineage.py index a4ddc6ad..c32a7983 100644 --- a/api/app/routers/lineage.py +++ b/api/app/routers/lineage.py @@ -26,6 +26,7 @@ "batch", "plan", "connection", + "baseline", ] diff --git a/api/app/routers/recipes.py b/api/app/routers/recipes.py index d417a367..0a25bc0b 100644 --- a/api/app/routers/recipes.py +++ b/api/app/routers/recipes.py @@ -58,7 +58,7 @@ def versions(dataset_id: int, user: CurrentUser) -> list[dict]: rows = conn.execute( """SELECT r.id, r.dataset_id, r.family, r.version, r.phase, r.scope, r.parent_id, r.created_by, cu.name AS created_by_name, r.created_at, r.finalized_by, fu.name AS finalized_by_name, - r.finalized_at, r.reuse, r.start_summary, + r.finalized_at, r.reuse, r.start_summary, r.derived_from_run_id, CASE WHEN r.scope = 'reopened' THEN (SELECT count(*) FROM meta.recipe_reopened o WHERE o.recipe_id = r.id) END AS reopened, p.dataset_id AS parent_dataset_id, p.family AS parent_family, p.version AS parent_version, diff --git a/api/app/routers/runs.py b/api/app/routers/runs.py index 4cb0991c..2be2ce82 100644 --- a/api/app/routers/runs.py +++ b/api/app/routers/runs.py @@ -14,7 +14,7 @@ from fastapi.responses import FileResponse from pydantic import BaseModel, Field -from .. import access, batches, experiments, jobkinds, jobs, paths, registry +from .. import access, batches, experiments, gpu, jobkinds, jobs, paths, registry from ..auth import CurrentUser from ..batches import WAITING from ..config import get_settings @@ -30,6 +30,7 @@ catalog, compatibility, dry_fit_problem, + encode_fields, label_for, one_vs_rest_target, order_problem, @@ -49,7 +50,7 @@ COALESCE((SELECT draft FROM meta.exports e WHERE e.id = export_id), false) AS draft""" RUN_COLS = f"""id, project_id, export_id, dataset_id, model, family, status, status_message, error, config, metrics, report, created_at, updated_at, finished_at, mlflow_experiment_id, - mlflow_run_id, created_by, experiment_id, note, chosen_at, batch_id, + mlflow_run_id, created_by, experiment_id, note, chosen_at, batch_id, job_id, trial_number, (SELECT name FROM meta.users u WHERE u.id = chosen_by) AS chosen_by_name, {EXPORT_COLS}""" @@ -66,12 +67,17 @@ def _with_plans(rows: list[dict]) -> list[dict]: names = plan_source_names([r.get("export_config") for r in rows]) datasets = {r["dataset_id"] for r in rows if not (r.get("export_config") or {}).get("plan")} names |= experiments.source_names(datasets) + job_ids = [r["job_id"] for r in rows if r.get("job_id") is not None and r["status"] == "pending"] with connection() as conn: used = registry.where_used(conn, "run", [r["id"] for r in rows]) + slots = jobs.queued_slots(conn, job_ids) if job_ids else {} + waiting = jobs.gpu_waiting(conn) if "gpu" in slots.values() else None out = [] for row in rows: row = dict(row) row["used_by"] = used[row["id"]] + if waiting and slots.get(row.get("job_id")) == "gpu": + row["status_message"] = waiting config = row.pop("export_config", None) plan = plan_summary(config, names, names.get(("dataset", row["dataset_id"]))) out.append( @@ -98,8 +104,10 @@ def system() -> dict: browse_url = mlflow_log.browse_base() if enabled else None with connection() as conn: found, workers = jobs.live_gpu(conn), jobs.workers(conn) + live = next((w["threads"] for w in workers if w["alive"] and w["threads"] is not None), None) return { "gpu": found, + "cpu_threads": live, "workers": workers, "mlflow": {"enabled": enabled, "browseUrl": browse_url, "uiOpen": enabled and mlflow_log.ui_open()}, "serve": serve_health(), @@ -142,8 +150,16 @@ def models_for_export(export_id: int, user: CurrentUser) -> list[dict]: continue spec = MODELS[m["key"]] reason = None if spec.available else "not installed" - reason = reason or compatibility(spec, exp["model_family"], bool(manifest.get("contains_nan"))) - out.append({**m, "compatible": reason is None, "reason": reason}) + reason = reason or compatibility( + spec, + exp["model_family"], + bool(manifest.get("contains_nan")), + manifest.get("category_levels"), + manifest.get("feature_source_field"), + ) + out.append( + {**m, "compatible": reason is None, "reason": reason, "encode_fields": encode_fields(spec, manifest)} + ) return out @@ -157,6 +173,19 @@ class Checked: label: str +def check_device(spec: ModelSpec, device: str) -> None: + """422 for a device the model can't train on; 409 for `cuda` with no usable GPU slot.""" + if device not in spec.devices: + raise HTTPException(422, f"{spec.label} can't train on {device!r} (supports: {', '.join(spec.devices)})") + if device == "cuda": + with connection() as conn: + info = jobs.live_gpu(conn) + if not (info and info.get("available")): + raise HTTPException(409, f"No usable GPU: {info['reason']}" if info else "No job worker has a GPU") + if not info.get("slots"): + raise HTTPException(409, "No usable GPU: The GPU worker has no GPU slots (GPU_SLOTS=0)") + + def check_model(exp: dict, manifest: dict, req: RunRequest) -> Checked: """The checks of a run request that don't depend on its settings: the model, device and targets. Raises the run's 4xx; fills in `req.target` for a one-vs-rest run of the export's only target.""" @@ -169,16 +198,16 @@ def check_model(exp: dict, manifest: dict, req: RunRequest) -> Checked: raise HTTPException(422, f"Unknown model {req.model!r}") if not spec.available: raise HTTPException(422, f"{spec.label} is not installed") - reason = compatibility(spec, exp["model_family"], bool(manifest.get("contains_nan"))) + reason = compatibility( + spec, + exp["model_family"], + bool(manifest.get("contains_nan")), + manifest.get("category_levels"), + manifest.get("feature_source_field"), + ) if reason: raise HTTPException(422, reason) - if req.device not in spec.devices: - raise HTTPException(422, f"{spec.label} can't train on {req.device!r} (supports: {', '.join(spec.devices)})") - if req.device == "cuda": - with connection() as conn: - info = jobs.live_gpu(conn) - if not (info and info.get("available")): - raise HTTPException(409, f"No usable GPU: {info['reason']}" if info else "No job worker has a GPU") + check_device(spec, req.device) if req.positive_class is not None: req.target, problem = one_vs_rest_target(manifest, req.target, req.positive_class) if problem: @@ -236,32 +265,36 @@ def insert_run( user_id: int, batch_id: int | None = None, plan: dict | None = None, + trial_number: int | None = None, ) -> dict: """The run's row, `pending`. A batch's runs wait with no job until the batch queues them, and keep the batch's - `plan` for them (the class's positives per split, the searched settings) in `config.batch_plan`.""" + `plan` for them (the class's positives per split, the searched settings) in `config.batch_plan`. A tuning study's + refit (`trial_number`) is queued at once by its caller.""" # A run that isn't one-vs-rest stores the config it always has. + # The label names this run only (its report's model_label), so Train again doesn't copy it. stored = ( - {"experiment_id"} + {"experiment_id", "label"} if req.positive_class is not None - else {"experiment_id", "positive_class", "target", "balance"} + else {"experiment_id", "positive_class", "target", "balance", "label"} ) # Named for its targets from the start ("Ridge regression" for numbers), so a queued or failed run reads right. return conn.execute( f"""INSERT INTO meta.model_runs (project_id, export_id, dataset_id, model, family, status, status_message, - config, created_by, experiment_id, report, batch_id) - VALUES (%s, %s, %s, %s, %s, 'pending', %s, %s, %s, %s, %s, %s) RETURNING {RUN_COLS}""", + config, created_by, experiment_id, report, batch_id, trial_number) + VALUES (%s, %s, %s, %s, %s, 'pending', %s, %s, %s, %s, %s, %s, %s) RETURNING {RUN_COLS}""", ( exp["project_id"], exp["id"], exp["dataset_id"], checked.spec.key, checked.spec.family, - "Queued" if batch_id is None else WAITING, + "Queued" if batch_id is None or trial_number is not None else WAITING, jsonb({**req.model_dump(exclude=stored), **({"batch_plan": plan} if plan is not None else {})}), user_id, exp["experiment_id"], - jsonb({"model_label": checked.label}), + jsonb({"model_label": req.label or checked.label}), batch_id, + trial_number, ), ).fetchone() @@ -288,6 +321,7 @@ def start_run(export_id: int, req: RunRequest, user: CurrentUser) -> dict: args={"export_dir": exp["output_dir"], "request": req.model_dump()}, dataset_id=exp["dataset_id"], created_by=user["id"], + needs=jobkinds.needs_for(req.device), ) return _with_url(run) @@ -447,11 +481,17 @@ def progress(msg: str) -> None: ) try: - project_id = fetch_one("SELECT project_id FROM meta.model_runs WHERE id = %s", (run_id,))["project_id"] - out_dir = paths.run_dir(project_id, run_id) / f"{req['model']}_{int(time.time())}" + row = fetch_one("SELECT project_id, trial_number FROM meta.model_runs WHERE id = %s", (run_id,)) + out_dir = paths.run_dir(row["project_id"], run_id) / f"{req['model']}_{int(time.time())}" params = req.get("params") or {} if "n_jobs" in MODELS[req["model"]].default_params: params = {"n_jobs": jobs.cpu_threads(), **params} + if row["trial_number"] is not None and req["model"] == "lightgbm": + # A tuning study's refit trains as its trials did. Not in the run's config: the catalogue doesn't list + # these, so Train again and a re-run never see them; the report's params record them. + from ..tuning.objective import LIGHTGBM_REPEATABLE + + params = {**params, **LIGHTGBM_REPEATABLE} report = train( Path(export_dir), req["model"], @@ -467,6 +507,14 @@ def progress(msg: str) -> None: positive_class=req.get("positive_class"), balance=req.get("balance", False), ) + if req.get("label"): + # The run's own name (a tuning study's refit), in its saved report too. + report["model_label"] = req["label"] + saved = Path(report["model_dir"]) / "report.json" + saved.write_text(json.dumps({**json.loads(saved.read_text()), "model_label": req["label"]}, indent=2)) + report["hardware"] = gpu.hardware( + req.get("device", "cpu"), report.get("effective_device") or "cpu", job.worker_id if job else None + ) if mlflow_log.enabled(): with connection() as conn: # MLflow has no access control, so a private experiment's run is logged once review opens. diff --git a/api/app/routers/team.py b/api/app/routers/team.py index 5de1529f..ddcc7efa 100644 --- a/api/app/routers/team.py +++ b/api/app/routers/team.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, HTTPException -from .. import access, jobkinds, recipes +from .. import access, derive, jobkinds, recipes from ..assessment import ( Source, adopt, @@ -140,6 +140,7 @@ def _team_state(conn, dataset_id: int, user: dict) -> dict: if mine is None else {"submitted": mine["submitted_at"] is not None, "reviewed": mine["reviewed"], "total": mine["total"]}, "review": None if recipe["phase"] == "assessing" else _review_summary(conn, recipe), + "derived": derive.state(conn, recipe), } @@ -468,24 +469,58 @@ def add_comment(field_id: int, body: CommentIn, user: CurrentUser) -> list[dict] @router.post("/datasets/{dataset_id}/finalize") def finalize(dataset_id: int, body: FinalizeIn, user: CurrentUser) -> dict: """Freeze the team decision as this recipe version's snapshot. Disagreements nobody settled need - `accept_open`, which keeps their current team decision. Needs the owner role.""" + `accept_open`, which keeps their current team decision. Needs the owner role, except for the creator of a version + derived from a run (`derive.py`) while it differs from its parent only by drops: they may finalize it themselves.""" + audit: list[dict] = [] with connection() as conn: - ds = access.load_dataset(dataset_id, user, "own", conn=conn) + ds = access.load_dataset(dataset_id, user, "read", conn=conn) + pid = ds["project_id"] + # An owner finalizes. The creator of a version derived from a run may too, while it is drops-only. Who that is + # is decided from the version read under the lock, the one finalized: a finalize that waited on the lock may + # find a different version (a pending derivation made the next one). + owner = not ds["borrowed"] and access.can(user, pid, "own") + may_write = not ds["borrowed"] and access.can(user, pid, "write") + if not owner and not may_write: + access.require(user, pid, "own", borrowed=ds["borrowed"]) refuse_while_refreshing(conn, dataset_id) recipe = recipes.current(conn, dataset_id, lock="update") + creator = not owner and recipe["derived_from_run_id"] is not None and recipe["created_by"] == user["id"] + if not owner and not creator: + access.require(user, pid, "own") _require_phase(recipe, "review", doing="finalizing") - if ds["status"] != "ready": - raise HTTPException(409, "The dataset is still processing") - summary = _review_summary(conn, recipe) - open_ = summary["open"] - if open_ and not body.accept_open: + if creator and not derive.drops_only(conn, recipe): raise HTTPException( - 409, - { - "message": f"{open_} disagreement(s) are still open. Settle them, or confirm " - "to keep their current team decision.", - "open": open_, - }, + 403, + f"Only a project owner can finalize v{recipe['version']}: it changed in review beyond the drops " + "it was derived with.", ) - recipes.finalize(conn, recipe, user["id"], review_record(summary)) - return _team_state(conn, dataset_id, user) + if ds["status"] != "ready": + raise HTTPException(409, "The dataset is still processing") + if creator: + fields = [ + r["name"] + for r in conn.execute( + "SELECT f.name FROM meta.recipe_reopened o JOIN meta.fields f ON f.id = o.field_id " + "WHERE o.recipe_id = %s ORDER BY f.ordinal", + (recipe["id"],), + ).fetchall() + ] + audit = [derive.self_finalize(conn, recipe, user["id"], recipe["derived_from_run_id"], fields)] + else: + # (The creator path above has no disagreements to settle: a derived version has no assessors.) + summary = _review_summary(conn, recipe) + open_ = summary["open"] + if open_ and not body.accept_open: + raise HTTPException( + 409, + { + "message": f"{open_} disagreement(s) are still open. Settle them, or confirm " + "to keep their current team decision.", + "open": open_, + }, + ) + done = recipes.finalize(conn, recipe, user["id"], review_record(summary)) + audit = derive.after_finalize(conn, done, user) + state = _team_state(conn, dataset_id, user) + derive.write_audit(ds, audit) + return state diff --git a/api/app/schemas.py b/api/app/schemas.py index 0f3df13e..ed9447a9 100644 --- a/api/app/schemas.py +++ b/api/app/schemas.py @@ -23,6 +23,20 @@ class DatasetPatch(BaseModel): target_columns: list[str] | None = Field(None, max_length=50, description="Exactly these targets; [] for none") +class FieldAboutIn(BaseModel): + """A field's description (one paragraph) and notes; send either or both, empty text clears. The limits are + checked in field_about.update, which answers in the app's words.""" + + description: str | None = None + field_notes: str | None = None + + @model_validator(mode="after") + def _one(self) -> "FieldAboutIn": + if not self.model_fields_set & {"description", "field_notes"}: + raise ValueError("Send a description, field_notes or both") + return self + + class DecisionIn(BaseModel): family: str | None = Field(None, description="Model family these settings are for (default: dataset's)") action: Literal["keep", "drop", "target"] @@ -224,6 +238,7 @@ class RunRequest(BaseModel): ) target: str | None = Field(None, max_length=500, description="The class target of a one-vs-rest run") balance: bool = Field(False, description="One-vs-rest only: weight the classes by their counts in training") + label: str | None = Field(None, max_length=200, description="The run's name in place of its model's label") @model_validator(mode="after") def _check(self) -> RunRequest: @@ -240,8 +255,28 @@ class Sampler(BaseModel): seed: int = 0 +class TuneIn(BaseModel): + """A tuning study's budget, evaluation and search (spec 3).""" + + trials: int = Field(50, ge=1, description="Most trials; the device's ceiling caps it") + seconds: int | None = Field(3600, ge=60, description="Time limit; null: the device's ceiling") + patience: int | None = Field(None, ge=5, le=1000, description="Stop after this many trials without improvement") + evaluation: Literal["valid", "cv"] = "valid" + folds: int = Field(5, ge=3, le=10, description="Cross-validation on the training rows: its folds") + pruner: Literal["median", "none"] = "median" + sampler: Literal["tpe", "random"] = "tpe" + seed: int = Field(0, description="Seeds the sampler; each trial's model uses the run seed") + target: str | None = Field( + None, + max_length=500, + description="The target whose metric is tuned (a chain's; default its first), or the " + "class target of a one-vs-rest study", + ) + positive_class: str | None = Field(None, max_length=500, description="Tune one class of `target` against the rest") + + class BatchRequest(BaseModel): - kind: Literal["one_vs_rest", "search"] + kind: Literal["one_vs_rest", "search", "tune"] name: str | None = Field(None, max_length=200) model: str = "lightgbm" device: Literal["cpu", "cuda"] = "cpu" @@ -252,7 +287,9 @@ class BatchRequest(BaseModel): target: str | None = Field(None, max_length=500) classes: list[str] | None = Field(None, description="One-vs-rest: the classes to train (default: all)") balance: bool = False - space: dict[str, list | dict] | None = Field(None, description="Search: each setting's values or range") + space: dict[str, list | dict] | None = Field( + None, description="Search: each setting's values or range; tune: the search space (default: the model's)" + ) sampler: Sampler | None = None ranking: str | None = Field(None, max_length=100) search_classes: list[str] | None = Field(None, description="One-vs-rest search: the classes it is tuned on") @@ -260,12 +297,23 @@ class BatchRequest(BaseModel): order: list[str] | None = Field( None, description="Search on the whole target: the targets each run trains, in chain order (default: all)" ) + tune: TuneIn | None = Field(None, description="A tuning study's settings (kind tune only)") @model_validator(mode="after") def _check(self) -> BatchRequest: if self.cv == 1 or self.cv_folds == 1: raise ValueError("cv and cv_folds take 0 or at least 2 folds") - if self.kind == "one_vs_rest": + if (self.kind == "tune") != (self.tune is not None): + raise ValueError("tune applies to a tuning study, and a tuning study needs it") + if self.kind == "tune": + extra = [k for k in ("target", "classes", "sampler", "search_classes") if getattr(self, k) is not None] + ( + ["include_other"] if self.include_other else [] + ) + if extra: + raise ValueError(f"{', '.join(extra)} don't apply to a tuning study (its target and class are in tune)") + if self.order is not None and self.tune.positive_class is not None: + raise ValueError("A one-vs-rest study trains only its target; leave the order out") + elif self.kind == "one_vs_rest": extra = [ k for k in ("space", "sampler", "ranking", "search_classes", "order") if getattr(self, k) is not None ] @@ -424,6 +472,22 @@ class NewVersionIn(BaseModel): ) +class DeriveIn(BaseModel): + target: str = Field(min_length=1, max_length=500, description="The run's target the ranking is of") + rule: str = Field(min_length=1, max_length=200, description="The cut, in words: what the history records") + field_ids: list[int] = Field(max_length=5000, description="The fields to drop") + finalize_now: bool = Field(False, description="Finalize the new version as soon as it exists (drops only)") + dry_run: bool = Field(False, description="Report what would happen and change nothing") + + +class RerunIn(BaseModel): + mode: Literal["as_was", "customise", "study"] = Field( + description="as_was: train the run's request again; customise: only the export; study: tune again" + ) + recipe_id: int | None = Field(None, description="The version derived from this run") + pending_id: int | None = Field(None, description="The drops waiting for this run's version to be finalized") + + class ApplyRecipeIn(BaseModel): recipe_id: int @@ -1027,6 +1091,7 @@ class JobCan(BaseModel): demote: bool move: bool priority: bool + move_to_cpu: bool class Job(BaseModel): @@ -1058,6 +1123,7 @@ class Job(BaseModel): href: str | None project: ProjectRef | None can: JobCan + waiting: str | None = Field(None, description="Why a queued GPU job isn't starting: no live worker has a GPU slot") class JobEvent(BaseModel): @@ -1072,7 +1138,7 @@ class JobDetail(Job): log_tail: str -class GpuInfo(BaseModel): +class WorkerGpu(BaseModel): """`gpu.detect()` as a worker saw it.""" available: bool @@ -1083,6 +1149,13 @@ class GpuInfo(BaseModel): reason: str | None +class GpuInfo(WorkerGpu): + """The GPU a live worker sees, with that worker's GPU slot count and id.""" + + slots: int + worker: str + + class Worker(BaseModel): id: str hostname: str @@ -1091,7 +1164,8 @@ class Worker(BaseModel): alive: bool slots: dict[str, int] busy: dict[str, int] - gpu: GpuInfo | None + gpu: WorkerGpu | None + threads: int | None = Field(None, description="The threads one job of this worker may use (JOB_THREADS)") class MlflowInfo(BaseModel): @@ -1115,6 +1189,7 @@ class ServeInfo(BaseModel): class SystemInfo(BaseModel): gpu: GpuInfo | None + cpu_threads: int | None = Field(None, description="The first live worker's job threads") workers: list[Worker] mlflow: MlflowInfo serve: ServeInfo @@ -1647,6 +1722,7 @@ class StorageVolume(BaseModel): class Cleanable(BaseModel): exports: int scoring: int + studies: int = Field(description="Tuning studies' folders whose batch is gone") bytes: int = Field(description="Everything the next clean-up would remove, the old deployment folders included") @@ -1679,7 +1755,7 @@ class StorageReport(BaseModel): class CleanupItem(BaseModel): - kind: Literal["export", "scoring"] + kind: Literal["export", "scoring", "study"] id: int path: str bytes: int diff --git a/api/app/scoring.py b/api/app/scoring.py index ec0c9685..e47a1117 100644 --- a/api/app/scoring.py +++ b/api/app/scoring.py @@ -13,16 +13,18 @@ import json import threading from collections import OrderedDict -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Any import joblib import numpy as np +import pandas as pd import polars as pl from . import derived from .config import get_settings +from .feature_frame import FeatureFrame from .ingest import NULL_TOKENS from .modeling.mlflow_log import MODEL_FILE from .modeling.train import scores_of @@ -49,7 +51,7 @@ PREPROCESSOR = "preprocessor.joblib" BUNDLE = "bundle.json" -BUNDLE_FORMAT = 1 +BUNDLE_FORMAT = 2 class ModelChanged(ValueError): @@ -102,6 +104,9 @@ class Input: datetime_format: str | None decimal_comma: bool | None # set for a field the features read as a number: text that isn't one fails integer_codes: bool = False + # The texts read as missing: ingest's null tokens, less the ones the dataset's column holds as real values (a + # Parquet or connection dataset can store "None" or "NA" as a category; training saw it as one). + null_tokens: tuple[str, ...] = tuple(NULL_TOKENS) @property def integer_number(self) -> bool: @@ -127,6 +132,7 @@ class Scorer: preprocessor: Any n_features: int models: tuple[Model, ...] + frame_of: FeatureFrame | None = field(default=None, compare=False) # codes -> model input @property def required(self) -> list[str]: @@ -145,7 +151,7 @@ def frame(self, df: pl.DataFrame) -> tuple[pl.DataFrame, dict[str, int]]: for i in self.inputs: s, failed = convert(df[i.name], i) typed.append(s) - seen.append(integer_text(df[i.name], s, bool(i.decimal_comma)) if i.integer_number else s) + seen.append(integer_text(df[i.name], s, bool(i.decimal_comma), i.null_tokens) if i.integer_number else s) if failed: warnings[i.name] = failed frame, view = pl.DataFrame(typed), pl.DataFrame(seen) @@ -154,13 +160,14 @@ def frame(self, df: pl.DataFrame) -> tuple[pl.DataFrame, dict[str, int]]: view, frame = view.with_columns(col), frame.with_columns(col) return frame, warnings - def features(self, df: pl.DataFrame) -> tuple[np.ndarray, dict[str, int]]: - """The export's features for the raw records in `df`, and the conversion failures per field.""" + def features(self, df: pl.DataFrame) -> tuple[np.ndarray | pd.DataFrame, dict[str, int]]: + """The export's features for the raw records in `df` (a frame with category columns when the export passes + categoricals through natively), and the conversion failures per field.""" frame, warnings = self.frame(df) specs = list(self.specs) # Fresh coercers per call: they count values as they go, and threads share the scorer. X = transform_into(self.preprocessor, to_pandas_frame(frame, specs, coercion_exprs(specs)), self.n_features) - return X, warnings + return (X if self.frame_of is None else self.frame_of.from_codes(X)), warnings def outputs(self, df: pl.DataFrame) -> tuple[list[np.ndarray], dict[str, int]]: X, warnings = self.features(df) @@ -180,7 +187,10 @@ def from_bundle(cls, path: str | Path) -> "Scorer": except (OSError, ValueError) as exc: raise Unreadable(f"{BUNDLE} can't be read ({type(exc).__name__})") from exc if b.get("format") != BUNDLE_FORMAT: - raise ValueError(f"The bundle's format {b.get('format')!r} isn't one this release reads") + raise ValueError( + f"The bundle's format {b.get('format')!r} isn't one this release reads; rebuild the version to " + "score with it" + ) changed = f"A file of the bundle {folder} changed after it was built; build the version again" models = [] for m in b["models"]: @@ -200,6 +210,7 @@ def from_bundle(cls, path: str | Path) -> "Scorer": i.get("datetime_format"), i.get("decimal_comma"), bool(i.get("integer_codes")), + tuple(i["null_tokens"]), ) for i in b["inputs"] ), @@ -208,6 +219,7 @@ def from_bundle(cls, path: str | Path) -> "Scorer": preprocessor=_checked(folder / b["preprocessor"]["file"], b["preprocessor"]["sha256"], changed), n_features=int(b["n_features"]), models=tuple(models), + frame_of=FeatureFrame(list(b["feature_names"]), {k: list(v) for k, v in b["category_levels"].items()}), ) @@ -261,15 +273,15 @@ def dtype_class(name: str): raise ValueError(f"Unknown field type {text!r}; {REEXPORT}") from exc -def missing_as_null(s: pl.Series) -> pl.Series: - """`s` with what the upload reader reads as missing (its null tokens, the empty string among them) and NaN as - null. Nothing else is missing: a blank category is a value, as it was in the dataset.""" +def missing_as_null(s: pl.Series, tokens=NULL_TOKENS) -> pl.Series: + """`s` with what the upload reader reads as missing (`tokens`: its null tokens, the empty string among them, less + any the dataset holds as values) and NaN as null. Nothing else is missing: a blank category is a value, as it was in the dataset.""" if s.dtype.is_nested() or s.dtype == pl.Object: raise ValueError(f"Field {s.name!r} takes a single value, not a list or an object") if s.dtype.is_float(): return s.fill_nan(None) if s.dtype == pl.String: - return pl.select(pl.when(~s.is_in(NULL_TOKENS)).then(s)).to_series().alias(s.name) + return pl.select(pl.when(~s.is_in(list(tokens))).then(s)).to_series().alias(s.name) return s @@ -285,10 +297,10 @@ def whole_numbers(given: pl.Series, decimal_comma: bool = False) -> pl.Series: return whole -def integer_text(s: pl.Series, out: pl.Series, decimal_comma: bool = False) -> pl.Series: +def integer_text(s: pl.Series, out: pl.Series, decimal_comma: bool = False, tokens=NULL_TOKENS) -> pl.Series: """An integer-typed number (`out`, its float) as derived fields read it from the dataset's integer column: a whole number as its integer text, so a groups or count value "30" matches 30.0 the way it matched 30.""" - whole = whole_numbers(missing_as_null(s), decimal_comma) + whole = whole_numbers(missing_as_null(s, tokens), decimal_comma) text = pl.when(whole.is_not_null() & out.is_not_null()).then(whole.cast(pl.String)).otherwise(out.cast(pl.String)) return pl.select(text).to_series().alias(s.name) @@ -331,7 +343,7 @@ def as_temporal(s: pl.Series, dtype) -> pl.Series: def convert(s: pl.Series, to: Input) -> tuple[pl.Series, int]: """`s` as the dataset stored the field, and how many given values didn't convert (they become missing).""" - given = missing_as_null(s) + given = missing_as_null(s, to.null_tokens) dtype = to.dtype if dtype.is_integer() and not to.integer_number: # "7.0" and "07" are code 7; 7.5 isn't any code, and truncating it would invent one. @@ -454,6 +466,7 @@ def number_format(name: str) -> bool | None: schema[n].get("datetime_format"), number_format(n), bool(schema[n].get("integer_codes")), + tuple(schema[n].get("null_tokens", NULL_TOKENS)), ) for n in sources ) @@ -506,6 +519,7 @@ def _build(conn, spec: dict) -> Scorer: ), n_features=int(manifest["n_features"]), models=loaded, + frame_of=FeatureFrame.from_manifest(manifest), ) @@ -589,7 +603,7 @@ def equality_check( raw = _raw_rows(src.origins, scorer.required, ds_ids, rows) if isinstance(raw, str): return raw - X = part.select(feats).to_numpy() + X = (scorer.frame_of or FeatureFrame.from_manifest(src.manifest)).from_polars(part) got_X, _ = scorer.features(raw) for m in scorer.models: got = scores_of(m.estimator, m.report, got_X) @@ -758,8 +772,23 @@ def _raw_rows( return pl.concat(frames, how="diagonal_relaxed")[back] -def _feature_differs(got: np.ndarray, want: np.ndarray, feats: list[str], ids: np.ndarray) -> str | None: - g, w = got.astype(np.float32), want.astype(np.float32) +def _codes(X: np.ndarray | pd.DataFrame) -> np.ndarray: + """Model input as one float32 matrix: a category column as its code (-1, unseen or missing, as NaN).""" + if not isinstance(X, pd.DataFrame): + return X.astype(np.float32) + cols = [ + np.where(c.cat.codes < 0, np.nan, c.cat.codes).astype(np.float32) + if isinstance(c.dtype, pd.CategoricalDtype) + else c.to_numpy(np.float32) + for _, c in X.items() + ] + return np.column_stack(cols) if cols else np.empty((len(X), 0), np.float32) + + +def _feature_differs( + got: np.ndarray | pd.DataFrame, want: np.ndarray | pd.DataFrame, feats: list[str], ids: np.ndarray +) -> str | None: + g, w = _codes(got), _codes(want) bad = ~((g == w) | (np.isnan(g) & np.isnan(w))) if not bad.any(): return None diff --git a/api/app/semantics.py b/api/app/semantics.py index 09e3cec3..60576a0c 100644 --- a/api/app/semantics.py +++ b/api/app/semantics.py @@ -100,6 +100,7 @@ } ENCODINGS = { "none": "None", + "native": "None: pass through as a category", "onehot": "One-hot (rare levels grouped)", "ordinal": "Ordinal codes", "frequency": "Frequency encoding", @@ -108,6 +109,12 @@ "text_stats": "Length / word-count features", "datetime_parts": "Date parts + epoch days", } +# Labels that depend on the type: "..