From bf5464269e5dcab6a9f5d15c6c9ba71327715f12 Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Sun, 15 Nov 2026 20:47:13 -0700 Subject: [PATCH 01/35] Phase 9 S1: migration 043, RLS and job plumbing Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- api/app/jobkinds.py | 11 + api/app/migrations/043_modeling.sql | 114 +++++++++ api/app/paths.py | 9 +- api/app/storage.py | 1 + api/tests/data/confusion_cases.json | 380 ++++++++++++++++++++++++++++ api/tests/test_modeling_rls.py | 126 +++++++++ api/tests/test_rls_app.py | 2 +- 7 files changed, 640 insertions(+), 3 deletions(-) create mode 100644 api/app/migrations/043_modeling.sql create mode 100644 api/tests/data/confusion_cases.json create mode 100644 api/tests/test_modeling_rls.py diff --git a/api/app/jobkinds.py b/api/app/jobkinds.py index 883c5747..e49fbdb2 100644 --- a/api/app/jobkinds.py +++ b/api/app/jobkinds.py @@ -43,6 +43,7 @@ "model_versions", "evaluations", "cleanups", + "run_batches", } ) @@ -136,9 +137,16 @@ def enqueue( "model_versions": "SELECT m.project_id FROM meta.model_versions v JOIN meta.registered_models m ON m.id = v.model_id WHERE v.id = %s", "evaluations": """SELECT m.project_id FROM meta.evaluations e JOIN meta.model_versions v ON v.id = e.version_id JOIN meta.registered_models m ON m.id = v.model_id WHERE e.id = %s""", + "run_batches": """SELECT x.project_id FROM meta.run_batches b JOIN meta.experiments x ON x.id = b.experiment_id + WHERE b.id = %s""", } # fmt: skip +def needs_for(device: str) -> dict | None: + """The queue slot a run on `device` waits in: a `cuda` job only ever runs in the gpu slot.""" + return {"slot": "gpu"} if device == "cuda" else None + + def project_of(conn, ref_table: str, ref_id: int) -> int: """The project of the domain row `ref_table` / `ref_id`, for `enqueue`'s `project_id` (an evaluation's is its model's, a pull's its dataset's).""" @@ -181,6 +189,9 @@ def _within(conn) -> Iterator: JOIN meta.model_versions v ON v.id = e.version_id JOIN meta.registered_models m ON m.id = v.model_id WHERE e.id = ANY(%s)""", "cleanups": "SELECT id, dry_run FROM meta.cleanups WHERE id = ANY(%s)", + "run_batches": """SELECT id, experiment_id, export_id, name, + (SELECT dataset_id FROM meta.exports e WHERE e.id = export_id) AS dataset_id + FROM meta.run_batches WHERE id = ANY(%s)""", } # table -> {id: row}, while `prefetched` is open; a table that isn't here is read row by row. _prefetched: ContextVar[dict[str, dict[int, dict]] | None] = ContextVar("jobkinds_prefetched", default=None) diff --git a/api/app/migrations/043_modeling.sql b/api/app/migrations/043_modeling.sql new file mode 100644 index 00000000..c0d9241c --- /dev/null +++ b/api/app/migrations/043_modeling.sql @@ -0,0 +1,114 @@ +-- Migration 043 (phase 9): modeling upgrades. Field descriptions and notes (with their history), derived baseline +-- versions and pending derivations, re-runs, tuning studies (a batch of kind 'tune' with one row per trial), and the +-- worker's job-thread count. No backward compatibility. Replay-safe. + +-- 1. Field descriptions and notes. +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS description TEXT; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS field_notes TEXT; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS about_by BIGINT REFERENCES meta.users ON DELETE SET NULL; +ALTER TABLE meta.fields ADD COLUMN IF NOT EXISTS about_at TIMESTAMPTZ; +ALTER TABLE meta.fields DROP CONSTRAINT IF EXISTS fields_description_check; +ALTER TABLE meta.fields ADD CONSTRAINT fields_description_check CHECK (char_length(description) <= 500); +ALTER TABLE meta.fields DROP CONSTRAINT IF EXISTS fields_field_notes_check; +ALTER TABLE meta.fields ADD CONSTRAINT fields_field_notes_check CHECK (char_length(field_notes) <= 10000); + +CREATE TABLE IF NOT EXISTS meta.field_about_history ( + id BIGSERIAL PRIMARY KEY, + dataset_id BIGINT NOT NULL REFERENCES meta.datasets ON DELETE CASCADE, + field_name TEXT NOT NULL, + user_id BIGINT REFERENCES meta.users ON DELETE SET NULL, + at TIMESTAMPTZ NOT NULL DEFAULT now(), + previous JSONB NOT NULL, + current JSONB NOT NULL +); +CREATE INDEX IF NOT EXISTS field_about_history_idx ON meta.field_about_history (dataset_id, field_name, at DESC); + +-- 2. Runs, versions and re-runs. A derived baseline version remembers the run it came from; a re-run's experiment +-- remembers its run and, while its baseline version isn't made yet, the pending derivation it waits for. +ALTER TABLE meta.model_runs ADD COLUMN IF NOT EXISTS trial_number INT; +ALTER TABLE meta.recipes ADD COLUMN IF NOT EXISTS derived_from_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; + +CREATE TABLE IF NOT EXISTS meta.pending_derivations ( + id BIGSERIAL PRIMARY KEY, + recipe_id BIGINT NOT NULL REFERENCES meta.recipes ON DELETE CASCADE, + run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL, + field_names TEXT[] NOT NULL, + rule JSONB NOT NULL, + finalize_now BOOLEAN NOT NULL DEFAULT false, + created_by BIGINT REFERENCES meta.users ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + done_recipe_id BIGINT REFERENCES meta.recipes ON DELETE SET NULL, + cancelled_at TIMESTAMPTZ, + note TEXT +); +CREATE UNIQUE INDEX IF NOT EXISTS pending_derivations_open_idx ON meta.pending_derivations (recipe_id) + WHERE done_recipe_id IS NULL AND cancelled_at IS NULL; + +ALTER TABLE meta.experiments ADD COLUMN IF NOT EXISTS rerun_of_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; +ALTER TABLE meta.experiments ADD COLUMN IF NOT EXISTS pending_derivation_id BIGINT REFERENCES meta.pending_derivations ON DELETE SET NULL; +ALTER TABLE meta.exports ADD COLUMN IF NOT EXISTS then_run JSONB; + +-- 3. Tuning studies: a batch of kind 'tune' (one job; a study stops through its job, so no 'stopping' status), one +-- row per trial. +ALTER TABLE meta.run_batches DROP CONSTRAINT IF EXISTS run_batches_kind_check; +ALTER TABLE meta.run_batches ADD CONSTRAINT run_batches_kind_check CHECK (kind IN ('one_vs_rest', 'search', 'tune')); +ALTER TABLE meta.run_batches DROP CONSTRAINT IF EXISTS run_batches_status_check; +ALTER TABLE meta.run_batches ADD CONSTRAINT run_batches_status_check + CHECK (status IN ('running', 'done', 'failed', 'cancelled')); +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS best_run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL; +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS job_id BIGINT; +ALTER TABLE meta.run_batches ADD COLUMN IF NOT EXISTS result JSONB; + +CREATE TABLE IF NOT EXISTS meta.tuning_trials ( + batch_id BIGINT NOT NULL REFERENCES meta.run_batches ON DELETE CASCADE, + number INT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('running', 'complete', 'pruned', 'failed')), + value DOUBLE PRECISION, + params JSONB NOT NULL DEFAULT '{}', + steps JSONB NOT NULL DEFAULT '[]', + best_iterations JSONB, + pruned_at INT, + error TEXT, + started_at TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + run_id BIGINT REFERENCES meta.model_runs ON DELETE SET NULL, + PRIMARY KEY (batch_id, number) +); + +-- 4. The worker's job-thread count (the live worker's JOB threads, shown beside CPU runs). +ALTER TABLE meta.workers ADD COLUMN IF NOT EXISTS threads INT; + +-- 5. Row-level security, in 041's style. Every policy is dropped before it is made, so a replay changes nothing. +DO $$ +DECLARE + s CONSTANT TEXT := '(SELECT meta.rf_system())'; + ids CONSTANT TEXT := '(SELECT meta.rf_ids(''projects''))::bigint[]'; + t TEXT; + rd TEXT; + wr TEXT; + rules TEXT[][] := ARRAY[ + ARRAY['field_about_history', + s || ' OR dataset_id IN (SELECT id FROM meta.datasets)', + s || ' OR dataset_id IN (SELECT id FROM meta.datasets WHERE project_id = ANY(' || ids || '))'], + ARRAY['pending_derivations', + s || ' OR recipe_id IN (SELECT id FROM meta.recipes WHERE project_id = ANY(' || ids || '))', + s || ' OR recipe_id IN (SELECT id FROM meta.recipes WHERE project_id = ANY(' || ids || '))'], + ARRAY['tuning_trials', + s || ' OR batch_id IN (SELECT id FROM meta.run_batches)', + s || ' OR batch_id IN (SELECT b.id FROM meta.run_batches b JOIN meta.experiments x ON x.id = b.experiment_id' + || ' WHERE x.project_id = ANY(' || ids || '))'] + ]; +BEGIN + FOR i IN 1 .. array_length(rules, 1) LOOP + t := rules[i][1]; rd := rules[i][2]; wr := rules[i][3]; + EXECUTE format('ALTER TABLE meta.%I ENABLE ROW LEVEL SECURITY', t); + EXECUTE format('DROP POLICY IF EXISTS rf_read ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_write ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_change ON meta.%I', t); + EXECUTE format('DROP POLICY IF EXISTS rf_delete ON meta.%I', t); + EXECUTE format('CREATE POLICY rf_read ON meta.%I AS PERMISSIVE FOR SELECT TO PUBLIC USING (%s)', t, rd); + EXECUTE format('CREATE POLICY rf_write ON meta.%I AS PERMISSIVE FOR INSERT TO PUBLIC WITH CHECK (%s)', t, wr); + EXECUTE format('CREATE POLICY rf_change ON meta.%I AS PERMISSIVE FOR UPDATE TO PUBLIC USING (%s) WITH CHECK (%s)', t, wr, wr); + EXECUTE format('CREATE POLICY rf_delete ON meta.%I AS PERMISSIVE FOR DELETE TO PUBLIC USING (%s)', t, wr); + END LOOP; +END $$; diff --git a/api/app/paths.py b/api/app/paths.py index 6942413c..5c66e6ac 100644 --- a/api/app/paths.py +++ b/api/app/paths.py @@ -12,8 +12,8 @@ log = logging.getLogger(__name__) -Kind = Literal["datasets", "exports", "compositions", "models"] -KINDS: tuple[str, ...] = ("datasets", "exports", "compositions", "models") +Kind = Literal["datasets", "exports", "compositions", "models", "studies"] +KINDS: tuple[str, ...] = ("datasets", "exports", "compositions", "models", "studies") # `notebooks` is the project's shared notebook folder, written by Labs (spawner.LAB_UID), so it is not a mounted kind. PROJECT_FOLDERS = (*KINDS, "notebooks") @@ -43,6 +43,11 @@ def run_dir(project_id: int, run_id: int) -> Path: return project_dir(project_id) / "models" / str(run_id) +def study_dir(project_id: int, batch_id: int) -> Path: + """A tuning study's journal and files: one folder per batch.""" + return project_dir(project_id) / "studies" / str(batch_id) + + def notebook_folder(project_id: int) -> Path: return project_dir(project_id) / "notebooks" diff --git a/api/app/storage.py b/api/app/storage.py index 7bbbabaf..eda629ba 100644 --- a/api/app/storage.py +++ b/api/app/storage.py @@ -25,6 +25,7 @@ "compositions", "exports", "models", + "studies", "registry", "notebooks", "scoring", diff --git a/api/tests/data/confusion_cases.json b/api/tests/data/confusion_cases.json new file mode 100644 index 00000000..90d26e32 --- /dev/null +++ b/api/tests/data/confusion_cases.json @@ -0,0 +1,380 @@ +[ + { + "name": "binary", + "cm": [ + [ + 50, + 10 + ], + [ + 5, + 35 + ] + ], + "positive": 1, + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.85, + "precision": 0.7777777777777778, + "recall": 0.875, + "f1": 0.8235294117647058, + "far": 0.16666666666666666 + }, + "macro": { + "accuracy": 0.8541666666666667, + "precision": 0.8434343434343434, + "recall": 0.8541666666666667, + "f1": 0.8465473145780051, + "far": 0.14583333333333331 + }, + "micro": { + "accuracy": 0.85, + "precision": 0.85, + "recall": 0.85, + "f1": 0.85, + "far": 0.15 + }, + "weighted": { + "precision": 0.8565656565656566, + "recall": 0.85, + "f1": 0.8511508951406648, + "far": 0.14166666666666666 + } + }, + "types": { + "reference": 0, + "type_i": 10, + "type_ii": 5, + "type_iii": null, + "errors": 15 + }, + "plain_far": 0.16666666666666666 + } + }, + { + "name": "three_class_ref", + "cm": [ + [ + 30, + 5, + 5 + ], + [ + 4, + 20, + 6 + ], + [ + 2, + 3, + 25 + ] + ], + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.75, + "precision": null, + "recall": null, + "f1": null, + "far": null + }, + "macro": { + "accuracy": 0.75, + "precision": 0.7473544973544973, + "recall": 0.75, + "f1": 0.745568204733359, + "far": 0.12380952380952381 + }, + "micro": { + "accuracy": 0.75, + "precision": 0.75, + "recall": 0.75, + "f1": 0.75, + "far": 0.125 + }, + "weighted": { + "precision": 0.755952380952381, + "recall": 0.75, + "f1": 0.7499587526810757, + "far": 0.12142857142857143 + } + }, + "types": { + "reference": 0, + "type_i": 10, + "type_ii": 6, + "type_iii": 9, + "errors": 25 + }, + "plain_far": 0.25 + } + }, + { + "name": "four_class_zero_class", + "cm": [ + [ + 20, + 2, + 1, + 0 + ], + [ + 3, + 15, + 2, + 0 + ], + [ + 0, + 0, + 0, + 0 + ], + [ + 1, + 2, + 1, + 10 + ] + ], + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.7894736842105263, + "precision": null, + "recall": null, + "f1": null, + "far": null + }, + "macro": { + "accuracy": 0.5834627329192547, + "precision": 0.6557017543859649, + "recall": 0.5834627329192547, + "f1": 0.6134069830878341, + "far": 0.0739826513820322 + }, + "micro": { + "accuracy": 0.7894736842105263, + "precision": 0.7894736842105263, + "recall": 0.7894736842105263, + "f1": 0.7894736842105263, + "far": 0.07017543859649122 + }, + "weighted": { + "precision": 0.8588796552785473, + "recall": 0.7894736842105263, + "f1": 0.8179956164278672, + "far": 0.08540428973865506 + } + }, + "types": { + "reference": 0, + "type_i": 3, + "type_ii": 4, + "type_iii": 5, + "errors": 12 + }, + "plain_far": 0.13043478260869565 + } + }, + { + "name": "never_predicted", + "cm": [ + [ + 20, + 0, + 5 + ], + [ + 3, + 0, + 7 + ], + [ + 2, + 0, + 23 + ] + ], + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.7166666666666667, + "precision": null, + "recall": null, + "f1": null, + "far": null + }, + "macro": { + "accuracy": 0.5733333333333334, + "precision": 0.48571428571428577, + "recall": 0.5733333333333334, + "f1": 0.5222222222222223, + "far": 0.1619047619047619 + }, + "micro": { + "accuracy": 0.7166666666666667, + "precision": 0.7166666666666667, + "recall": 0.7166666666666667, + "f1": 0.7166666666666667, + "far": 0.14166666666666666 + }, + "weighted": { + "precision": 0.6071428571428572, + "recall": 0.7166666666666667, + "f1": 0.6527777777777779, + "far": 0.20238095238095238 + } + }, + "types": { + "reference": 0, + "type_i": 5, + "type_ii": 5, + "type_iii": 7, + "errors": 17 + }, + "plain_far": 0.2 + } + }, + { + "name": "single_row", + "cm": [ + [ + 4, + 1, + 0 + ], + [ + 0, + 0, + 0 + ], + [ + 0, + 0, + 0 + ] + ], + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.8, + "precision": null, + "recall": null, + "f1": null, + "far": null + }, + "macro": { + "accuracy": 0.26666666666666666, + "precision": 0.3333333333333333, + "recall": 0.26666666666666666, + "f1": 0.2962962962962963, + "far": 0.1 + }, + "micro": { + "accuracy": 0.8, + "precision": 0.8, + "recall": 0.8, + "f1": 0.8, + "far": 0.1 + }, + "weighted": { + "precision": 1.0, + "recall": 0.8, + "f1": 0.888888888888889, + "far": null + } + }, + "types": { + "reference": 0, + "type_i": 1, + "type_ii": 0, + "type_iii": 0, + "errors": 1 + }, + "plain_far": 0.2 + } + }, + { + "name": "label_outside_include", + "cm": [ + [ + 20, + 3, + 2, + 5 + ], + [ + 4, + 18, + 3, + 5 + ], + [ + 1, + 2, + 25, + 2 + ], + [ + 0, + 0, + 0, + 0 + ] + ], + "include": [ + 0, + 1, + 2 + ], + "k": 3, + "reference": 0, + "expected": { + "scores": { + "plain": { + "accuracy": 0.7, + "precision": null, + "recall": null, + "f1": null, + "far": null + }, + "macro": { + "accuracy": 0.7000000000000001, + "precision": 0.8053140096618359, + "recall": 0.7000000000000001, + "f1": 0.7466171145416429, + "far": 0.08333333333333333 + }, + "micro": { + "accuracy": 0.7, + "precision": 0.8076923076923077, + "recall": 0.7, + "f1": 0.75, + "far": 0.08333333333333333 + }, + "weighted": { + "precision": 0.8053140096618358, + "recall": 0.7, + "f1": 0.7466171145416428, + "far": 0.08333333333333333 + } + }, + "types": { + "reference": 0, + "type_i": 10, + "type_ii": 5, + "type_iii": 12, + "errors": 27 + }, + "plain_far": 0.3333333333333333 + } + } +] \ No newline at end of file diff --git a/api/tests/test_modeling_rls.py b/api/tests/test_modeling_rls.py new file mode 100644 index 00000000..7a482a4d --- /dev/null +++ b/api/tests/test_modeling_rls.py @@ -0,0 +1,126 @@ +"""Row-level security of migration 043's tables: field about history follows the dataset (and a lend), tuning trials +follow their batch's project, pending derivations stay in their project. Needs RAINFOREST_TEST_DATABASE_URL.""" + +import os +import uuid + +import psycopg +import pytest + +from .team_helpers import app_ # noqa: F401 (fixture) + +pytestmark = pytest.mark.skipif( + not os.environ.get("RAINFOREST_TEST_DATABASE_URL"), reason="set RAINFOREST_TEST_DATABASE_URL to run the API tests" +) + + +@pytest.fixture(scope="module") +def world(app_): # noqa: F811 + from app import access, db, users + + tag = uuid.uuid4().hex[:6] + o = users.create(f"Olive {tag}", None, "member") # owner of A + c = users.create(f"Cal {tag}", None, "member") # contributor of A + x = users.create(f"Xena {tag}", None, "member") # owner of B + with db.connection(system=True) as conn: + ids = {} + for slug in ("a", "b"): + ids[slug] = conn.execute( + "INSERT INTO meta.projects (slug, name, created_by) VALUES (%s, %s, %s) RETURNING id", + (f"m43-{slug}-{tag}", slug, o["id"]), + ).fetchone()["id"] + for pid, uid, role in ( + (ids["a"], o["id"], "owner"), + (ids["a"], c["id"], "contributor"), + (ids["b"], x["id"], "owner"), + ): + conn.execute( + "INSERT INTO meta.memberships (project_id, user_id, role) VALUES (%s, %s, %s)", (pid, uid, role) + ) + d = conn.execute( + "INSERT INTO meta.datasets (name, file_format, project_id) VALUES (%s, 'csv', %s) RETURNING id", + (f"d-{tag}", ids["a"]), + ).fetchone()["id"] + r = conn.execute( + "INSERT INTO meta.recipes (dataset_id, version, phase, project_id) VALUES (%s, 1, 'finalized', %s) RETURNING id", + (d, ids["a"]), + ).fetchone()["id"] + exp = conn.execute( + "INSERT INTO meta.experiments (dataset_id, name, recipe_id, source, project_id) VALUES (%s, 'x', %s, %s, %s) RETURNING id", + (d, r, f"recipe:{r}", ids["a"]), + ).fetchone()["id"] + e = conn.execute( + "INSERT INTO meta.exports (dataset_id, experiment_id, project_id) VALUES (%s, %s, %s) RETURNING id", + (d, exp, ids["a"]), + ).fetchone()["id"] + b = conn.execute( + "INSERT INTO meta.run_batches (experiment_id, export_id, kind, name, spec) VALUES (%s, %s, 'tune', 'study', '{}') RETURNING id", + (exp, e), + ).fetchone()["id"] + conn.execute("INSERT INTO meta.tuning_trials (batch_id, number, state) VALUES (%s, 0, 'complete')", (b,)) + conn.execute( + "INSERT INTO meta.field_about_history (dataset_id, field_name, previous, current) VALUES (%s, 'f', '{}', '{}')", + (d,), + ) + conn.execute( + "INSERT INTO meta.pending_derivations (recipe_id, field_names, rule) VALUES (%s, '{f}', '{}')", (r,) + ) + conn.execute( + "INSERT INTO meta.shares (kind, object_id, from_project, to_project) VALUES ('dataset', %s, %s, %s)", + (d, ids["a"], ids["b"]), + ) + return { + "d": d, "r": r, "b": b, + "o": access.scope(o, fresh=True), "c": access.scope(c, fresh=True), "x": access.scope(x, fresh=True), + } # fmt: skip + + +def _count(world, who: str, table: str) -> int: + from app import db + + with db.use_rls(world[who].rls()), db.connection() as conn: + return conn.execute(f"SELECT count(*) AS n FROM meta.{table}").fetchone()["n"] + + +def test_about_history_follows_the_dataset(world): + from app import db + + for who in ("o", "c", "x"): # x sees it through the lend + assert _count(world, who, "field_about_history") == 1, who + row = ( + "INSERT INTO meta.field_about_history (dataset_id, field_name, previous, current) VALUES (%s, 'g', '{}', '{}')" + ) + with db.use_rls(world["c"].rls()), db.connection() as conn: + conn.execute(row, (world["d"],)) + with pytest.raises(psycopg.errors.InsufficientPrivilege): + with db.use_rls(world["x"].rls()), db.connection() as conn: + conn.execute(row, (world["d"],)) + with db.connection(system=True) as conn: + conn.execute("DELETE FROM meta.field_about_history WHERE field_name = 'g'") + + +def test_trials_follow_the_batch(world): + from app import db + + assert _count(world, "c", "tuning_trials") == 1 + assert _count(world, "x", "tuning_trials") == 0 + with pytest.raises(psycopg.errors.InsufficientPrivilege): + with db.use_rls(world["x"].rls()), db.connection() as conn: + conn.execute( + "INSERT INTO meta.tuning_trials (batch_id, number, state) VALUES (%s, 1, 'running')", (world["b"],) + ) + with db.use_rls(world["c"].rls()), db.connection() as conn: + conn.execute( + "INSERT INTO meta.tuning_trials (batch_id, number, state) VALUES (%s, 1, 'running')", (world["b"],) + ) + assert conn.execute("UPDATE meta.tuning_trials SET state = 'failed' WHERE number = 1").rowcount == 1 + + +def test_pending_derivations_stay_home(world): + from app import db + + assert _count(world, "o", "pending_derivations") == 1 + assert _count(world, "c", "pending_derivations") == 1 + assert _count(world, "x", "pending_derivations") == 0 # the dataset is lent, its pending derivations are not + with db.use_rls(world["x"].rls()), db.connection() as conn: + assert conn.execute("DELETE FROM meta.pending_derivations WHERE recipe_id = %s", (world["r"],)).rowcount == 0 diff --git a/api/tests/test_rls_app.py b/api/tests/test_rls_app.py index 0664aa8d..c5255187 100644 --- a/api/tests/test_rls_app.py +++ b/api/tests/test_rls_app.py @@ -333,7 +333,7 @@ def policies(conn): ).fetchall() paths = [p for p in sorted(db.MIGRATIONS_DIR.glob("*.sql")) if p.stem >= "008"] - assert paths[-1].stem.startswith("042_") + assert paths[-1].stem.startswith("043_") with db.owner_connection() as conn: before = policies(conn) assert conn.execute("SELECT count(*) AS n FROM meta.datasets").fetchone()["n"] >= 2 From 0198ebc325008a49478c0d01f8cfc06c076a3d21 Mon Sep 17 00:00:00 2001 From: Tanner Overcash Date: Sun, 15 Nov 2026 20:52:43 -0700 Subject: [PATCH 02/35] Phase 9 B1: FeatureFrame and native categoricals in the export Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LqNnEU86Q4fmrJnN2rXovS --- api/app/feature_frame.py | 70 ++++++++++ api/app/semantics.py | 39 ++++-- api/app/transform.py | 166 ++++++++++++++++++++--- api/app/transforms/builtin.py | 22 +++ api/tests/test_export_memory.py | 9 +- api/tests/test_native_export.py | 230 ++++++++++++++++++++++++++++++++ api/tests/test_pipeline.py | 2 + api/tests/test_sentinels.py | 8 +- 8 files changed, 511 insertions(+), 35 deletions(-) create mode 100644 api/app/feature_frame.py create mode 100644 api/tests/test_native_export.py diff --git a/api/app/feature_frame.py b/api/app/feature_frame.py new file mode 100644 index 00000000..b1ad801d --- /dev/null +++ b/api/app/feature_frame.py @@ -0,0 +1,70 @@ +"""FeatureFrame: the one place a native categorical feature becomes a category column. + +The preprocessor writes a native categorical as a float code (the level's index, NaN for missing or unseen) so +`transform_into` keeps its shape. The export stores those columns dictionary-encoded; training, scoring, the bundle +and the code script turn codes (or text) back into `pd.Categorical` here. With no native feature every method is +the float path as it was. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa + + +@dataclass +class FeatureFrame: + names: list[str] + levels: dict[str, list[str]] = field(default_factory=dict) # native feature -> its categories, in code order + + @classmethod + def from_manifest(cls, m: dict) -> "FeatureFrame": + return cls(list(m["feature_names"]), {k: list(v) for k, v in (m.get("category_levels") or {}).items()}) + + @property + def native(self) -> bool: + return bool(self.levels) + + def from_codes(self, X: np.ndarray) -> np.ndarray | pd.DataFrame: + """The preprocessor's float codes (columns in `names` order) as model input.""" + if not self.native: + return X + cols: dict[str, object] = {} + for k, n in enumerate(self.names): + col = X[:, k] + if n in self.levels: + codes = np.where(np.isnan(col), -1, col).astype(np.int64) + cols[n] = pd.Categorical.from_codes(codes, categories=self.levels[n]) + else: + cols[n] = np.asarray(col, dtype=np.float32) + return pd.DataFrame(cols) + + def from_polars(self, df: pl.DataFrame) -> np.ndarray | pd.DataFrame: + """A model-ready frame (as read from model_ready.parquet) as model input. A native column is mapped by its + text, never by polars' own category codes.""" + if not self.native: + return df.select(self.names).to_numpy().astype(np.float32) + cols: dict[str, object] = {} + for n in self.names: + if n in self.levels: + cols[n] = pd.Categorical(df[n].cast(pl.String).to_numpy(), categories=self.levels[n]) + else: + cols[n] = df[n].to_numpy().astype(np.float32) + return pd.DataFrame(cols) + + def arrow_field(self, name: str) -> pa.Field: + if name in self.levels: + return pa.field(name, pa.dictionary(pa.int32(), pa.string())) + return pa.field(name, pa.float32()) + + def arrow_column(self, name: str, values: np.ndarray) -> pa.Array: + """One feature's chunk for the export file: float32, or dictionary-encoded codes (null where NaN).""" + if name not in self.levels: + return pa.array(values) + missing = np.isnan(values) + codes = pa.array(np.where(missing, 0, values).astype(np.int32), pa.int32(), mask=missing) + return pa.DictionaryArray.from_arrays(codes, pa.array(self.levels[name], pa.string())) diff --git a/api/app/semantics.py b/api/app/semantics.py index 09e3cec3..60576a0c 100644 --- a/api/app/semantics.py +++ b/api/app/semantics.py @@ -100,6 +100,7 @@ } ENCODINGS = { "none": "None", + "native": "None: pass through as a category", "onehot": "One-hot (rare levels grouped)", "ordinal": "Ordinal codes", "frequency": "Frequency encoding", @@ -108,6 +109,12 @@ "text_stats": "Length / word-count features", "datetime_parts": "Date parts + epoch days", } +# Labels that depend on the type: "..