feat: task eval + training hardening batch (review items 1-10)

.github/workflows/ci.yml

@@ -14,6 +14,19 @@ permissions:
   contents: read
 
 jobs:
+  smoke-cpu:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jdx/mise-action@v2
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/uv
+          key: uv-${{ runner.os }}-${{ hashFiles('uv.lock') }}
+      - run: uv sync --extra training
+      - run: uv run python -m brainforge.training.smoke_cpu
+        env:
+          HF_HOME: ${{ runner.temp }}/hf-cache
   lint:
     name: Lint (ruff)
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -9,6 +9,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Task-level evaluation harness (`train task-eval`): verdict accuracy, FP/FN
+  rates and CWE classification accuracy on the post-cutoff holdout, scored
+  against the judge-verified answer stored in each record (`task_eval.json`).
+- Interactive chat with a trained student: `train chat` (adapter or merged
+  model, greedy decoding via the model chat template).
+- Training checkpointing: `save_strategy="steps"` with `TrainingConfig.save_steps`
+  (default 100, keep last 2), seeded runs (`TrainingConfig.seed`, default 42)
+  and `train run --resume` (TRL native resume from the latest checkpoint).
+- CPU smoke training job in CI: 1 LoRA step on a tiny model, no GPU or
+  bitsandbytes needed (`make train-smoke-cpu`, `smoke-cpu` CI job). The smoke
+  test skips only when torch is missing; missing-training-deps or an
+  unreachable HF hub now fail the job instead of silently passing.
+- `train evaluate`/`train export`/`train task-eval`/`train chat` now default
+  `--model` to the latest run directory under `experiments/`.
+
+### Changed
+
+- `evaluate` now honors the configured quantization (`--quantization
+  4bit|8bit|none`) instead of always loading in 4-bit.
+- `train prepare` writes the default output to `datasets/prepared/`, the
+  directory `train run`/`evaluate`/`task-eval` read by default; the `--output`
+  override is unchanged.
+- `train run --resume` resumes the latest existing run directory instead of
+  creating a new one.
+- `latest_run_dir` resolves a run by modification time and honors the
+  configured `training.output_dir`.
+
+### Fixed
+
+- `train task-eval` (CLI path): real generation now flattens the message list
+  into a single content string, matching the typed `generate(str | list[str])`
+  contract; `prediction_from_text` no longer swallows unexpected errors as
+  "miss".
+- `train run`/`evaluate`/`task-eval`/`chat`/`export` fail with a clear message
+  instead of a raw traceback when an input file or model path is missing
+  (OSError clamped to a clean exit).
+- `train run --resume` on a run without checkpoints fails fast with a clear
+  message instead of a raw HF Trainer `ValueError`.
+- Removed dead code in `train chat` (misleading `name_or_path` argument,
+  redundant `generate_reply_fn`) and the pointless double dataset read in
+  `dataset split`.
+- `case.id` is validated against `[A-Za-z0-9_-]{1,64}`, preventing path
+  traversal via crafted case files.
+- Post-cutoff records no longer leak into the `train` split: they are now
+  exclusive to `test_postcutoff` (the anti-contamination benchmark).
+- Response cache is now consulted by `structured()` calls; previously every
+  pipeline structured call bypassed the sqlite cache.
+
+### Security
+
+- `ProviderConfig.base_url` now validates scheme (http/https) and rejects
+  link-local and cloud-metadata hosts (SSRF guard); loopback stays allowed for
+  local gateways.
+- Removed the committed `MLGW_API_KEY:deadbeef` placeholder default from the
+  sample config and docs (empty default like the other providers; the provider
+  fails at call time when the key is unset).
+
 - Real QLoRA training pipeline: `train_qlora` (TRL `SFTTrainer`, bitsandbytes
   NF4/8bit quantization, gradient checkpointing, `TrainingConfig`-driven),
   `evaluate` (loss + perplexity on the post-cutoff split, `eval.json`) and

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install test lint format secrets audit build ci train-smoke train-run config models rag dataset validate train evaluate clean
+.PHONY: install test lint format secrets audit build ci train-smoke train-smoke-cpu train-run config models rag dataset validate train evaluate clean
 
 install:
 	uv sync
@@ -33,6 +33,11 @@ train-smoke:
 	uv sync --extra training
 	uv run python -m brainforge.training.smoke
 
+# CPU smoke: tiny model, no GPU/bitsandbytes needed; skips gracefully offline.
+train-smoke-cpu:
+	uv sync --extra training
+	uv run python -m brainforge.training.smoke_cpu
+
 train-run:
 	uv run brainforge train run
 

config/config.json

@@ -14,7 +14,7 @@
     "mlgw": {
       "type": "mlgw",
       "base_url": "${MLGW_BASE_URL:http://localhost:8080/v1}",
-      "api_key": "${MLGW_API_KEY:deadbeef}"
+      "api_key": "${MLGW_API_KEY:}"
     },
     "mock": {
       "type": "mock"

config/schema.json

@@ -349,6 +349,18 @@
           "title": "Quantization",
           "type": "string"
         },
+        "save_steps": {
+          "default": 100,
+          "minimum": 1,
+          "title": "Save Steps",
+          "type": "integer"
+        },
+        "seed": {
+          "default": 42,
+          "minimum": 0,
+          "title": "Seed",
+          "type": "integer"
+        },
         "output_dir": {
           "default": "experiments",
           "title": "Output Dir",
@@ -415,6 +427,8 @@
         "batch_size": 1,
         "gradient_accumulation": 16,
         "quantization": "4bit",
+        "save_steps": 100,
+        "seed": 42,
         "output_dir": "experiments"
       }
     }

docs/reference/providers.md

@@ -49,11 +49,11 @@ styles.
 ## MLGW
 
 ```json
-{"type": "mlgw", "base_url": "${MLGW_BASE_URL:http://localhost:8080/v1}", "api_key": "${MLGW_API_KEY:deadbeef}"}
+{"type": "mlgw", "base_url": "${MLGW_BASE_URL:http://localhost:8080/v1}", "api_key": "${MLGW_API_KEY:}"}
 ```
 
-Local OpenAI-compatible gateway (llama.cpp, Ollama, vLLM backends). The
-`deadbeef` development key is intentionally fake.
+Local OpenAI-compatible gateway (llama.cpp, Ollama, vLLM backends). With an
+empty key the provider fails at call time; set `MLGW_API_KEY` to use it.
 
 ## Local
 

src/brainforge/case.py

@@ -1,4 +1,8 @@
-from pydantic import BaseModel, Field
+import re
+
+from pydantic import BaseModel, Field, field_validator
+
+_CASE_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
 
 
 class CaseSource(BaseModel):
@@ -22,6 +26,13 @@ class Case(BaseModel):
     input: CaseInput
     metadata: dict = Field(default_factory=dict)
 
+    @field_validator("id")
+    @classmethod
+    def _validate_id(cls, value: str) -> str:
+        if _CASE_ID_PATTERN.match(value) is None:
+            raise ValueError("case id must match [A-Za-z0-9_-]{1,64}")
+        return value
+
     @property
     def language(self) -> str | None:
         return self.metadata.get("language")

src/brainforge/cli/dataset_cmd.py

@@ -111,8 +111,6 @@ def split(
     """Split a dataset into train/validation/test (+ post-cutoff holdout)."""
     from brainforge.training.prepare import prepare
 
-    records = _load_records(path)
-    del records
     target = output_dir or path.parent / (path.stem + "_split")
     try:
         stats = prepare(path, target, train_ratio, val_ratio, seed)

src/brainforge/cli/train_cmd.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from functools import wraps
 from pathlib import Path
 
 import typer
@@ -11,7 +12,40 @@
 )
 
 
+def _clamp_errors(label: str):
+    """Turn expected failures (BrainforgeError and OS errors) into a clean exit."""
+
+    def decorator(fn):
+        @wraps(fn)
+        def wrapper(*args, **kwargs):
+            try:
+                return fn(*args, **kwargs)
+            except (BrainforgeError, OSError) as exc:
+                err_console.print(f"[red]{label} failed:[/red] {exc}")
+                raise typer.Exit(code=1) from exc
+
+        return wrapper
+
+    return decorator
+
+
+def latest_run_dir(base: Path | None = None) -> Path:
+    """Resolve the most recent training run directory, by modification time."""
+    if base is None:
+        from brainforge.config import load_config
+
+        base = Path(load_config().training.output_dir)
+    base = Path(base).resolve()
+    candidates = [d for d in base.glob("run-*") if d.is_dir()]
+    if not candidates:
+        raise BrainforgeError(
+            f"no training runs found under {base}; run 'brainforge train run' first"
+        )
+    return max(candidates, key=lambda d: d.stat().st_mtime)
+
+
 @app.command("prepare")
+@_clamp_errors("prepare")
 def prepare(
     dataset: Path = typer.Argument(..., help="Path to dataset.jsonl"),
     output_dir: Path = typer.Option(None, "--output", "-o"),
@@ -22,18 +56,15 @@ def prepare(
     """Validate, split and export a dataset for TRL training."""
     from brainforge.training.prepare import prepare
 
-    target = output_dir or Path("datasets") / (dataset.stem + "_prepared")
-    try:
-        stats = prepare(dataset, target, train_ratio, val_ratio, seed)
-    except BrainforgeError as exc:
-        err_console.print(f"[red]prepare failed:[/red] {exc}")
-        raise typer.Exit(code=1) from exc
+    target = output_dir or Path("datasets") / "prepared"
+    stats = prepare(dataset, target, train_ratio, val_ratio, seed)
     console.print(f"[green]prepared dataset[/green] -> {target}")
     for name, count in stats.items():
         console.print(f"  {name}: {count}")
 
 
 @app.command("run")
+@_clamp_errors("training")
 def run(
     dataset_dir: Path = typer.Option(
         Path("datasets/prepared"),
@@ -51,18 +82,18 @@ def run(
         "-o",
         help="Output dir (default: <config.output_dir>/<run name>)",
     ),
+    resume: bool = typer.Option(
+        False,
+        "--resume/--no-resume",
+        help="Resume from the latest checkpoint in the output dir",
+    ),
     config: str = typer.Option(None, "--config", "-c"),
 ):
     """Run QLoRA training (requires a CUDA GPU and the training extra)."""
     from brainforge.config import load_config
-    from brainforge.errors import ConfigError
     from brainforge.training.qlora import train_qlora
 
-    try:
-        cfg = load_config(config)
-    except ConfigError as exc:
-        err_console.print(f"[red]config error:[/red] {exc}")
-        raise typer.Exit(code=1) from exc
+    cfg = load_config(config)
     training = cfg.training
     updates = {}
     if epochs:
@@ -71,22 +102,22 @@ def run(
         updates["base_model"] = base_model
     if updates:
         training = training.model_copy(update=updates)
-    run_name = datetime.now().strftime("run-%Y%m%d-%H%M%S")
-    output_dir = output or Path(training.output_dir) / run_name
-    try:
-        summary = train_qlora(training, dataset_dir, output_dir)
-    except BrainforgeError as exc:
-        err_console.print(f"[red]training failed:[/red] {exc}")
-        raise typer.Exit(code=1) from exc
+    run_name = datetime.now().strftime("run-%Y%m%d-%H%M%S-%f")
+    if resume and output is None:
+        output_dir = latest_run_dir(base=Path(training.output_dir))
+    else:
+        output_dir = output or Path(training.output_dir) / run_name
+    summary = train_qlora(training, dataset_dir, output_dir, resume=resume)
     console.print(f"[green]training done[/green] -> {summary['output_dir']}")
     for key, value in summary.items():
         console.print(f"  {key}: {value}")
 
 
 @app.command("evaluate")
+@_clamp_errors("evaluation")
 def evaluate(
     model: Path = typer.Option(
-        Path("experiments"), "--model", "-m", help="Trained adapter directory"
+        None, "--model", "-m", help="Trained adapter directory (default: latest run)"
     ),
     dataset: Path = typer.Option(
         Path("datasets/prepared/test_postcutoff.jsonl"),
@@ -98,20 +129,58 @@ def evaluate(
     """Evaluate a trained student (loss + perplexity, written to eval.json)."""
     from brainforge.training.qlora import evaluate as evaluate_model
 
-    try:
-        result = evaluate_model(model, dataset)
-    except BrainforgeError as exc:
-        err_console.print(f"[red]evaluation failed:[/red] {exc}")
-        raise typer.Exit(code=1) from exc
+    result = evaluate_model(model or latest_run_dir(), dataset)
     console.print(f"[green]evaluated[/green] {result['n_records']} records")
     console.print(f"  eval_loss: {result['eval_loss']:.4f}")
     console.print(f"  perplexity: {result['perplexity']:.4f}")
 
 
+@app.command("chat")
+@_clamp_errors("chat")
+def chat(
+    model: Path = typer.Option(
+        None, "--model", "-m", help="Trained adapter or merged model (default: latest run)"
+    ),
+    max_new_tokens: int = typer.Option(512, "--max-new-tokens"),
+):
+    """Interactive chat with a trained student model (requires CUDA + training extra)."""
+    from brainforge.training.chat import chat_loop
+
+    chat_loop(model or latest_run_dir(), max_new_tokens=max_new_tokens)
+
+
+@app.command("task-eval")
+@_clamp_errors("task evaluation")
+def task_eval(
+    model: Path = typer.Option(
+        None, "--model", "-m", help="Trained adapter or merged model (default: latest run)"
+    ),
+    dataset: Path = typer.Option(
+        Path("datasets/prepared/test_postcutoff.jsonl"),
+        "--dataset",
+        "-d",
+        help="Eval split (JSONL)",
+    ),
+    quantization: str = typer.Option("4bit", "--quantization", help="4bit, 8bit or none"),
+    max_new_tokens: int = typer.Option(512, "--max-new-tokens"),
+):
+    """Task-level evaluation: verdict + CWE accuracy on a held-out split."""
+    from brainforge.training.task_eval import evaluate_model_on_records
+
+    result = evaluate_model_on_records(
+        model or latest_run_dir(), dataset, quantization, max_new_tokens
+    )
+    console.print(f"[green]task evaluation done[/green] ({result['n_records']} records)")
+    for key in ("accuracy", "false_positive_rate", "false_negative_rate", "cwe_accuracy"):
+        value = result.get(key)
+        console.print(f"  {key}: {value:.4f}" if isinstance(value, float) else f"  {key}: {value}")
+
+
 @app.command("export")
+@_clamp_errors("export")
 def export(
     model: Path = typer.Option(
-        Path("experiments"), "--model", "-m", help="Trained adapter directory"
+        None, "--model", "-m", help="Trained adapter directory (default: latest run)"
     ),
     output: Path = typer.Option(
         Path("models/export"), "--output", "-o", help="Merged model output directory"
@@ -120,9 +189,5 @@ def export(
     """Merge the LoRA adapter into the base model and export it standalone."""
     from brainforge.training.qlora import export as export_model
 
-    try:
-        result = export_model(model, output)
-    except BrainforgeError as exc:
-        err_console.print(f"[red]export failed:[/red] {exc}")
-        raise typer.Exit(code=1) from exc
+    result = export_model(model or latest_run_dir(), output)
     console.print(f"[green]exported[/green] -> {result['output_dir']}")