feat(eval): add eval redteam to probe grading checks with planted answers

dokimi/changelog.d/20261028_120100_napetrov_feat-verifier-redteam.md

@@ -0,0 +1,3 @@
+### Added
+
+- **`eval redteam` checks that grading blocks reject wrong answers** — grades a planted wrong, an empty and a hedged answer against each question's `grading` block, and a canary right answer built from its one literal check. A wrong answer the grader accepts, or a canary it rejects, is a violation: the command prints pass rates, exits 1 and, with `--out FILE`, writes a `redteam.v1` artifact (it records the judge model and provider). A run in which a graded question was not measured (judge checks without `--judge`, a block of only negated checks) is `inconclusive` and also exits 1. Judge checks are unscored unless `--judge` is given, so the default run makes no model call.

dokimi/docs/answer-formats.md

@@ -98,6 +98,38 @@ artifact for an `arms run` file. Each block carries `scorer: reference`, and
 the run's pass rates and mean scores are under `run_metadata.summary` (or
 `reference_scoring`).
 
+## Red-teaming the grading
+
+`eval redteam QUESTIONS.json` grades synthetic answers against each question's
+`grading` block and reports how often the grader accepts them:
+
+| Attack | Answer | The grader must |
+|---|---|---|
+| `canary` | The right answer, built from the one literal check (`match`, `fuzzy`, `exact`, `choice`, `json`); a question with several checks, a `regex`, a `judge` or a `negate` has none | pass it |
+| `wrong` | A planted wrong answer, picked per block so that no positive literal check accepts it, asked with the check's own rules (the probe is skipped only when no candidate is clean) | fail it |
+| `empty` | `""` | fail it |
+| `hedge` | A fixed non-committal sentence; skipped when a positive literal check accepts it | fail it |
+
+An attack the grader accepts, or a canary it rejects, is a violation, and any
+violation makes the run `blocked`: the command prints the pass rates and the
+violations and exits 1 (`--out FILE` writes `redteam.v1`). A run in which a graded
+question was not measured is `inconclusive` and also exits 1: a question whose
+`judge` checks have no verdict without `--judge` (no probe scored), or a block
+of only negated checks. A `negate` check passes an answer that lacks its value,
+so a block of only negated checks gets none of `wrong`, `empty` and `hedge`: it
+is marked `negated_only` in the artifact (the table prints a line "N questions with only negated checks: wrong, empty and hedge not run"),
+and it has no probe at all. The table names up to 20 questions that were not
+measured and counts them as `unprobed`; `--out FILE` keeps the complete list; only a run that measured every graded
+question and found no violation is `pass`. An unwritable `--out` exits 2.
+A block with a positive check
+is graded as written, `threshold` included, so a negated check's share of the score
+counts: a block it lifts over the line is blocked.
+A probe the grader cannot
+score is unscored and does not block. `judge` checks are unscored unless
+`--judge` is given, so the default run makes no model call; with `--judge` the
+artifact records `judge_model` and `judge_provider` (both `null` without it). Nothing here
+changes a score; it tells you whether a grading block is worth trusting.
+
 ## What does not carry over
 
 | Format | Lossy |

dokimi/docs/reference/cli-interface.json

@@ -5714,6 +5714,78 @@
           "subcommand_required": null,
           "subcommands": {}
         },
+        "redteam": {
+          "allow_abbrev": true,
+          "params": [
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "questions",
+              "kind": "argument",
+              "nargs": null,
+              "position": 0,
+              "required": true,
+              "type": null
+            },
+            {
+              "action": "_HelpAction",
+              "choices": null,
+              "const": null,
+              "default": "==SUPPRESS==",
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--help/-h",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--judge",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": "dokimi.commands.redteam._judge_ref#a59ddb60ae31"
+            },
+            {
+              "action": "_StoreAction",
+              "choices": null,
+              "const": null,
+              "default": null,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": false,
+              "key": "--out",
+              "kind": "option",
+              "nargs": null,
+              "position": null,
+              "required": false,
+              "type": null
+            }
+          ],
+          "path": "dokimi eval redteam",
+          "subcommand_required": null,
+          "subcommands": {}
+        },
         "reference": {
           "allow_abbrev": true,
           "params": [
@@ -13046,6 +13118,7 @@
                 "'plan'",
                 "'plugin_delta'",
                 "'questions'",
+                "'redteam'",
                 "'repo_budget'",
                 "'repo_environment'",
                 "'repo_readiness'",

dokimi/dokimi/artifacts.py

@@ -58,6 +58,7 @@
     "result": "result.v1",
     "repo_budget": "repo_budget.v1",
     "task_inspect": "task_inspect.v1",
+    "redteam": "redteam.v1",
     "cli_routes": "cli_routes.v1",
     "need_graph": "need_graph.v1",
     "eval_coverage": "eval_coverage.v1",

dokimi/dokimi/cli_routes.py

@@ -150,6 +150,7 @@ class Tombstone:
     Leaf(("eval", "pairwise"), PAID),
     Leaf(("eval", "reference"), PAID),
     Leaf(("eval", "native-import"), FREE),
+    Leaf(("eval", "redteam"), PAID),
     Leaf(("eval", "panel-score"), PAID),
     Leaf(("dataset", "export"), FREE),
     Leaf(("ingest", "docling"), FREE),

dokimi/dokimi/commands/answer_formats.py

@@ -35,10 +35,9 @@ def cmd_questions_import(args: argparse.Namespace) -> None:
 def _ask(args: argparse.Namespace):
     if args.no_judge:
         return None
-    from dokimi.eval.judge import Judge
+    from dokimi.eval.judge import judge_ask
 
-    llm = Judge(model=args.judge_model, provider=args.judge_provider).llm
-    return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content
+    return judge_ask(args.judge_model, args.judge_provider)
 
 
 def cmd_eval_reference(args: argparse.Namespace) -> None:

dokimi/dokimi/commands/evaluate.py

@@ -580,6 +580,10 @@ def register(sub, positive_int) -> None:
 
     answer_formats.register_eval(eval_sub, positive_int)  # eval reference, eval native-import
 
+    from dokimi.commands import redteam
+
+    redteam.register_eval(eval_sub)  # eval redteam
+
     # eval panel-score
     panel_p = eval_sub.add_parser("panel-score", help="Score answers using a multi-judge panel")
     panel_p.add_argument("--answers", required=True, help="Path to answers JSON file")

dokimi/dokimi/commands/redteam.py

@@ -0,0 +1,72 @@
+# Copyright (C) 2026 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""``eval redteam``: check that a question set's grading rejects the answers it should.
+
+Reads a questions set (with ``grading`` blocks), grades synthetic attacks against it and prints
+the pass rates. Exits 1 when any attack is accepted, a canary is rejected, a graded question was not
+measured, or nothing was graded (``inconclusive``); exits 2 on an unreadable input, a malformed
+``--judge`` or an unwritable ``--out``. Judge checks are not scored unless ``--judge PROVIDER/MODEL``
+is given, so the default run makes no model call.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def _ask(args: argparse.Namespace):
+    if not args.judge:
+        return None
+    from dokimi.eval.judge import judge_ask
+
+    provider, model = args.judge
+    return judge_ask(model, provider)
+
+
+def _judge_ref(value: str) -> tuple[str, str]:
+    from dokimi.eval.judge import JUDGE_PROVIDERS
+
+    provider, _, model = value.partition("/")
+    if provider not in JUDGE_PROVIDERS or not model.strip():
+        raise argparse.ArgumentTypeError(
+            f"expected PROVIDER/MODEL with PROVIDER one of {', '.join(JUDGE_PROVIDERS)}, got {value!r}")
+    return provider, model.strip()
+
+
+def cmd_eval_redteam(args: argparse.Namespace) -> None:
+    from dokimi.artifacts import save_artifact
+    from dokimi.eval.redteam import format_table, run
+    from dokimi.questions.artifact import load_questions
+
+    try:
+        questions = load_questions(args.questions).artifact.get("questions") or []
+        result = run(questions, ask=_ask(args))
+    except (OSError, ValueError, RecursionError) as e:  # RecursionError: nested too deep
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(2)
+    result["questions_path"] = str(args.questions)
+    result["judge_model"] = args.judge[1] if args.judge else None
+    result["judge_provider"] = args.judge[0] if args.judge else None
+    print(format_table(result, output=args.out))
+    if args.out:
+        try:
+            save_artifact("redteam", result, args.out)
+        except (OSError, ValueError) as e:
+            print(f"Error: {e}", file=sys.stderr)
+            sys.exit(2)
+        print(f"Wrote redteam artifact to {args.out}")
+    if result["status"] != "pass":
+        sys.exit(1)
+
+
+def register_eval(eval_sub) -> None:
+    r = eval_sub.add_parser(
+        "redteam",
+        help="Check grading blocks reject wrong, empty and hedged answers and accept a canary")
+    r.add_argument("questions", help="questions JSON with grading blocks (from `questions import`)")
+    r.add_argument("--judge", metavar="PROVIDER/MODEL", type=_judge_ref, default=None,
+                   help="Grade judge checks with this judge model (model calls); default leaves them unscored")
+    r.add_argument("--out", metavar="FILE", default=None, help="redteam.v1 JSON to write")
+    r.set_defaults(func=cmd_eval_redteam)

dokimi/dokimi/eval/judge.py

@@ -678,3 +678,9 @@ def load_evaluations(input_path: Path) -> Dict[str, Any]:
     def _get_timestamp() -> str:
         from datetime import datetime, timezone
         return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z"
+
+
+def judge_ask(model: str, provider: str) -> Callable[..., str]:
+    """The ``Ask`` of the reference checks (one prompt in, the reply text out) backed by a ``Judge``'s model."""
+    llm = Judge(model=model, provider=provider).llm
+    return lambda prompt, draw=0, cache_if=None: llm.invoke(prompt, draw=draw, cache_if=cache_if).content