feat(catalog): known-positive guard for catalog trigger-probe

dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md

@@ -0,0 +1,14 @@
+### Added
+
+- **`catalog trigger-probe` known-positive guard** — before the labelled queries, one
+  explicit run (`Use the <skill> skill.`, for `--target` or the alphabetically first skill name of the catalog)
+  must load that skill. If it does not, the probe writes `"status": "invalid: skill not
+  loaded"` (Claude Code listed it) or `"invalid: skill not visible"` (it did not), with
+  `known_positive` holding the guard run, the probe's `max_turns_hit`/`not_listed`/`skill_selection_summary` keys
+  at 0, `[]` and `{}`, no answers, no stale `<out>.partial`, prints the same status on stdout, and
+  exits 1. A skill call whose result is an error, or another plugin's `plugin:skill`, does
+  not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run
+  that ended in an error writes `"status": "error: known-positive run failed"` instead. An
+  empty `--catalog` is refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. Pass
+  `--no-known-positive` to skip the check; the output then says `"status": "unchecked"` (or `invalid: skill not listed` when a
+  catalog skill is never listed).

dokimi/docs/evaluating-treatments.md

@@ -594,10 +594,20 @@ left on as a user has them (`--sealed` turns them off). The skill chosen is read
 transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the
 same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs
 that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code
-did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed`
+did not list a catalog skill (`not_listed`; `status` is then `invalid: skill not listed`, and the labelled results are still written). When no run listed any skill, `not_listed`
 stays empty and a warning on stderr says it was not checked. A trigger set with no
-labelled query exits 2 before any run. Records are kept in `<out>.partial` while the
-probe runs:
+labelled query exits 2 before any run. Before the labelled queries, one explicit run
+(`Use the <skill> skill.`, for the `--target` skill or the alphabetically first skill name of the
+catalog) must load that skill; if it does not, no labelled query runs, `status` is
+`invalid: skill not loaded` (Claude Code listed the skill but the request did not load it) or
+`invalid: skill not visible` (it did not list it) in the output and on stdout, and the command
+exits 1; a leftover `<out>.partial` is removed. A skill call that returned an error, or a
+`plugin:skill` of another plugin, does not count as a load. If that run ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is
+refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. This
+known-positive check is what separates "the model did not pick the skill" from "the
+harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked` (`invalid: skill not listed` still applies
+when a catalog skill is never listed).
+Records are kept in `<out>.partial` while the probe runs:
 
 ```bash
 python cli.py catalog trigger-probe --catalog <skills-checkout>/skills \

dokimi/docs/reference/cli-interface.json

@@ -2803,6 +2803,22 @@
               "required": false,
               "type": null
             },
+            {
+              "action": "_StoreTrueAction",
+              "choices": null,
+              "const": true,
+              "default": false,
+              "group_key": null,
+              "group_required": null,
+              "hidden": false,
+              "is_flag": true,
+              "key": "--no-known-positive",
+              "kind": "option",
+              "nargs": 0,
+              "position": null,
+              "required": false,
+              "type": null
+            },
             {
               "action": "_StoreAction",
               "choices": null,

dokimi/dokimi/commands/catalog_probe.py

@@ -16,6 +16,13 @@
 from dokimi.exit_codes import usage_exit
 
 
+#: Status of a run whose known-positive guard failed: no labelled query ran.
+_INVALID = "invalid: skill not visible"
+_NOT_LOADED = "invalid: skill not loaded"  # the harness listed the skill, the explicit request did not load it
+_ERROR = "error: known-positive run failed"  # the guard run ended in an error, not a missing skill
+_NOT_LISTED = "invalid: skill not listed"
+
+
 def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None:
     from dokimi.eval.arm_summary import summarize_skill_selection
     from dokimi.harnesses import trigger_probe as tp
@@ -47,6 +54,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None:
     cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout,
                          stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction,
                          claude=os.path.abspath(claude))
+    known = _known_positive_guard(args, names, catalog, cfg, out_path)
 
     done: list = []
     lock = threading.Lock()
@@ -73,26 +81,21 @@ def one(q):
     # a nested layout) reads as a 0% recall; say so instead of scoring it.
     not_listed = sorted(names - listed) if listed else []
     cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values())
+    cost += (known or {}).get("cost_usd") or 0
+    version = next((v["tool_usage"]["skill"]["claude_code_version"] for r in records
+                    for v in r["arms"].values() if v["tool_usage"]["skill"]["claude_code_version"]), None)
+    status = _NOT_LISTED if not_listed else ("ok" if known else "unchecked")
     out = {
-        "harness": "claude-code",
-        "mode": "sealed" if args.sealed else "stock",
-        "model": args.model,
-        "catalog": str(catalog),
-        "trials": args.trials,
-        "max_turns": args.max_turns,
-        "listing_budget_fraction": args.listing_budget_fraction,
-        "claude_code_version": next((v["tool_usage"]["skill"]["claude_code_version"] for r in records
-                                     for v in r["arms"].values()
-                                     if v["tool_usage"]["skill"]["claude_code_version"]), None),
-        "cost_usd": round(cost, 4),
+        **_header(args, catalog, known, status, cost, version),
         "max_turns_hit": sum(v["metrics"].get("result_subtype") == "error_max_turns" for v in runs),
         "not_listed": not_listed,
         "answers": records,
         "skill_selection_summary": summarize_skill_selection(records),
     }
     _write(out_path, out)
     partial.unlink(missing_ok=True)
-    print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"],
+    print(json.dumps({"status": out["status"], "cost_usd": out["cost_usd"],
+                      "claude_code_version": out["claude_code_version"],
                       "max_turns_hit": out["max_turns_hit"], "not_listed": not_listed,
                       "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")}
                                   for a, s in out["skill_selection_summary"].items()}}, indent=2))
@@ -101,6 +104,53 @@ def one(q):
         raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}")
 
 
+def _header(args: argparse.Namespace, catalog: Path, known, status: str, cost: float, version) -> dict:
+    """The fields every output carries, whether the run was probed or refused by the guard."""
+    return {
+        "harness": "claude-code",
+        "status": status,
+        "known_positive": known,
+        "mode": "sealed" if args.sealed else "stock",
+        "model": args.model,
+        "catalog": str(catalog),
+        "trials": args.trials,
+        "max_turns": args.max_turns,
+        "listing_budget_fraction": args.listing_budget_fraction,
+        "claude_code_version": version,
+        "cost_usd": round(cost, 4),
+    }
+
+
+def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, cfg, out_path: Path):
+    """The known-positive record, or None when skipped. A failed guard writes the invalid output and exits 1.
+
+    A skill the harness never loads on an explicit request would score every query 0%, which reads as a
+    finding; this refuses the run before any labelled query, which costs one query.
+    """
+    if args.no_known_positive:
+        return None
+    from dokimi.harnesses import trigger_probe as tp
+    from dokimi.utils import InputError
+
+    if not names:  # the guard's default target is the first catalog skill: with none, nothing to probe
+        raise InputError(f"--catalog holds no skills: {catalog}")
+    known = tp.known_positive(args.target or sorted(names)[0], catalog, cfg)
+    if known["passed"]:
+        return known
+    # A run that errored says so: "not visible" would read as the harness hiding the skill.
+    if known["error"]:
+        status = _ERROR
+    else:
+        status = _NOT_LOADED if known["skill"] in known["listed"] else _INVALID
+    _write(out_path, {**_header(args, catalog, known, status, known.get("cost_usd") or 0,
+                                known.get("claude_code_version")),
+                      "max_turns_hit": 0, "not_listed": [], "answers": [], "skill_selection_summary": {}})
+    out_path.with_name(out_path.name + ".partial").unlink(missing_ok=True)
+    print(json.dumps({"status": status, "known_positive": known}, indent=2))
+    raise SystemExit(f"{status}: {known['query']!r} did not load {known['skill']!r}"
+                     + (f" ({known['error']})" if known["error"] else "") + "; no labelled query was run")
+
+
 def _labelled_queries(args: argparse.Namespace, names: set) -> list:
     """The labelled queries to run; a label naming no *names* skill is an input error."""
     from dokimi.external import trigger_set as ts
@@ -152,5 +202,9 @@ def register(catalog_sub, positive_int) -> None:
     p.add_argument("--sealed", action="store_true",
                    help="Switch off bundled skills as the docker-claude arms do (default: stock harness)")
     p.add_argument("--claude", default="claude", help="claude executable (default: claude)")
+    p.add_argument("--no-known-positive", action="store_true", dest="no_known_positive",
+                   help="Skip the known-positive guard: one explicit run that must load a catalog skill "
+                        "before the labelled queries; a failed guard exits 1 with an 'invalid: ...' "
+                        "or 'error: ...' status")
     p.add_argument("--out", required=True, help="Output JSON path")
     p.set_defaults(func=cmd_catalog_trigger_probe)

dokimi/dokimi/harnesses/trigger_probe.py

@@ -145,7 +145,13 @@ def parse_stream(stream: str) -> Dict[str, Any]:
     # share message.id. Subagent events (parent_tool_use_id set) are not the choice.
     by_turn: Dict[str, List[str]] = {}
     other_tools: List[str] = []
+    ok_ids: set = set()  # tool_use ids whose tool_result arrived and is not an error
+    calls: List[tuple] = []  # (tool_use id, raw skill name) of top-level Skill calls
     for n, ev in enumerate(_events(stream)):
+        if ev.get("type") == "user":
+            for c in (ev.get("message") or {}).get("content") or []:
+                if isinstance(c, dict) and c.get("type") == "tool_result" and not c.get("is_error"):
+                    ok_ids.add(c.get("tool_use_id"))
         if ev.get("type") == "system" and ev.get("subtype") == "init":
             listed = [str(s) for s in ev.get("skills") or []]
             version = ev.get("claude_code_version")
@@ -154,6 +160,8 @@ def parse_stream(stream: str) -> Dict[str, Any]:
         message = ev.get("message") or {}
         uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"]
         skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"]
+        calls += [(c.get("id"), str((c.get("input") or {}).get("skill") or ""))
+                  for c in uses if c.get("name") == "Skill"]
         if not by_turn:  # tools after the choice are not "before" it
             other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"]
         if skills:
@@ -167,6 +175,9 @@ def parse_stream(stream: str) -> Dict[str, Any]:
         "selected": first[0] if first else None,
         "first_turn_loaded": first,
         "loaded": [s for t in turns for s in t],
+        # Raw names (plugin namespace kept) of Skill calls with an observed result that was not an error.
+        # A call whose result never arrived (a hang, a kill) did not load anything.
+        "loaded_ok": [name for tid, name in calls if tid is not None and tid in ok_ids],
         "tools_before_choice": other_tools,
         "claude_code_version": version,
     }
@@ -186,22 +197,48 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru
         skill = parse_stream(stream)
         result = result_event(stream) or {}
         error = None
+        # The failure as it is, before the Skill-call suppression below (the guard reports it).
+        terminal = None
+        if not result:
+            terminal = "timeout" if done.timed_out else f"no result event (exit {done.returncode})"
+        elif result.get("is_error") and result.get("subtype") != "error_max_turns":
+            terminal = str(result.get("subtype") or "error")
         # A Skill call seen is the choice, whatever came after it (a hang, a timeout, an
         # error). A result event read decides the rest, even if the CLI then hung.
         if skill["first_turn_loaded"]:
             pass
-        elif not result:
-            error = "timeout" if done.timed_out else f"no result event (exit {done.returncode})"
-        # Out of turns without a Skill call is an observation (nothing triggered), not a failure.
-        elif result.get("is_error") and result.get("subtype") != "error_max_turns":
-            error = str(result.get("subtype") or "error")
+        else:  # out of turns without a Skill call is an observation (nothing triggered), not a failure
+            error = terminal
         return {"skill": skill, "cost_usd": result.get("total_cost_usd"),
-                "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error}
+                "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error,
+                "terminal_error": terminal}
     finally:
         kill_strays(ws.root)
         ws.cleanup()
 
 
+def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Runner = process.run,
+                   source_env: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
+    """One run of an explicit request for *skill*, through the same harness; ``passed`` if it loaded it.
+
+    A harness that never shows the skill scores every labelled query 0%, which reads as a
+    finding. This run is the check that the skill can be loaded at all, before those queries.
+    """
+    query = f"Use the {skill} skill."
+    try:
+        r = run_query(query, catalog, cfg, runner=runner, source_env=source_env)
+    except (OSError, RuntimeError) as exc:
+        return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None,
+                "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"}
+    # The guard passes on an observed, successful call of exactly this skill (not another plugin's) with no
+    # terminal error; ``listed`` records whether the harness showed it.
+    passed = skill in r["skill"]["loaded_ok"] and r["terminal_error"] is None
+    return {"skill": skill, "query": query, "passed": passed,
+            "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"],
+            "claude_code_version": r["skill"]["claude_code_version"],
+            "error": None if passed else r["terminal_error"]}
+
+
 def kill_strays(root: Path) -> List[int]:
     """SIGKILL every process whose cwd is under *root*; return their pids (Linux only).
 

dokimi/tests/data/cli_flags_allowlist.txt

@@ -119,6 +119,7 @@
 --no-confirm-carriers
 --no-json
 --no-judge
+--no-known-positive  # catalog trigger-probe known-positive guard: no registry concept yet
 --no-paraphrase
 --no-plugin-deltas
 --no-recursive