From b64cb67e16cc609ef36c0929ebc1b7915b01d7bb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 30 Oct 2026 21:53:00 -0700 Subject: [PATCH 01/11] feat(catalog): known-positive guard for catalog trigger-probe Before the labelled queries, one explicit run ('Use the skill.') must load that skill. If it does not, the probe writes status 'invalid: skill not visible', prints it, exits 1 and runs no labelled query, so a skill the harness never shows cannot read as a 0% trigger rate. --no-known-positive skips the check (status 'unchecked'). Signed-off-by: Nikolay Petrov --- ...30_120000_napetrov_known-positive-guard.md | 8 +++ dokimi/docs/evaluating-treatments.md | 9 ++- dokimi/docs/reference/cli-interface.json | 16 +++++ dokimi/dokimi/commands/catalog_probe.py | 36 ++++++++++- dokimi/dokimi/harnesses/trigger_probe.py | 17 ++++++ dokimi/tests/data/cli_flags_allowlist.txt | 1 + dokimi/tests/test_trigger_probe.py | 61 ++++++++++++++++++- 7 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md new file mode 100644 index 000000000..f96d9717b --- /dev/null +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -0,0 +1,8 @@ +### Added + +- **`catalog trigger-probe` known-positive guard** — before the labelled queries, one + explicit run (`Use the skill.`, for `--target` or the first skill of the catalog) + must load that skill. If it does not, the probe writes `"status": "invalid: skill not + visible"` with `known_positive` and no answers, prints the same status on stdout, and + exits 1, so a skill the harness never shows cannot read as a 0% trigger rate. Pass + `--no-known-positive` to skip the check; the output then says `"status": "unchecked"`. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 7219d8f44..23e0d87b5 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -596,8 +596,13 @@ same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` cou that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed` stays empty and a warning on stderr says it was not checked. A trigger set with no -labelled query exits 2 before any run. Records are kept in `.partial` while the -probe runs: +labelled query exits 2 before any run. Before the labelled queries, one explicit run +(`Use the skill.`, for the `--target` skill or the first skill of the catalog) must +load that skill; if it does not, no labelled query runs, `status` is +`invalid: skill not visible` in the output and on stdout, and the command exits 1. This +known-positive check is what separates "the model did not pick the skill" from "the +harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked`. +Records are kept in `.partial` while the probe runs: ```bash python cli.py catalog trigger-probe --catalog /skills \ diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..a0eed3b28 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -2803,6 +2803,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-known-positive", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 4cfb97a28..a8c7342ee 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -16,6 +16,10 @@ from dokimi.exit_codes import usage_exit +#: Status of a run whose known-positive guard failed: no labelled query ran. +_INVALID = "invalid: skill not visible" + + def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.harnesses import trigger_probe as tp @@ -47,6 +51,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, claude=os.path.abspath(claude)) + known = _known_positive_guard(args, names, catalog, cfg, out_path) done: list = [] lock = threading.Lock() @@ -73,8 +78,11 @@ def one(q): # a nested layout) reads as a 0% recall; say so instead of scoring it. not_listed = sorted(names - listed) if listed else [] cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) + cost += (known or {}).get("cost_usd") or 0 out = { "harness": "claude-code", + "status": "ok" if known else "unchecked", + "known_positive": known, "mode": "sealed" if args.sealed else "stock", "model": args.model, "catalog": str(catalog), @@ -92,7 +100,8 @@ def one(q): } _write(out_path, out) partial.unlink(missing_ok=True) - print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], + print(json.dumps({"status": out["status"], "cost_usd": out["cost_usd"], + "claude_code_version": out["claude_code_version"], "max_turns_hit": out["max_turns_hit"], "not_listed": not_listed, "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} for a, s in out["skill_selection_summary"].items()}}, indent=2)) @@ -101,6 +110,27 @@ def one(q): raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}") +def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, cfg, out_path: Path): + """The known-positive record, or None when skipped. A failed guard writes the invalid output and exits 1. + + A skill the harness never loads on an explicit request would score every query 0%, which reads as a + finding; this refuses the run before any labelled query, which costs one query. + """ + if args.no_known_positive: + return None + from dokimi.harnesses import trigger_probe as tp + + known = tp.known_positive(args.target or sorted(names)[0], catalog, cfg) + if known["passed"]: + return known + _write(out_path, {"harness": "claude-code", "status": _INVALID, "known_positive": known, + "mode": "sealed" if args.sealed else "stock", "model": args.model, + "catalog": str(catalog), "answers": []}) + print(json.dumps({"status": _INVALID, "known_positive": known}, indent=2)) + raise SystemExit(f"{_INVALID}: {known['query']!r} did not load {known['skill']!r}" + + (f" ({known['error']})" if known["error"] else "") + "; no labelled query was run") + + def _labelled_queries(args: argparse.Namespace, names: set) -> list: """The labelled queries to run; a label naming no *names* skill is an input error.""" from dokimi.external import trigger_set as ts @@ -152,5 +182,9 @@ def register(catalog_sub, positive_int) -> None: p.add_argument("--sealed", action="store_true", help="Switch off bundled skills as the docker-claude arms do (default: stock harness)") p.add_argument("--claude", default="claude", help="claude executable (default: claude)") + p.add_argument("--no-known-positive", action="store_true", dest="no_known_positive", + help="Skip the known-positive guard: one explicit run that must load a catalog skill " + "before the labelled queries; a failed guard exits 1 with status " + "'invalid: skill not visible'") p.add_argument("--out", required=True, help="Output JSON path") p.set_defaults(func=cmd_catalog_trigger_probe) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 328078be1..6b18c0274 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -202,6 +202,23 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru ws.cleanup() +def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Runner = process.run, + source_env: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """One run of an explicit request for *skill*, through the same harness; ``passed`` if it loaded it. + + A harness that never shows the skill scores every labelled query 0%, which reads as a + finding. This run is the check that the skill can be loaded at all, before those queries. + """ + query = f"Use the {skill} skill." + try: + r = run_query(query, catalog, cfg, runner=runner, source_env=source_env) + except (OSError, RuntimeError) as exc: + return {"skill": skill, "query": query, "passed": False, "loaded": [], "cost_usd": None, + "error": f"{type(exc).__name__}: {exc}"} + return {"skill": skill, "query": query, "passed": skill in r["skill"]["loaded"], + "loaded": r["skill"]["loaded"], "cost_usd": r["cost_usd"], "error": r["error"]} + + def kill_strays(root: Path) -> List[int]: """SIGKILL every process whose cwd is under *root*; return their pids (Linux only). diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..55edc434c 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -119,6 +119,7 @@ --no-confirm-carriers --no-json --no-judge +--no-known-positive # catalog trigger-probe known-positive guard: no registry concept yet --no-paraphrase --no-plugin-deltas --no-recursive diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 81bad7980..1c6935a7e 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -219,9 +219,11 @@ def test_cli_runs_a_relative_claude_from_the_scratch_cwd(tmp_path, monkeypatch, _catalog(tmp_path) monkeypatch.chdir(tmp_path) args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", - "ts.json", "--claude", "bin/claude", "--out", "probe.json"]) + "ts.json", "--claude", "bin/claude", "--target", "sum-fast", + "--out", "probe.json"]) args.func(args) out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "ok" and out["known_positive"]["passed"] is True arm = out["answers"][0]["arms"]["claude_code#1"] assert arm["error"] is None and arm["tool_usage"]["skill"]["first_turn_loaded"] == ["sum-fast"] @@ -311,11 +313,16 @@ def runner(argv, **kw): shutil.rmtree(procs["sibling_dir"], ignore_errors=True) -def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast"): +def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast", known_passed=True): from dokimi.commands import catalog_probe seen = {} + def fake_known(skill, catalog, cfg, **kw): + seen.setdefault("known", []).append(skill) + return {"skill": skill, "query": f"Use the {skill} skill.", "passed": known_passed, + "loaded": [skill] if known_passed else [], "cost_usd": 0.01, "error": None} + def fake_probe(queries, catalog, cfg, *, trials=1, **kw): seen.update(cfg=cfg, trials=trials, catalog=catalog) seen.setdefault("queries", []).extend(q.id for q in queries) @@ -332,6 +339,7 @@ def __init__(self, n): super().__init__(n) monkeypatch.setattr(tp, "probe_trigger_set", fake_probe) + monkeypatch.setattr(tp, "known_positive", fake_known) monkeypatch.setattr(catalog_probe, "ThreadPoolExecutor", Pool) (tmp_path / "ts.json").write_text(json.dumps({"questions": [ {"id": q, "question": f"q {q}", "expected_skill": expected} for q in ("a", "b", "c")]}), @@ -416,3 +424,52 @@ def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monke with pytest.raises(SystemExit, match="did not list: io-read"): _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz")) assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == ["io-read"] + + +@pytest.mark.parametrize("stream, passed", [(_stream(["sum-fast"]), True), (_stream(), False)], + ids=["loads-the-skill", "never-loads-it"]) +def test_known_positive_passes_only_when_the_explicit_request_loads_the_skill(tmp_path, stream, passed): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + assert argv[list(argv).index("-p") + 1] == "Use the sum-fast skill." + return process.reply(stream) + + r = tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["passed"] is passed and r["error"] is None + + +def test_known_positive_that_cannot_run_fails_without_raising(tmp_path): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + raise FileNotFoundError(2, "No such file or directory", argv[0]) + + r = tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["passed"] is False and r["error"].startswith("FileNotFoundError") + + +def test_cli_probes_the_target_or_the_first_skill_before_the_set(tmp_path, monkeypatch): + assert _cli(tmp_path, monkeypatch, "--out", "probe.json")["known"] == ["io-read"] + assert _cli(tmp_path, monkeypatch, "--target", "sum-fast", "--out", "probe.json")["known"] == ["sum-fast"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "ok" and out["known_positive"]["passed"] is True + + +def test_cli_stops_before_the_labelled_runs_when_the_skill_is_not_visible(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="invalid: skill not visible"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "invalid: skill not visible" and out["answers"] == [] + assert out["known_positive"]["passed"] is False + assert not (tmp_path / "probe.json.partial").exists() + assert '"status": "invalid: skill not visible"' in capsys.readouterr().out + + +def test_no_known_positive_skips_the_guard(tmp_path, monkeypatch): + seen = _cli(tmp_path, monkeypatch, "--no-known-positive", "--out", "probe.json", known_passed=False) + assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "unchecked" and out["known_positive"] is None From 48cbd0fc9997bd6c740d381734ecea1432f1e965 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 12:37:22 -0700 Subject: [PATCH 02/11] Address review: reject an empty catalog before the guard A negative-only trigger set against a catalog with no skills reached sorted(names)[0] in the known-positive guard and raised IndexError. Refuse the empty catalog as an input error before the guard picks its skill. Runs with --no-known-positive are unchanged. Cover both with regression tests. Thread: PRRT_kwDOVBg7g86yOGu2 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/catalog_probe.py | 3 +++ dokimi/tests/test_trigger_probe.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index a8c7342ee..607ac6861 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -119,7 +119,10 @@ def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, c if args.no_known_positive: return None from dokimi.harnesses import trigger_probe as tp + from dokimi.utils import InputError + if not names: # the guard's default target is the first catalog skill: with none, nothing to probe + raise InputError(f"--catalog holds no skills: {catalog}") known = tp.known_positive(args.target or sorted(names)[0], catalog, cfg) if known["passed"]: return known diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 1c6935a7e..f6284cb4e 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -473,3 +473,27 @@ def test_no_known_positive_skips_the_guard(tmp_path, monkeypatch): assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) assert out["status"] == "unchecked" and out["known_positive"] is None + + +def test_cli_refuses_an_empty_catalog_before_the_guard_picks_a_skill(tmp_path, monkeypatch): + # A negative-only set has no label naming a skill, so only the guard's default target could + # name one; with no skill in the catalog that was an IndexError, not an input error. + from dokimi.utils import InputError + + (tmp_path / "cat").mkdir() + with pytest.raises(InputError, match="--catalog holds no skills"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=None) + assert not (tmp_path / "probe.json.partial").exists() and not (tmp_path / "probe.json").exists() + + +def test_no_known_positive_still_runs_a_negative_only_set_on_an_empty_catalog(tmp_path, monkeypatch): + # The guard is what needs a skill; with it skipped the run is what it was before the refusal. + (tmp_path / "cat").mkdir() + seen = _cli(tmp_path, monkeypatch, "--no-known-positive", "--out", "probe.json", expected=None) + assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] + + +def test_negative_only_set_on_a_catalog_with_skills_still_runs_the_guard(tmp_path, monkeypatch): + # names come from the catalog's skills, not the labels: a negative-only set on a non-empty + # catalog is not refused, and the guard probes the first catalog skill as before. + assert _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=None)["known"] == ["io-read"] From c60be7d9109d99de3edb2a5e53b2fcabf7ce5e43 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:30:41 -0700 Subject: [PATCH 03/11] fix(probe): a guard run that errors is not "skill not visible" An error in the known-positive run (timeout, API or auth error, an error result) wrote "invalid: skill not visible", which reads as the harness hiding the skill. It now writes "error: known-positive run failed"; "invalid: skill not visible" is kept for a run that finished without an error and did not load the skill. - not_listed no longer reports status "ok" when the same run exits 1: the status is "invalid: skill not listed". - Every output carries the same header (trials, max_turns, listing_budget_fraction, claude_code_version, cost_usd), built by one helper; the refused output carries the guard's cost. - known_positive keeps "listed" and "claude_code_version". - The docs and changelog say "alphabetically first skill name", and that an empty catalog is refused before the guard runs. Tests: an errored guard run, the not-listed status, the invalid output's header and cost. Signed-off-by: Nikolay Petrov --- ...30_120000_napetrov_known-positive-guard.md | 6 ++- dokimi/docs/evaluating-treatments.md | 8 ++-- dokimi/dokimi/commands/catalog_probe.py | 47 ++++++++++++------- dokimi/dokimi/harnesses/trigger_probe.py | 8 ++-- dokimi/tests/test_trigger_probe.py | 25 ++++++++-- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md index f96d9717b..a69c05813 100644 --- a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -1,8 +1,10 @@ ### Added - **`catalog trigger-probe` known-positive guard** — before the labelled queries, one - explicit run (`Use the skill.`, for `--target` or the first skill of the catalog) + explicit run (`Use the skill.`, for `--target` or the alphabetically first skill name of the catalog) must load that skill. If it does not, the probe writes `"status": "invalid: skill not visible"` with `known_positive` and no answers, prints the same status on stdout, and - exits 1, so a skill the harness never shows cannot read as a 0% trigger rate. Pass + exits 1, so a skill the harness never shows cannot read as a 0% trigger rate. A guard run + that ended in an error writes `"status": "error: known-positive run failed"` instead. An + empty `--catalog` is refused (exit 2) before the guard runs. Pass `--no-known-positive` to skip the check; the output then says `"status": "unchecked"`. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 23e0d87b5..7c6b2dfea 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -597,9 +597,11 @@ that ended on `--max-turns` (scored as no load), and the command exits 1 when Cl did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed` stays empty and a warning on stderr says it was not checked. A trigger set with no labelled query exits 2 before any run. Before the labelled queries, one explicit run -(`Use the skill.`, for the `--target` skill or the first skill of the catalog) must -load that skill; if it does not, no labelled query runs, `status` is -`invalid: skill not visible` in the output and on stdout, and the command exits 1. This +(`Use the skill.`, for the `--target` skill or the alphabetically first skill name of the +catalog) must load that skill; if it does not, no labelled query runs, `status` is +`invalid: skill not visible` in the output and on stdout, and the command exits 1. If that run +ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is +refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. This known-positive check is what separates "the model did not pick the skill" from "the harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked`. Records are kept in `.partial` while the probe runs: diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 607ac6861..37eb87d89 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -18,6 +18,8 @@ #: Status of a run whose known-positive guard failed: no labelled query ran. _INVALID = "invalid: skill not visible" +_ERROR = "error: known-positive run failed" # the guard run ended in an error, not a missing skill +_NOT_LISTED = "invalid: skill not listed" def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: @@ -79,20 +81,11 @@ def one(q): not_listed = sorted(names - listed) if listed else [] cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) cost += (known or {}).get("cost_usd") or 0 + version = next((v["tool_usage"]["skill"]["claude_code_version"] for r in records + for v in r["arms"].values() if v["tool_usage"]["skill"]["claude_code_version"]), None) + status = _NOT_LISTED if not_listed else ("ok" if known else "unchecked") out = { - "harness": "claude-code", - "status": "ok" if known else "unchecked", - "known_positive": known, - "mode": "sealed" if args.sealed else "stock", - "model": args.model, - "catalog": str(catalog), - "trials": args.trials, - "max_turns": args.max_turns, - "listing_budget_fraction": args.listing_budget_fraction, - "claude_code_version": next((v["tool_usage"]["skill"]["claude_code_version"] for r in records - for v in r["arms"].values() - if v["tool_usage"]["skill"]["claude_code_version"]), None), - "cost_usd": round(cost, 4), + **_header(args, catalog, known, status, cost, version), "max_turns_hit": sum(v["metrics"].get("result_subtype") == "error_max_turns" for v in runs), "not_listed": not_listed, "answers": records, @@ -110,6 +103,23 @@ def one(q): raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}") +def _header(args: argparse.Namespace, catalog: Path, known, status: str, cost: float, version) -> dict: + """The fields every output carries, whether the run was probed or refused by the guard.""" + return { + "harness": "claude-code", + "status": status, + "known_positive": known, + "mode": "sealed" if args.sealed else "stock", + "model": args.model, + "catalog": str(catalog), + "trials": args.trials, + "max_turns": args.max_turns, + "listing_budget_fraction": args.listing_budget_fraction, + "claude_code_version": version, + "cost_usd": round(cost, 4), + } + + def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, cfg, out_path: Path): """The known-positive record, or None when skipped. A failed guard writes the invalid output and exits 1. @@ -126,11 +136,12 @@ def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, c known = tp.known_positive(args.target or sorted(names)[0], catalog, cfg) if known["passed"]: return known - _write(out_path, {"harness": "claude-code", "status": _INVALID, "known_positive": known, - "mode": "sealed" if args.sealed else "stock", "model": args.model, - "catalog": str(catalog), "answers": []}) - print(json.dumps({"status": _INVALID, "known_positive": known}, indent=2)) - raise SystemExit(f"{_INVALID}: {known['query']!r} did not load {known['skill']!r}" + # A run that errored says so: "not visible" would read as the harness hiding the skill. + status = _ERROR if known["error"] else _INVALID + _write(out_path, {**_header(args, catalog, known, status, known.get("cost_usd") or 0, + known.get("claude_code_version")), "answers": []}) + print(json.dumps({"status": status, "known_positive": known}, indent=2)) + raise SystemExit(f"{status}: {known['query']!r} did not load {known['skill']!r}" + (f" ({known['error']})" if known["error"] else "") + "; no labelled query was run") diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 6b18c0274..3e8bf2cf3 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -213,10 +213,12 @@ def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: proce try: r = run_query(query, catalog, cfg, runner=runner, source_env=source_env) except (OSError, RuntimeError) as exc: - return {"skill": skill, "query": query, "passed": False, "loaded": [], "cost_usd": None, - "error": f"{type(exc).__name__}: {exc}"} + return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None, + "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"} + # The guard passes on the Skill call alone; ``listed`` records whether the harness showed the skill. return {"skill": skill, "query": query, "passed": skill in r["skill"]["loaded"], - "loaded": r["skill"]["loaded"], "cost_usd": r["cost_usd"], "error": r["error"]} + "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"], + "claude_code_version": r["skill"]["claude_code_version"], "error": r["error"]} def kill_strays(root: Path) -> List[int]: diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index f6284cb4e..7628ad56d 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -313,7 +313,8 @@ def runner(argv, **kw): shutil.rmtree(procs["sibling_dir"], ignore_errors=True) -def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast", known_passed=True): +def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast", known_passed=True, + known_error=None): from dokimi.commands import catalog_probe seen = {} @@ -321,7 +322,8 @@ def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), exp def fake_known(skill, catalog, cfg, **kw): seen.setdefault("known", []).append(skill) return {"skill": skill, "query": f"Use the {skill} skill.", "passed": known_passed, - "loaded": [skill] if known_passed else [], "cost_usd": 0.01, "error": None} + "loaded": [skill] if known_passed else [], "listed": list(init_skills), "cost_usd": 0.01, + "claude_code_version": "2.1.3xx", "error": known_error} def fake_probe(queries, catalog, cfg, *, trials=1, **kw): seen.update(cfg=cfg, trials=trials, catalog=catalog) @@ -423,7 +425,24 @@ def test_cli_warns_when_no_run_listed_any_skill(tmp_path, monkeypatch, capsys): def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch): with pytest.raises(SystemExit, match="did not list: io-read"): _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz")) - assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == ["io-read"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["not_listed"] == ["io-read"] and out["status"] == "invalid: skill not listed" + + +def test_cli_reports_a_guard_run_that_errored_apart_from_a_skill_not_shown(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="error: known-positive run failed"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, known_error="error_during_execution") + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "error: known-positive run failed" and out["answers"] == [] + assert '"status": "error: known-positive run failed"' in capsys.readouterr().out + + +def test_the_invalid_output_has_the_header_fields_and_the_guard_cost(tmp_path, monkeypatch): + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["cost_usd"] == 0.01 and out["claude_code_version"] == "2.1.3xx" + assert out["trials"] == 1 and out["max_turns"] and "listing_budget_fraction" in out @pytest.mark.parametrize("stream, passed", [(_stream(["sum-fast"]), True), (_stream(), False)], From ca1849a643251de9bcd26a3fb3e07a16cbfa06a8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 23:45:06 -0700 Subject: [PATCH 04/11] fix(probe): known-positive guard needs a successful exact load, keeps the real failure Skill results with is_error, plugin-namespaced names and a timeout after another skill call no longer pass or mask the guard; a listed-but-unloaded skill is 'invalid: skill not loaded'; the refusal output carries the probe keys and drops a stale .partial. Signed-off-by: Nikolay Petrov --- ...30_120000_napetrov_known-positive-guard.md | 7 ++- dokimi/docs/evaluating-treatments.md | 6 +- dokimi/dokimi/commands/catalog_probe.py | 10 +++- dokimi/dokimi/harnesses/trigger_probe.py | 35 ++++++++--- dokimi/tests/test_trigger_probe.py | 58 ++++++++++++++++++- 5 files changed, 100 insertions(+), 16 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md index a69c05813..5925b26f1 100644 --- a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -3,8 +3,11 @@ - **`catalog trigger-probe` known-positive guard** — before the labelled queries, one explicit run (`Use the skill.`, for `--target` or the alphabetically first skill name of the catalog) must load that skill. If it does not, the probe writes `"status": "invalid: skill not - visible"` with `known_positive` and no answers, prints the same status on stdout, and - exits 1, so a skill the harness never shows cannot read as a 0% trigger rate. A guard run + loaded"` (Claude Code listed it) or `"invalid: skill not visible"` (it did not), with + `known_positive`, the probe's `max_turns_hit`/`not_listed`/`skill_selection_summary` keys + empty, no answers, no stale `.partial`, prints the same status on stdout, and + exits 1. A skill call whose result is an error, or another plugin's `plugin:skill`, does + not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run that ended in an error writes `"status": "error: known-positive run failed"` instead. An empty `--catalog` is refused (exit 2) before the guard runs. Pass `--no-known-positive` to skip the check; the output then says `"status": "unchecked"`. diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 7c6b2dfea..a7a57b9f3 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -599,8 +599,10 @@ stays empty and a warning on stderr says it was not checked. A trigger set with labelled query exits 2 before any run. Before the labelled queries, one explicit run (`Use the skill.`, for the `--target` skill or the alphabetically first skill name of the catalog) must load that skill; if it does not, no labelled query runs, `status` is -`invalid: skill not visible` in the output and on stdout, and the command exits 1. If that run -ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is +`invalid: skill not loaded` (Claude Code listed the skill but the request did not load it) or +`invalid: skill not visible` (it did not list it) in the output and on stdout, and the command +exits 1; a leftover `.partial` is removed. A skill call that returned an error, or a +`plugin:skill` of another plugin, does not count as a load. If that run ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. This known-positive check is what separates "the model did not pick the skill" from "the harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked`. diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 37eb87d89..38632a008 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -18,6 +18,7 @@ #: Status of a run whose known-positive guard failed: no labelled query ran. _INVALID = "invalid: skill not visible" +_NOT_LOADED = "invalid: skill not loaded" # the harness listed the skill, the explicit request did not load it _ERROR = "error: known-positive run failed" # the guard run ended in an error, not a missing skill _NOT_LISTED = "invalid: skill not listed" @@ -137,9 +138,14 @@ def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, c if known["passed"]: return known # A run that errored says so: "not visible" would read as the harness hiding the skill. - status = _ERROR if known["error"] else _INVALID + if known["error"]: + status = _ERROR + else: + status = _NOT_LOADED if known["skill"] in known["listed"] else _INVALID _write(out_path, {**_header(args, catalog, known, status, known.get("cost_usd") or 0, - known.get("claude_code_version")), "answers": []}) + known.get("claude_code_version")), + "max_turns_hit": 0, "not_listed": [], "answers": [], "skill_selection_summary": {}}) + out_path.with_name(out_path.name + ".partial").unlink(missing_ok=True) print(json.dumps({"status": status, "known_positive": known}, indent=2)) raise SystemExit(f"{status}: {known['query']!r} did not load {known['skill']!r}" + (f" ({known['error']})" if known["error"] else "") + "; no labelled query was run") diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 3e8bf2cf3..11726ce3b 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -145,7 +145,13 @@ def parse_stream(stream: str) -> Dict[str, Any]: # share message.id. Subagent events (parent_tool_use_id set) are not the choice. by_turn: Dict[str, List[str]] = {} other_tools: List[str] = [] + failed_ids: set = set() # tool_use ids whose tool_result is an error + calls: List[tuple] = [] # (tool_use id, raw skill name) of top-level Skill calls for n, ev in enumerate(_events(stream)): + if ev.get("type") == "user": + for c in (ev.get("message") or {}).get("content") or []: + if isinstance(c, dict) and c.get("type") == "tool_result" and c.get("is_error"): + failed_ids.add(c.get("tool_use_id")) if ev.get("type") == "system" and ev.get("subtype") == "init": listed = [str(s) for s in ev.get("skills") or []] version = ev.get("claude_code_version") @@ -154,6 +160,8 @@ def parse_stream(stream: str) -> Dict[str, Any]: message = ev.get("message") or {} uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"] skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] + calls += [(c.get("id"), str((c.get("input") or {}).get("skill") or "")) + for c in uses if c.get("name") == "Skill"] if not by_turn: # tools after the choice are not "before" it other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] if skills: @@ -167,6 +175,8 @@ def parse_stream(stream: str) -> Dict[str, Any]: "selected": first[0] if first else None, "first_turn_loaded": first, "loaded": [s for t in turns for s in t], + # Raw names (plugin namespace kept) of Skill calls whose result was not an error. + "loaded_ok": [name for tid, name in calls if tid is None or tid not in failed_ids], "tools_before_choice": other_tools, "claude_code_version": version, } @@ -186,17 +196,21 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru skill = parse_stream(stream) result = result_event(stream) or {} error = None + # The failure as it is, before the Skill-call suppression below (the guard reports it). + terminal = None + if not result: + terminal = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" + elif result.get("is_error") and result.get("subtype") != "error_max_turns": + terminal = str(result.get("subtype") or "error") # A Skill call seen is the choice, whatever came after it (a hang, a timeout, an # error). A result event read decides the rest, even if the CLI then hung. if skill["first_turn_loaded"]: pass - elif not result: - error = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" - # Out of turns without a Skill call is an observation (nothing triggered), not a failure. - elif result.get("is_error") and result.get("subtype") != "error_max_turns": - error = str(result.get("subtype") or "error") + else: # out of turns without a Skill call is an observation (nothing triggered), not a failure + error = terminal return {"skill": skill, "cost_usd": result.get("total_cost_usd"), - "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error} + "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error, + "terminal_error": terminal} finally: kill_strays(ws.root) ws.cleanup() @@ -215,10 +229,13 @@ def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: proce except (OSError, RuntimeError) as exc: return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None, "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"} - # The guard passes on the Skill call alone; ``listed`` records whether the harness showed the skill. - return {"skill": skill, "query": query, "passed": skill in r["skill"]["loaded"], + # The guard passes on a successful call of exactly this skill (a `plugin:skill` is another one); + # ``listed`` records whether the harness showed it. A failure is reported even after a Skill call. + passed = skill in r["skill"]["loaded_ok"] + return {"skill": skill, "query": query, "passed": passed, "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"], - "claude_code_version": r["skill"]["claude_code_version"], "error": r["error"]} + "claude_code_version": r["skill"]["claude_code_version"], + "error": None if passed else r["terminal_error"]} def kill_strays(root: Path) -> List[int]: diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 7628ad56d..b21f5a0c1 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -459,6 +459,40 @@ def runner(argv, **kw): assert r["passed"] is passed and r["error"] is None +def _skill_stream(name, *, is_error=False, result=True): + events = [{"type": "system", "subtype": "init", "skills": ["sum-fast"], "claude_code_version": "2.1.304"}, + {"type": "assistant", "message": {"id": "m1", "content": [ + {"type": "tool_use", "id": "tu1", "name": "Skill", "input": {"skill": name}}]}}, + {"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "tu1", "is_error": is_error, "content": "x"}]}}] + if result: + events.append({"type": "result", "subtype": "success", "is_error": False, "total_cost_usd": 0.01, "num_turns": 2}) + return "\n".join(json.dumps(e) for e in events) + "\n" + + +def _known(tmp_path, stream, timed_out=False): + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream, timed_out=timed_out) + + return tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + + +@pytest.mark.parametrize("name, is_error, passed", [ + ("sum-fast", False, True), ("sum-fast", True, False), ("plug:sum-fast", False, False)], + ids=["loaded", "result-is-error", "other-namespace"]) +def test_known_positive_needs_the_exact_skill_loaded_without_error(tmp_path, name, is_error, passed): + r = _known(tmp_path, _skill_stream(name, is_error=is_error)) + assert r["passed"] is passed + # Labelled scoring still sees the namespace-stripped name. + assert r["loaded"] == ["sum-fast"] + + +def test_known_positive_keeps_the_timeout_when_another_skill_was_called_first(tmp_path): + r = _known(tmp_path, _skill_stream("io-read", result=False), timed_out=True) + assert r["passed"] is False and r["error"] == "timeout" + + def test_known_positive_that_cannot_run_fails_without_raising(tmp_path): def runner(argv, **kw): if argv[0] == "git": @@ -479,7 +513,7 @@ def test_cli_probes_the_target_or_the_first_skill_before_the_set(tmp_path, monke def test_cli_stops_before_the_labelled_runs_when_the_skill_is_not_visible(tmp_path, monkeypatch, capsys): with pytest.raises(SystemExit, match="invalid: skill not visible"): - _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, init_skills=("sum-fast",)) out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) assert out["status"] == "invalid: skill not visible" and out["answers"] == [] assert out["known_positive"]["passed"] is False @@ -487,6 +521,28 @@ def test_cli_stops_before_the_labelled_runs_when_the_skill_is_not_visible(tmp_pa assert '"status": "invalid: skill not visible"' in capsys.readouterr().out +def test_cli_says_not_loaded_when_the_harness_listed_the_skill(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="invalid: skill not loaded"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "invalid: skill not loaded" + assert '"status": "invalid: skill not loaded"' in capsys.readouterr().out + + +def test_the_refusal_output_has_the_probe_keys_of_an_unrun_probe(tmp_path, monkeypatch): + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, init_skills=("sum-fast",)) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["max_turns_hit"] == 0 and out["not_listed"] == [] and out["skill_selection_summary"] == {} + + +def test_a_refusal_removes_the_stale_partial_of_an_earlier_run(tmp_path, monkeypatch): + (tmp_path / "probe.json.partial").write_text("{}", encoding="utf-8") + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + assert not (tmp_path / "probe.json.partial").exists() + + def test_no_known_positive_skips_the_guard(tmp_path, monkeypatch): seen = _cli(tmp_path, monkeypatch, "--no-known-positive", "--out", "probe.json", known_passed=False) assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] From 89285d7a620ffafb74891c163d0238d1a5c31a0e Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:36:23 +0000 Subject: [PATCH 05/11] Name the error and invalid statuses in the guard help text Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/catalog_probe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 38632a008..db6119027 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -204,7 +204,7 @@ def register(catalog_sub, positive_int) -> None: p.add_argument("--claude", default="claude", help="claude executable (default: claude)") p.add_argument("--no-known-positive", action="store_true", dest="no_known_positive", help="Skip the known-positive guard: one explicit run that must load a catalog skill " - "before the labelled queries; a failed guard exits 1 with status " - "'invalid: skill not visible'") + "before the labelled queries; a failed guard exits 1 with an 'invalid: ...' " + "or 'error: ...' status") p.add_argument("--out", required=True, help="Output JSON path") p.set_defaults(func=cmd_catalog_trigger_probe) From e38ba84e91cf7374bf83b0606bc9fd678bb055c9 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 14:36:43 +0000 Subject: [PATCH 06/11] Document the skill not listed status in the trigger-probe docs Signed-off-by: Anton Volkov --- dokimi/docs/evaluating-treatments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index a7a57b9f3..1f8887fb5 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -594,7 +594,7 @@ left on as a user has them (`--sealed` turns them off). The skill chosen is read transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code -did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed` +did not list a catalog skill (`not_listed`; `status` is then `invalid: skill not listed`, and the labelled results are still written). When no run listed any skill, `not_listed` stays empty and a warning on stderr says it was not checked. A trigger set with no labelled query exits 2 before any run. Before the labelled queries, one explicit run (`Use the skill.`, for the `--target` skill or the alphabetically first skill name of the From 66cb1632686d5c3a1f828e35fbc0510228f21659 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:55:26 -0800 Subject: [PATCH 07/11] fix(probe): the known-positive guard needs a loaded result, and docs say so A Skill call counts as loaded only when its tool_result arrived and was not an error, so a call that hung or was killed is no longer a pass. A terminal error after the target loaded still fails the guard and is reported. The docs no longer say an unlisted target is refused first with `invalid: skill not listed`: that status comes from the catalog check, and the guard reports an unlisted target as `invalid: skill not visible`. Signed-off-by: Nikolay Petrov --- ...30_120000_napetrov_known-positive-guard.md | 3 +- dokimi/docs/evaluating-treatments.md | 3 +- dokimi/dokimi/harnesses/trigger_probe.py | 18 +++++----- dokimi/tests/test_trigger_probe.py | 36 ++++++++++++++----- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md index 5925b26f1..86581e11e 100644 --- a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -10,4 +10,5 @@ not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run that ended in an error writes `"status": "error: known-positive run failed"` instead. An empty `--catalog` is refused (exit 2) before the guard runs. Pass - `--no-known-positive` to skip the check; the output then says `"status": "unchecked"`. + `--no-known-positive` to skip the check; the output then says `"status": "unchecked"` (or `invalid: skill not listed` when a + catalog skill is never listed). diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 1f8887fb5..fb106fb1c 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -605,7 +605,8 @@ exits 1; a leftover `.partial` is removed. A skill call that returned an er `plugin:skill` of another plugin, does not count as a load. If that run ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. This known-positive check is what separates "the model did not pick the skill" from "the -harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked`. +harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked` (`invalid: skill not listed` still applies +when a catalog skill is never listed). Records are kept in `.partial` while the probe runs: ```bash diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 11726ce3b..7fa2cd8fd 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -145,13 +145,13 @@ def parse_stream(stream: str) -> Dict[str, Any]: # share message.id. Subagent events (parent_tool_use_id set) are not the choice. by_turn: Dict[str, List[str]] = {} other_tools: List[str] = [] - failed_ids: set = set() # tool_use ids whose tool_result is an error + ok_ids: set = set() # tool_use ids whose tool_result arrived and is not an error calls: List[tuple] = [] # (tool_use id, raw skill name) of top-level Skill calls for n, ev in enumerate(_events(stream)): if ev.get("type") == "user": for c in (ev.get("message") or {}).get("content") or []: - if isinstance(c, dict) and c.get("type") == "tool_result" and c.get("is_error"): - failed_ids.add(c.get("tool_use_id")) + if isinstance(c, dict) and c.get("type") == "tool_result" and not c.get("is_error"): + ok_ids.add(c.get("tool_use_id")) if ev.get("type") == "system" and ev.get("subtype") == "init": listed = [str(s) for s in ev.get("skills") or []] version = ev.get("claude_code_version") @@ -175,8 +175,9 @@ def parse_stream(stream: str) -> Dict[str, Any]: "selected": first[0] if first else None, "first_turn_loaded": first, "loaded": [s for t in turns for s in t], - # Raw names (plugin namespace kept) of Skill calls whose result was not an error. - "loaded_ok": [name for tid, name in calls if tid is None or tid not in failed_ids], + # Raw names (plugin namespace kept) of Skill calls with an observed result that was not an error. + # A call whose result never arrived (a hang, a kill) did not load anything. + "loaded_ok": [name for tid, name in calls if tid is not None and tid in ok_ids], "tools_before_choice": other_tools, "claude_code_version": version, } @@ -229,9 +230,10 @@ def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: proce except (OSError, RuntimeError) as exc: return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None, "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"} - # The guard passes on a successful call of exactly this skill (a `plugin:skill` is another one); - # ``listed`` records whether the harness showed it. A failure is reported even after a Skill call. - passed = skill in r["skill"]["loaded_ok"] + # The guard passes on an observed, successful call of exactly this skill (a `plugin:skill` is another + # one) in a run that then ended without a terminal error; ``listed`` records whether the harness showed + # it. A terminal failure is reported even after the skill loaded. + passed = skill in r["skill"]["loaded_ok"] and r["terminal_error"] is None return {"skill": skill, "query": query, "passed": passed, "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"], "claude_code_version": r["skill"]["claude_code_version"], diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index b21f5a0c1..47732e4d5 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -35,10 +35,13 @@ def _stream(*skills_per_turn, other: str = "", version: str = "2.1.304") -> str: if other: lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "name": other, "input": {}}]}}) - for turn in skills_per_turn: - lines.append({"type": "assistant", "message": {"content": [ - {"type": "text", "text": "ok"}] + [ - {"type": "tool_use", "name": "Skill", "input": {"skill": s}} for s in turn]}}) + for t, turn in enumerate(skills_per_turn): + uses = [{"type": "tool_use", "id": f"t{t}-{i}", "name": "Skill", "input": {"skill": s}} + for i, s in enumerate(turn)] + lines.append({"type": "assistant", "message": {"content": [{"type": "text", "text": "ok"}] + uses}}) + if uses: # each call got its result back, not an error + lines.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": u["id"], "is_error": False, "content": "x"} for u in uses]}}) lines.append({"type": "result", "subtype": "success", "is_error": False, "total_cost_usd": 0.01, "num_turns": 2}) return "\n".join(json.dumps(x) for x in lines) + "\n" @@ -459,13 +462,16 @@ def runner(argv, **kw): assert r["passed"] is passed and r["error"] is None -def _skill_stream(name, *, is_error=False, result=True): +def _skill_stream(name, *, is_error=False, result=True, answered=True, final=None): events = [{"type": "system", "subtype": "init", "skills": ["sum-fast"], "claude_code_version": "2.1.304"}, {"type": "assistant", "message": {"id": "m1", "content": [ - {"type": "tool_use", "id": "tu1", "name": "Skill", "input": {"skill": name}}]}}, - {"type": "user", "message": {"content": [ - {"type": "tool_result", "tool_use_id": "tu1", "is_error": is_error, "content": "x"}]}}] - if result: + {"type": "tool_use", "id": "tu1", "name": "Skill", "input": {"skill": name}}]}}] + if answered: + events.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "tu1", "is_error": is_error, "content": "x"}]}}) + if final is not None: + events.append(final) + elif result: events.append({"type": "result", "subtype": "success", "is_error": False, "total_cost_usd": 0.01, "num_turns": 2}) return "\n".join(json.dumps(e) for e in events) + "\n" @@ -493,6 +499,18 @@ def test_known_positive_keeps_the_timeout_when_another_skill_was_called_first(tm assert r["passed"] is False and r["error"] == "timeout" +def test_known_positive_fails_when_the_target_call_timed_out_before_its_result(tmp_path): + # The target was called, but its result never arrived: nothing shows it loaded. + r = _known(tmp_path, _skill_stream("sum-fast", answered=False, result=False), timed_out=True) + assert r["passed"] is False and r["error"] == "timeout" and r["loaded"] == ["sum-fast"] + + +def test_known_positive_keeps_a_terminal_error_after_the_target_loaded(tmp_path): + final = {"type": "result", "subtype": "error_during_execution", "is_error": True, "num_turns": 2} + r = _known(tmp_path, _skill_stream("sum-fast", final=final)) + assert r["passed"] is False and r["error"] == "error_during_execution" + + def test_known_positive_that_cannot_run_fails_without_raising(tmp_path): def runner(argv, **kw): if argv[0] == "git": From 5a0e2b0591dcc47fa9cc3504330e0d9adfa8cf05 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 11:17:51 -0800 Subject: [PATCH 08/11] docs(probe): qualify the empty-catalog refusal; two-line guard comment Signed-off-by: Nikolay Petrov --- .../20261030_120000_napetrov_known-positive-guard.md | 2 +- dokimi/dokimi/harnesses/trigger_probe.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md index 86581e11e..23b876dd9 100644 --- a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -9,6 +9,6 @@ exits 1. A skill call whose result is an error, or another plugin's `plugin:skill`, does not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run that ended in an error writes `"status": "error: known-positive run failed"` instead. An - empty `--catalog` is refused (exit 2) before the guard runs. Pass + empty `--catalog` is refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. Pass `--no-known-positive` to skip the check; the output then says `"status": "unchecked"` (or `invalid: skill not listed` when a catalog skill is never listed). diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 7fa2cd8fd..445112def 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -230,9 +230,8 @@ def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: proce except (OSError, RuntimeError) as exc: return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None, "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"} - # The guard passes on an observed, successful call of exactly this skill (a `plugin:skill` is another - # one) in a run that then ended without a terminal error; ``listed`` records whether the harness showed - # it. A terminal failure is reported even after the skill loaded. + # The guard passes on an observed, successful call of exactly this skill (not another plugin's) with no + # terminal error; ``listed`` records whether the harness showed it. passed = skill in r["skill"]["loaded_ok"] and r["terminal_error"] is None return {"skill": skill, "query": query, "passed": passed, "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"], From 75bfb9c905df82efda27632ca19c460f902b273c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:13:02 -0800 Subject: [PATCH 09/11] test(probe): the guard refusal tests fail if a labelled run came first The fake guard records a labelled query recorded before it, and the test helper checks that record after the CLI returns or exits, so no handler in the CLI can swallow it. A refusal then costs no labelled runs, as the tests intend. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_trigger_probe.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 47732e4d5..9ddd7398f 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -323,6 +323,9 @@ def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), exp seen = {} def fake_known(skill, catalog, cfg, **kw): + # the guard runs before any labelled query: a refusal must not cost the labelled runs + if seen.get("queries"): + seen.setdefault("order_violation", []).append(skill) seen.setdefault("known", []).append(skill) return {"skill": skill, "query": f"Use the {skill} skill.", "passed": known_passed, "loaded": [skill] if known_passed else [], "listed": list(init_skills), "cost_usd": 0.01, @@ -354,7 +357,12 @@ def __init__(self, n): _catalog(tmp_path) args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", "ts.json", "--claude", "sh", *extra]) - args.func(args) + try: + args.func(args) + finally: + # checked here, after the CLI returns or exits, so no handler in the CLI can swallow it + assert not seen.get("order_violation"), ( + f"labelled queries ran before the known-positive guard: {seen.get('order_violation')}") return seen From c8769eafb37fd7457144db70bbc2ba8c0e6316c0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 13:14:16 -0800 Subject: [PATCH 10/11] test(probe): pin the guard's cost on the success path The guard's run is part of the reported cost; a success run that drops it now fails the test (mutation: cost += 0 fails it). Signed-off-by: Nikolay Petrov --- dokimi/tests/test_trigger_probe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 9ddd7398f..afa7a02d0 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -535,6 +535,7 @@ def test_cli_probes_the_target_or_the_first_skill_before_the_set(tmp_path, monke assert _cli(tmp_path, monkeypatch, "--target", "sum-fast", "--out", "probe.json")["known"] == ["sum-fast"] out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) assert out["status"] == "ok" and out["known_positive"]["passed"] is True + assert out["cost_usd"] == 0.04 # 3 labelled runs at 0.01 each, plus the guard's 0.01 def test_cli_stops_before_the_labelled_runs_when_the_skill_is_not_visible(tmp_path, monkeypatch, capsys): From 6248d032021869d17dab78fe8b84707a758f453c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 21:32:17 +0000 Subject: [PATCH 11/11] Say known_positive holds the guard run in the changelog entry Signed-off-by: Anton Volkov --- .../20261030_120000_napetrov_known-positive-guard.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md index 23b876dd9..f741339b0 100644 --- a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -4,8 +4,8 @@ explicit run (`Use the skill.`, for `--target` or the alphabetically first skill name of the catalog) must load that skill. If it does not, the probe writes `"status": "invalid: skill not loaded"` (Claude Code listed it) or `"invalid: skill not visible"` (it did not), with - `known_positive`, the probe's `max_turns_hit`/`not_listed`/`skill_selection_summary` keys - empty, no answers, no stale `.partial`, prints the same status on stdout, and + `known_positive` holding the guard run, the probe's `max_turns_hit`/`not_listed`/`skill_selection_summary` keys + at 0, `[]` and `{}`, no answers, no stale `.partial`, prints the same status on stdout, and exits 1. A skill call whose result is an error, or another plugin's `plugin:skill`, does not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run that ended in an error writes `"status": "error: known-positive run failed"` instead. An