diff --git a/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md new file mode 100644 index 000000000..f741339b0 --- /dev/null +++ b/dokimi/changelog.d/20261030_120000_napetrov_known-positive-guard.md @@ -0,0 +1,14 @@ +### Added + +- **`catalog trigger-probe` known-positive guard** — before the labelled queries, one + explicit run (`Use the skill.`, for `--target` or the alphabetically first skill name of the catalog) + must load that skill. If it does not, the probe writes `"status": "invalid: skill not + loaded"` (Claude Code listed it) or `"invalid: skill not visible"` (it did not), with + `known_positive` holding the guard run, the probe's `max_turns_hit`/`not_listed`/`skill_selection_summary` keys + at 0, `[]` and `{}`, no answers, no stale `.partial`, prints the same status on stdout, and + exits 1. A skill call whose result is an error, or another plugin's `plugin:skill`, does + not count as a load. So a skill the harness never shows cannot read as a 0% trigger rate. A guard run + that ended in an error writes `"status": "error: known-positive run failed"` instead. An + empty `--catalog` is refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. Pass + `--no-known-positive` to skip the check; the output then says `"status": "unchecked"` (or `invalid: skill not listed` when a + catalog skill is never listed). diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index 7219d8f44..fb106fb1c 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -594,10 +594,20 @@ left on as a user has them (`--sealed` turns them off). The skill chosen is read transcript's `Skill` calls, and the output's `skill_selection_summary` is scored by the same code as the `skill-catalog:` arm, one column per trial. `max_turns_hit` counts runs that ended on `--max-turns` (scored as no load), and the command exits 1 when Claude Code -did not list a catalog skill (`not_listed`). When no run listed any skill, `not_listed` +did not list a catalog skill (`not_listed`; `status` is then `invalid: skill not listed`, and the labelled results are still written). When no run listed any skill, `not_listed` stays empty and a warning on stderr says it was not checked. A trigger set with no -labelled query exits 2 before any run. Records are kept in `.partial` while the -probe runs: +labelled query exits 2 before any run. Before the labelled queries, one explicit run +(`Use the skill.`, for the `--target` skill or the alphabetically first skill name of the +catalog) must load that skill; if it does not, no labelled query runs, `status` is +`invalid: skill not loaded` (Claude Code listed the skill but the request did not load it) or +`invalid: skill not visible` (it did not list it) in the output and on stdout, and the command +exits 1; a leftover `.partial` is removed. A skill call that returned an error, or a +`plugin:skill` of another plugin, does not count as a load. If that run ended in an error, `status` is `error: known-positive run failed` instead. An empty catalog is +refused (exit 2) before the guard runs, unless `--no-known-positive` is passed. This +known-positive check is what separates "the model did not pick the skill" from "the +harness never showed it"; `--no-known-positive` skips it, and `status` is then `unchecked` (`invalid: skill not listed` still applies +when a catalog skill is never listed). +Records are kept in `.partial` while the probe runs: ```bash python cli.py catalog trigger-probe --catalog /skills \ diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..a0eed3b28 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -2803,6 +2803,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--no-known-positive", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, diff --git a/dokimi/dokimi/commands/catalog_probe.py b/dokimi/dokimi/commands/catalog_probe.py index 4cfb97a28..db6119027 100644 --- a/dokimi/dokimi/commands/catalog_probe.py +++ b/dokimi/dokimi/commands/catalog_probe.py @@ -16,6 +16,13 @@ from dokimi.exit_codes import usage_exit +#: Status of a run whose known-positive guard failed: no labelled query ran. +_INVALID = "invalid: skill not visible" +_NOT_LOADED = "invalid: skill not loaded" # the harness listed the skill, the explicit request did not load it +_ERROR = "error: known-positive run failed" # the guard run ended in an error, not a missing skill +_NOT_LISTED = "invalid: skill not listed" + + def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: from dokimi.eval.arm_summary import summarize_skill_selection from dokimi.harnesses import trigger_probe as tp @@ -47,6 +54,7 @@ def cmd_catalog_trigger_probe(args: argparse.Namespace) -> None: cfg = tp.ProbeConfig(model=args.model, max_turns=args.max_turns, timeout_sec=args.timeout, stock=not args.sealed, listing_budget_fraction=args.listing_budget_fraction, claude=os.path.abspath(claude)) + known = _known_positive_guard(args, names, catalog, cfg, out_path) done: list = [] lock = threading.Lock() @@ -73,18 +81,12 @@ def one(q): # a nested layout) reads as a 0% recall; say so instead of scoring it. not_listed = sorted(names - listed) if listed else [] cost = sum(v["metrics"]["cost_usd"] or 0 for r in records for v in r["arms"].values()) + cost += (known or {}).get("cost_usd") or 0 + version = next((v["tool_usage"]["skill"]["claude_code_version"] for r in records + for v in r["arms"].values() if v["tool_usage"]["skill"]["claude_code_version"]), None) + status = _NOT_LISTED if not_listed else ("ok" if known else "unchecked") out = { - "harness": "claude-code", - "mode": "sealed" if args.sealed else "stock", - "model": args.model, - "catalog": str(catalog), - "trials": args.trials, - "max_turns": args.max_turns, - "listing_budget_fraction": args.listing_budget_fraction, - "claude_code_version": next((v["tool_usage"]["skill"]["claude_code_version"] for r in records - for v in r["arms"].values() - if v["tool_usage"]["skill"]["claude_code_version"]), None), - "cost_usd": round(cost, 4), + **_header(args, catalog, known, status, cost, version), "max_turns_hit": sum(v["metrics"].get("result_subtype") == "error_max_turns" for v in runs), "not_listed": not_listed, "answers": records, @@ -92,7 +94,8 @@ def one(q): } _write(out_path, out) partial.unlink(missing_ok=True) - print(json.dumps({"cost_usd": out["cost_usd"], "claude_code_version": out["claude_code_version"], + print(json.dumps({"status": out["status"], "cost_usd": out["cost_usd"], + "claude_code_version": out["claude_code_version"], "max_turns_hit": out["max_turns_hit"], "not_listed": not_listed, "summary": {a: {k: s[k] for k in ("recall", "false_trigger_rate", "misroute_rate", "n_failed")} for a, s in out["skill_selection_summary"].items()}}, indent=2)) @@ -101,6 +104,53 @@ def one(q): raise SystemExit(f"catalog skills Claude Code did not list: {', '.join(not_listed)}") +def _header(args: argparse.Namespace, catalog: Path, known, status: str, cost: float, version) -> dict: + """The fields every output carries, whether the run was probed or refused by the guard.""" + return { + "harness": "claude-code", + "status": status, + "known_positive": known, + "mode": "sealed" if args.sealed else "stock", + "model": args.model, + "catalog": str(catalog), + "trials": args.trials, + "max_turns": args.max_turns, + "listing_budget_fraction": args.listing_budget_fraction, + "claude_code_version": version, + "cost_usd": round(cost, 4), + } + + +def _known_positive_guard(args: argparse.Namespace, names: set, catalog: Path, cfg, out_path: Path): + """The known-positive record, or None when skipped. A failed guard writes the invalid output and exits 1. + + A skill the harness never loads on an explicit request would score every query 0%, which reads as a + finding; this refuses the run before any labelled query, which costs one query. + """ + if args.no_known_positive: + return None + from dokimi.harnesses import trigger_probe as tp + from dokimi.utils import InputError + + if not names: # the guard's default target is the first catalog skill: with none, nothing to probe + raise InputError(f"--catalog holds no skills: {catalog}") + known = tp.known_positive(args.target or sorted(names)[0], catalog, cfg) + if known["passed"]: + return known + # A run that errored says so: "not visible" would read as the harness hiding the skill. + if known["error"]: + status = _ERROR + else: + status = _NOT_LOADED if known["skill"] in known["listed"] else _INVALID + _write(out_path, {**_header(args, catalog, known, status, known.get("cost_usd") or 0, + known.get("claude_code_version")), + "max_turns_hit": 0, "not_listed": [], "answers": [], "skill_selection_summary": {}}) + out_path.with_name(out_path.name + ".partial").unlink(missing_ok=True) + print(json.dumps({"status": status, "known_positive": known}, indent=2)) + raise SystemExit(f"{status}: {known['query']!r} did not load {known['skill']!r}" + + (f" ({known['error']})" if known["error"] else "") + "; no labelled query was run") + + def _labelled_queries(args: argparse.Namespace, names: set) -> list: """The labelled queries to run; a label naming no *names* skill is an input error.""" from dokimi.external import trigger_set as ts @@ -152,5 +202,9 @@ def register(catalog_sub, positive_int) -> None: p.add_argument("--sealed", action="store_true", help="Switch off bundled skills as the docker-claude arms do (default: stock harness)") p.add_argument("--claude", default="claude", help="claude executable (default: claude)") + p.add_argument("--no-known-positive", action="store_true", dest="no_known_positive", + help="Skip the known-positive guard: one explicit run that must load a catalog skill " + "before the labelled queries; a failed guard exits 1 with an 'invalid: ...' " + "or 'error: ...' status") p.add_argument("--out", required=True, help="Output JSON path") p.set_defaults(func=cmd_catalog_trigger_probe) diff --git a/dokimi/dokimi/harnesses/trigger_probe.py b/dokimi/dokimi/harnesses/trigger_probe.py index 328078be1..445112def 100644 --- a/dokimi/dokimi/harnesses/trigger_probe.py +++ b/dokimi/dokimi/harnesses/trigger_probe.py @@ -145,7 +145,13 @@ def parse_stream(stream: str) -> Dict[str, Any]: # share message.id. Subagent events (parent_tool_use_id set) are not the choice. by_turn: Dict[str, List[str]] = {} other_tools: List[str] = [] + ok_ids: set = set() # tool_use ids whose tool_result arrived and is not an error + calls: List[tuple] = [] # (tool_use id, raw skill name) of top-level Skill calls for n, ev in enumerate(_events(stream)): + if ev.get("type") == "user": + for c in (ev.get("message") or {}).get("content") or []: + if isinstance(c, dict) and c.get("type") == "tool_result" and not c.get("is_error"): + ok_ids.add(c.get("tool_use_id")) if ev.get("type") == "system" and ev.get("subtype") == "init": listed = [str(s) for s in ev.get("skills") or []] version = ev.get("claude_code_version") @@ -154,6 +160,8 @@ def parse_stream(stream: str) -> Dict[str, Any]: message = ev.get("message") or {} uses = [c for c in message.get("content") or [] if isinstance(c, dict) and c.get("type") == "tool_use"] skills = [_skill_name((c.get("input") or {}).get("skill")) for c in uses if c.get("name") == "Skill"] + calls += [(c.get("id"), str((c.get("input") or {}).get("skill") or "")) + for c in uses if c.get("name") == "Skill"] if not by_turn: # tools after the choice are not "before" it other_tools += [str(c.get("name")) for c in uses if c.get("name") != "Skill"] if skills: @@ -167,6 +175,9 @@ def parse_stream(stream: str) -> Dict[str, Any]: "selected": first[0] if first else None, "first_turn_loaded": first, "loaded": [s for t in turns for s in t], + # Raw names (plugin namespace kept) of Skill calls with an observed result that was not an error. + # A call whose result never arrived (a hang, a kill) did not load anything. + "loaded_ok": [name for tid, name in calls if tid is not None and tid in ok_ids], "tools_before_choice": other_tools, "claude_code_version": version, } @@ -186,22 +197,48 @@ def run_query(query: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Ru skill = parse_stream(stream) result = result_event(stream) or {} error = None + # The failure as it is, before the Skill-call suppression below (the guard reports it). + terminal = None + if not result: + terminal = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" + elif result.get("is_error") and result.get("subtype") != "error_max_turns": + terminal = str(result.get("subtype") or "error") # A Skill call seen is the choice, whatever came after it (a hang, a timeout, an # error). A result event read decides the rest, even if the CLI then hung. if skill["first_turn_loaded"]: pass - elif not result: - error = "timeout" if done.timed_out else f"no result event (exit {done.returncode})" - # Out of turns without a Skill call is an observation (nothing triggered), not a failure. - elif result.get("is_error") and result.get("subtype") != "error_max_turns": - error = str(result.get("subtype") or "error") + else: # out of turns without a Skill call is an observation (nothing triggered), not a failure + error = terminal return {"skill": skill, "cost_usd": result.get("total_cost_usd"), - "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error} + "num_turns": result.get("num_turns"), "result_subtype": result.get("subtype"), "error": error, + "terminal_error": terminal} finally: kill_strays(ws.root) ws.cleanup() +def known_positive(skill: str, catalog: Path, cfg: ProbeConfig, *, runner: process.Runner = process.run, + source_env: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + """One run of an explicit request for *skill*, through the same harness; ``passed`` if it loaded it. + + A harness that never shows the skill scores every labelled query 0%, which reads as a + finding. This run is the check that the skill can be loaded at all, before those queries. + """ + query = f"Use the {skill} skill." + try: + r = run_query(query, catalog, cfg, runner=runner, source_env=source_env) + except (OSError, RuntimeError) as exc: + return {"skill": skill, "query": query, "passed": False, "loaded": [], "listed": [], "cost_usd": None, + "claude_code_version": None, "error": f"{type(exc).__name__}: {exc}"} + # The guard passes on an observed, successful call of exactly this skill (not another plugin's) with no + # terminal error; ``listed`` records whether the harness showed it. + passed = skill in r["skill"]["loaded_ok"] and r["terminal_error"] is None + return {"skill": skill, "query": query, "passed": passed, + "loaded": r["skill"]["loaded"], "listed": r["skill"]["listed"], "cost_usd": r["cost_usd"], + "claude_code_version": r["skill"]["claude_code_version"], + "error": None if passed else r["terminal_error"]} + + def kill_strays(root: Path) -> List[int]: """SIGKILL every process whose cwd is under *root*; return their pids (Linux only). diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index f8e037fda..55edc434c 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -119,6 +119,7 @@ --no-confirm-carriers --no-json --no-judge +--no-known-positive # catalog trigger-probe known-positive guard: no registry concept yet --no-paraphrase --no-plugin-deltas --no-recursive diff --git a/dokimi/tests/test_trigger_probe.py b/dokimi/tests/test_trigger_probe.py index 81bad7980..afa7a02d0 100644 --- a/dokimi/tests/test_trigger_probe.py +++ b/dokimi/tests/test_trigger_probe.py @@ -35,10 +35,13 @@ def _stream(*skills_per_turn, other: str = "", version: str = "2.1.304") -> str: if other: lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "name": other, "input": {}}]}}) - for turn in skills_per_turn: - lines.append({"type": "assistant", "message": {"content": [ - {"type": "text", "text": "ok"}] + [ - {"type": "tool_use", "name": "Skill", "input": {"skill": s}} for s in turn]}}) + for t, turn in enumerate(skills_per_turn): + uses = [{"type": "tool_use", "id": f"t{t}-{i}", "name": "Skill", "input": {"skill": s}} + for i, s in enumerate(turn)] + lines.append({"type": "assistant", "message": {"content": [{"type": "text", "text": "ok"}] + uses}}) + if uses: # each call got its result back, not an error + lines.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": u["id"], "is_error": False, "content": "x"} for u in uses]}}) lines.append({"type": "result", "subtype": "success", "is_error": False, "total_cost_usd": 0.01, "num_turns": 2}) return "\n".join(json.dumps(x) for x in lines) + "\n" @@ -219,9 +222,11 @@ def test_cli_runs_a_relative_claude_from_the_scratch_cwd(tmp_path, monkeypatch, _catalog(tmp_path) monkeypatch.chdir(tmp_path) args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", - "ts.json", "--claude", "bin/claude", "--out", "probe.json"]) + "ts.json", "--claude", "bin/claude", "--target", "sum-fast", + "--out", "probe.json"]) args.func(args) out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "ok" and out["known_positive"]["passed"] is True arm = out["answers"][0]["arms"]["claude_code#1"] assert arm["error"] is None and arm["tool_usage"]["skill"]["first_turn_loaded"] == ["sum-fast"] @@ -311,11 +316,21 @@ def runner(argv, **kw): shutil.rmtree(procs["sibling_dir"], ignore_errors=True) -def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast"): +def _cli(tmp_path, monkeypatch, *extra, init_skills=("sum-fast", "io-read"), expected="sum-fast", known_passed=True, + known_error=None): from dokimi.commands import catalog_probe seen = {} + def fake_known(skill, catalog, cfg, **kw): + # the guard runs before any labelled query: a refusal must not cost the labelled runs + if seen.get("queries"): + seen.setdefault("order_violation", []).append(skill) + seen.setdefault("known", []).append(skill) + return {"skill": skill, "query": f"Use the {skill} skill.", "passed": known_passed, + "loaded": [skill] if known_passed else [], "listed": list(init_skills), "cost_usd": 0.01, + "claude_code_version": "2.1.3xx", "error": known_error} + def fake_probe(queries, catalog, cfg, *, trials=1, **kw): seen.update(cfg=cfg, trials=trials, catalog=catalog) seen.setdefault("queries", []).extend(q.id for q in queries) @@ -332,6 +347,7 @@ def __init__(self, n): super().__init__(n) monkeypatch.setattr(tp, "probe_trigger_set", fake_probe) + monkeypatch.setattr(tp, "known_positive", fake_known) monkeypatch.setattr(catalog_probe, "ThreadPoolExecutor", Pool) (tmp_path / "ts.json").write_text(json.dumps({"questions": [ {"id": q, "question": f"q {q}", "expected_skill": expected} for q in ("a", "b", "c")]}), @@ -341,7 +357,12 @@ def __init__(self, n): _catalog(tmp_path) args = build_parser().parse_args(["catalog", "trigger-probe", "--catalog", "cat", "--trigger-set", "ts.json", "--claude", "sh", *extra]) - args.func(args) + try: + args.func(args) + finally: + # checked here, after the CLI returns or exits, so no handler in the CLI can swallow it + assert not seen.get("order_violation"), ( + f"labelled queries ran before the known-positive guard: {seen.get('order_violation')}") return seen @@ -415,4 +436,166 @@ def test_cli_warns_when_no_run_listed_any_skill(tmp_path, monkeypatch, capsys): def test_cli_fails_when_claude_code_did_not_list_a_catalog_skill(tmp_path, monkeypatch): with pytest.raises(SystemExit, match="did not list: io-read"): _cli(tmp_path, monkeypatch, "--out", "probe.json", init_skills=("sum-fast", "dataviz")) - assert json.loads((tmp_path / "probe.json").read_text(encoding="utf-8"))["not_listed"] == ["io-read"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["not_listed"] == ["io-read"] and out["status"] == "invalid: skill not listed" + + +def test_cli_reports_a_guard_run_that_errored_apart_from_a_skill_not_shown(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="error: known-positive run failed"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, known_error="error_during_execution") + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "error: known-positive run failed" and out["answers"] == [] + assert '"status": "error: known-positive run failed"' in capsys.readouterr().out + + +def test_the_invalid_output_has_the_header_fields_and_the_guard_cost(tmp_path, monkeypatch): + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["cost_usd"] == 0.01 and out["claude_code_version"] == "2.1.3xx" + assert out["trials"] == 1 and out["max_turns"] and "listing_budget_fraction" in out + + +@pytest.mark.parametrize("stream, passed", [(_stream(["sum-fast"]), True), (_stream(), False)], + ids=["loads-the-skill", "never-loads-it"]) +def test_known_positive_passes_only_when_the_explicit_request_loads_the_skill(tmp_path, stream, passed): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + assert argv[list(argv).index("-p") + 1] == "Use the sum-fast skill." + return process.reply(stream) + + r = tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["passed"] is passed and r["error"] is None + + +def _skill_stream(name, *, is_error=False, result=True, answered=True, final=None): + events = [{"type": "system", "subtype": "init", "skills": ["sum-fast"], "claude_code_version": "2.1.304"}, + {"type": "assistant", "message": {"id": "m1", "content": [ + {"type": "tool_use", "id": "tu1", "name": "Skill", "input": {"skill": name}}]}}] + if answered: + events.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "tu1", "is_error": is_error, "content": "x"}]}}) + if final is not None: + events.append(final) + elif result: + events.append({"type": "result", "subtype": "success", "is_error": False, "total_cost_usd": 0.01, "num_turns": 2}) + return "\n".join(json.dumps(e) for e in events) + "\n" + + +def _known(tmp_path, stream, timed_out=False): + def runner(argv, **kw): + return process.run(argv, **kw) if argv[0] == "git" else process.reply(stream, timed_out=timed_out) + + return tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + + +@pytest.mark.parametrize("name, is_error, passed", [ + ("sum-fast", False, True), ("sum-fast", True, False), ("plug:sum-fast", False, False)], + ids=["loaded", "result-is-error", "other-namespace"]) +def test_known_positive_needs_the_exact_skill_loaded_without_error(tmp_path, name, is_error, passed): + r = _known(tmp_path, _skill_stream(name, is_error=is_error)) + assert r["passed"] is passed + # Labelled scoring still sees the namespace-stripped name. + assert r["loaded"] == ["sum-fast"] + + +def test_known_positive_keeps_the_timeout_when_another_skill_was_called_first(tmp_path): + r = _known(tmp_path, _skill_stream("io-read", result=False), timed_out=True) + assert r["passed"] is False and r["error"] == "timeout" + + +def test_known_positive_fails_when_the_target_call_timed_out_before_its_result(tmp_path): + # The target was called, but its result never arrived: nothing shows it loaded. + r = _known(tmp_path, _skill_stream("sum-fast", answered=False, result=False), timed_out=True) + assert r["passed"] is False and r["error"] == "timeout" and r["loaded"] == ["sum-fast"] + + +def test_known_positive_keeps_a_terminal_error_after_the_target_loaded(tmp_path): + final = {"type": "result", "subtype": "error_during_execution", "is_error": True, "num_turns": 2} + r = _known(tmp_path, _skill_stream("sum-fast", final=final)) + assert r["passed"] is False and r["error"] == "error_during_execution" + + +def test_known_positive_that_cannot_run_fails_without_raising(tmp_path): + def runner(argv, **kw): + if argv[0] == "git": + return process.run(argv, **kw) + raise FileNotFoundError(2, "No such file or directory", argv[0]) + + r = tp.known_positive("sum-fast", _catalog(tmp_path), tp.ProbeConfig(), runner=runner, + source_env={"PATH": "/usr/bin:/bin"}) + assert r["passed"] is False and r["error"].startswith("FileNotFoundError") + + +def test_cli_probes_the_target_or_the_first_skill_before_the_set(tmp_path, monkeypatch): + assert _cli(tmp_path, monkeypatch, "--out", "probe.json")["known"] == ["io-read"] + assert _cli(tmp_path, monkeypatch, "--target", "sum-fast", "--out", "probe.json")["known"] == ["sum-fast"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "ok" and out["known_positive"]["passed"] is True + assert out["cost_usd"] == 0.04 # 3 labelled runs at 0.01 each, plus the guard's 0.01 + + +def test_cli_stops_before_the_labelled_runs_when_the_skill_is_not_visible(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="invalid: skill not visible"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, init_skills=("sum-fast",)) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "invalid: skill not visible" and out["answers"] == [] + assert out["known_positive"]["passed"] is False + assert not (tmp_path / "probe.json.partial").exists() + assert '"status": "invalid: skill not visible"' in capsys.readouterr().out + + +def test_cli_says_not_loaded_when_the_harness_listed_the_skill(tmp_path, monkeypatch, capsys): + with pytest.raises(SystemExit, match="invalid: skill not loaded"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "invalid: skill not loaded" + assert '"status": "invalid: skill not loaded"' in capsys.readouterr().out + + +def test_the_refusal_output_has_the_probe_keys_of_an_unrun_probe(tmp_path, monkeypatch): + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False, init_skills=("sum-fast",)) + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["max_turns_hit"] == 0 and out["not_listed"] == [] and out["skill_selection_summary"] == {} + + +def test_a_refusal_removes_the_stale_partial_of_an_earlier_run(tmp_path, monkeypatch): + (tmp_path / "probe.json.partial").write_text("{}", encoding="utf-8") + with pytest.raises(SystemExit): + _cli(tmp_path, monkeypatch, "--out", "probe.json", known_passed=False) + assert not (tmp_path / "probe.json.partial").exists() + + +def test_no_known_positive_skips_the_guard(tmp_path, monkeypatch): + seen = _cli(tmp_path, monkeypatch, "--no-known-positive", "--out", "probe.json", known_passed=False) + assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] + out = json.loads((tmp_path / "probe.json").read_text(encoding="utf-8")) + assert out["status"] == "unchecked" and out["known_positive"] is None + + +def test_cli_refuses_an_empty_catalog_before_the_guard_picks_a_skill(tmp_path, monkeypatch): + # A negative-only set has no label naming a skill, so only the guard's default target could + # name one; with no skill in the catalog that was an IndexError, not an input error. + from dokimi.utils import InputError + + (tmp_path / "cat").mkdir() + with pytest.raises(InputError, match="--catalog holds no skills"): + _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=None) + assert not (tmp_path / "probe.json.partial").exists() and not (tmp_path / "probe.json").exists() + + +def test_no_known_positive_still_runs_a_negative_only_set_on_an_empty_catalog(tmp_path, monkeypatch): + # The guard is what needs a skill; with it skipped the run is what it was before the refusal. + (tmp_path / "cat").mkdir() + seen = _cli(tmp_path, monkeypatch, "--no-known-positive", "--out", "probe.json", expected=None) + assert "known" not in seen and sorted(seen["queries"]) == ["a", "b", "c"] + + +def test_negative_only_set_on_a_catalog_with_skills_still_runs_the_guard(tmp_path, monkeypatch): + # names come from the catalog's skills, not the labels: a negative-only set on a non-empty + # catalog is not refused, and the guard probes the first catalog skill as before. + assert _cli(tmp_path, monkeypatch, "--out", "probe.json", expected=None)["known"] == ["io-read"]