From a134e4f1faf1470fe52d2d6dee709856ac3a516a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 10:12:12 -0700 Subject: [PATCH 01/14] feat(routing): estimate-only, all-vs-all repeats and cost, description queries - routing --estimate-only prints the calls, tokens and list-price USD that --model would make, as JSON, and makes no model call. - all-vs-all accepts --repeats with the same noise block as routing, and prints a cost block; the JSON gets the cost too. - all-vs-all refuses queries for a skill that only sets disable-model-invocation, names it, and suggests --include-disabled. - --queries-from-headings falls back to the description's sentences when the headings give no query; queries_source records which one was used. - --seed help says it fixes the card order only and points at --repeats for answer noise. - Routing console lines and query loading move to commands/novelty_routing_report.py to keep novelty.py under its ratchet. Signed-off-by: Nikolay Petrov --- ...261031_120000_napetrov_routing-cli-gaps.md | 16 ++ dokimi/docs/reference/cli-interface.json | 16 ++ dokimi/docs/skill-novelty.md | 25 ++- dokimi/dokimi/commands/novelty.py | 144 ++++++++---------- .../dokimi/commands/novelty_routing_report.py | 141 +++++++++++++++++ dokimi/dokimi/novelty/all_vs_all.py | 27 ++-- dokimi/dokimi/novelty/routing.py | 55 +++++++ dokimi/scripts/growth_baseline.json | 4 +- dokimi/tests/test_novelty_routing.py | 3 +- .../tests/test_novelty_routing_all_vs_all.py | 79 +++++++++- dokimi/tests/test_novelty_routing_cli_gaps.py | 109 +++++++++++++ 11 files changed, 513 insertions(+), 106 deletions(-) create mode 100644 dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md create mode 100644 dokimi/dokimi/commands/novelty_routing_report.py create mode 100644 dokimi/tests/test_novelty_routing_cli_gaps.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md new file mode 100644 index 000000000..d669f3469 --- /dev/null +++ b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md @@ -0,0 +1,16 @@ + + +### Added + +- **`novelty routing --estimate-only`** prints, as JSON, the calls, input and output tokens and the list-price `usd` that `--model` would make, from the same selection prompt the run sends, and makes no model call. Without `--model` the bill is zero. `--all-vs-all` has no estimate yet; its stderr line still says how many calls the model arm makes. +- **`novelty routing --all-vs-all --repeats N`** asks each query N times, as a single-artifact run does, and prints and writes the `noise` block (agreement per query, the rate of each repeat, `half_range`). The model arm's counts are then calls. `--repeats` still needs `--model`. +- **`novelty routing --all-vs-all --model`** writes the same `cost` block as the single-artifact run, on stdout and in the result file. + +### Fixed + +- **A query that names a skill left out for `disable-model-invocation` says so.** `--all-vs-all` refused it as "not in the field" with no hint of why; the message now names each such skill and points to `--include-disabled`. +- **`--queries-from-headings` no longer gives up on a skill with no usable heading.** Only `##` and `###` headings of two or more content words were offered, so a skill whose headings were one word each, or a `#` title alone, had no queries (24 of 144 real skills). Its description's sentences are now the fallback, still stamped as self-derived triage, and the result records `queries_source`. A skill with neither still exits non-zero with the reason. +- **`--seed` help says what it fixes.** It seeds the card order only; it does not make the model's answers repeatable, which is what `--repeats` measures. The help no longer calls the run reproducible. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index c9eb437d1..611cc6773 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -8277,6 +8277,22 @@ "required": false, "type": null }, + { + "action": "_StoreTrueAction", + "choices": null, + "const": true, + "default": false, + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": true, + "key": "--estimate-only", + "kind": "option", + "nargs": 0, + "position": null, + "required": false, + "type": null + }, { "action": "_HelpAction", "choices": null, diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 0894076c7..ce5a38719 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -618,6 +618,10 @@ A catalog that holds the same `SKILL.md` under two paths (a plugin copy of its ` tree) counts it once: the copy nearest the catalog root stands for the card, and the others are named on stderr. +`--estimate-only` prints, as JSON, the calls, input and output tokens and the list-price `usd` the +`--model` arm would make, and makes no model call. Without `--model` the run is lexical and the bill +is zero. Input is the selection prompt sized at chars / 4, so it is a projection, not a count. + Two numbers, reported side by side and never merged: - **Lexical** — BM25 over each candidate's `name + description`, which is exactly the text @@ -626,7 +630,9 @@ Two numbers, reported side by side and never merged: a **seeded random permutation per query** (`--seed`), so a model that always picks the first option tends to `1/N` rather than to a figure set by how the query count lines up with the candidate count. It *tends* to it: at five queries the realised rate is a fixed - offset from `1/N`, so read small query sets as such. Each permutation and the position + offset from `1/N`, so read small query sets as such. `--seed` fixes that card order and + nothing else: the model's answers are still sampled and can differ between runs, so measure + that spread with `--repeats`. Each permutation and the position the gold card was shown at are recorded per query. **An abstention counts against `selection_rate`** — not picking anything is the routing failure — with `selection_rate_among_answered` beside it; a failed call is excluded from both and @@ -651,7 +657,7 @@ the provider they may or may not be counted inside `input_tokens`. A run without no calls and has no `cost` block. `usd` is also `null` when a call returned no usage, and a call that raised is not counted. A call replayed from the response cache is counted in `replayed_calls` and left out of the token and `usd` totals, since this run did not pay for it; -`llm_calls` still counts it. `--all-vs-all` writes no `cost` block. +`llm_calls` still counts it. `--all-vs-all` writes the same `cost` block over its model arm. `recall_at_3` comes back `null` with three or fewer candidates, where every rank is `<= 3` and the figure would be a structural 1.0. The lexical arm also reports `n_queries_given` @@ -674,10 +680,14 @@ is no attractor, and a tied skill's own rate is lower by the picks inside its gr model call is unscored, and a skill with none scored has an own rate of `null`, not 0.0. A run with any failed model call writes the result and exits non-zero. The saved file records the field rule under `field` (`include_disabled`, the `competitors` and what was `left_out`); a -single-artifact run records it too. It takes no `--artifact`, `--queries`, -`--queries-from-headings` or `--repeats` (one model call per query). The model arm also records the selector `model`, so two runs with +single-artifact run records it too. It takes no `--artifact`, `--queries` or `--queries-from-headings`. `--repeats` needs `--model` +and asks each query that many times, as a single-artifact run does: the model arm then carries a +`noise` block (agreement per query, the rate of each repeat, `half_range`), and its counts are +calls, one per query per repeat. The model arm also records the selector `model`, so two runs with different models can be told apart. It is refused with fewer than two skills, a name repeated -in the field, a query file that names a skill outside the field, or no queries at all. +in the field, or no queries at all. A query file that names a skill outside the field is refused; +when that skill is one the field leaves out for `disable-model-invocation`, the message names it and +points to `--include-disabled`. **Cost of `--model` in all-vs-all.** The model arm makes one selector call per query, across every skill in the field: the call count is the number of queries in `queries.json` that have a @@ -698,7 +708,10 @@ Three properties worth knowing before quoting either: and its own relevance. `--queries-from-headings` exists for triage, warns, and stamps `queries_self_derived: true` into the result; corpus-wide those self-derived queries gave recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667. Headings inside - fenced code blocks are not offered as queries. A run with no usable query exits non-zero, + fenced code blocks are not offered as queries. Only `##` and `###` headings of two or more + content words count; a skill whose headings give none falls back to the sentences of its + description with two or more content words, and `queries_source` (`headings`, `description` or + `none`) says which was used. A run with no usable query exits non-zero, says why, and still writes the result with `"status": "no_queries"`. Every rate line prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer than five scored queries draw a warning. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index ae625fd9c..006838765 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -20,6 +20,12 @@ from typing import TYPE_CHECKING, NoReturn from dokimi.commands.inputs import placebo_text_for_arms_or_exit +from dokimi.commands.novelty_routing_report import ( + print_all_vs_all_model, + print_routing_estimate, + print_routing_summary, + routing_queries, +) from dokimi.defaults import ( DEFAULT_ANSWER_MODEL, DEFAULT_ANSWER_PROVIDER, @@ -349,6 +355,7 @@ def _routing_field(args: argparse.Namespace, exclude: set[str], def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: """Every skill in the field is the gold in turn: confusion matrix and attractors.""" from dokimi.novelty.all_vs_all import all_vs_all, load_queries_by_skill + from dokimi.novelty.client import roll_up_cost from dokimi.novelty.routing import RoutingInputError, tokenize _check_all_vs_all_args(args) @@ -364,6 +371,7 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: queries_by_skill = load_queries_by_skill(args.queries_by_skill) except (OSError, UnicodeDecodeError) as exc: usage_exit(f"cannot read the queries: {exc}") + _refuse_queries_naming_disabled(args, field, queries_by_skill) selector = _client(args.model, args.provider, "selector", args) if args.model else None if selector is not None: # One call per query with a content word, each sending every card in the field. @@ -373,7 +381,8 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: f"{len(field)} cards each", file=sys.stderr, ) - result = all_vs_all(field, queries_by_skill, client=selector, seed=args.seed) + result = all_vs_all(field, queries_by_skill, client=selector, seed=args.seed, + repeats=args.repeats) except RoutingInputError as exc: usage_exit(str(exc)) without = [c.name for c in field if c.name not in queries_by_skill] @@ -395,6 +404,8 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: payload = {"mode": "all_vs_all", "queries_by_skill": args.queries_by_skill, "n_candidates": len(field), "result": result} payload["field"] = _field_record(args, field, left_out) + if selector is not None: + payload["cost"] = roll_up_cost([selector]) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") if failed: raise SystemExit(f"{failed} of {sent} model call(s) failed " @@ -404,6 +415,31 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print(f"all-vs-all: {len(field)} skills in the field, {result['n_queries']} queries over " f"{len(queries_by_skill)} skills with queries") _print_attractors(result) + print_all_vs_all_model(model_arm, payload.get("cost")) + + +def _refuse_queries_naming_disabled(args: argparse.Namespace, field: list, + queries_by_skill: dict[str, list[str]]) -> None: + """Refuse queries for a skill the field leaves out only because it sets disable-model-invocation. + + The generic "not in the field" refusal does not say why, so this names the disabled skills + and the flag that keeps them. Other unknown names fall through to that refusal. + """ + if args.include_disabled: + return + held = {c.name for c in field} + unknown = sorted(set(queries_by_skill) - held) + if not unknown: + return + everyone = _routing_field(argparse.Namespace(**{**vars(args), "include_disabled": True}), + set()) + disabled = [name for name in unknown if name in {c.name for c in everyone}] + if disabled: + usage_exit("the queries name skills that are not in the field: " + + ", ".join(f"`{n}`" for n in unknown) + ". " + + "; ".join(f"`{n}` sets disable-model-invocation, so the field leaves it out" + for n in disabled) + + "; pass --include-disabled to keep those skills in the field") def _field_record(args: argparse.Namespace, competitors: list, left_out: list[str]) -> dict: @@ -418,8 +454,6 @@ def _check_all_vs_all_args(args: argparse.Namespace) -> None: usage_exit("--all-vs-all takes no --artifact: every skill in the field is the gold in turn") if args.queries or args.queries_from_headings: usage_exit("--all-vs-all takes --queries-by-skill, not --queries or --queries-from-headings") - if args.repeats > 1: - usage_exit("--all-vs-all takes no --repeats: it makes one call per query") if not args.queries_by_skill: usage_exit("--all-vs-all needs --queries-by-skill : a JSON object of skill " "name to the requests that should select it") @@ -475,9 +509,7 @@ def cmd_novelty_routing(args: argparse.Namespace) -> None: RoutingInputError, check_field, disables_model_invocation, - headings_as_queries, lexical_routing, - load_queries, model_routing, skill_file, ) @@ -505,34 +537,24 @@ def cmd_novelty_routing(args: argparse.Namespace) -> None: except RoutingInputError as exc: usage_exit(str(exc)) - self_derived = False - if args.queries: - try: - queries = load_queries(args.queries) - except RoutingInputError as exc: - usage_exit(str(exc)) - elif args.queries_from_headings: - queries = headings_as_queries(body) - self_derived = True - print("WARNING: queries derived from the artifact's own headings. The artifact then " - "defines both the problem scope and its own relevance, so this is triage, not " - "a routing rate", file=sys.stderr) - else: - usage_exit("pass --queries with requests written independently of the artifact, or " - "--queries-from-headings for a self-derived triage run") + queries, self_derived, queries_source = routing_queries(args, body, gold) candidates = [gold, *competitors] + if args.estimate_only: + print_routing_estimate(args, candidates, queries) + return lexical = dict(lexical_routing(candidates, queries, gold=0), queries_self_derived=self_derived) out = Path(args.out) if args.out else _default_dir(args.artifact) / "routing.json" out.parent.mkdir(parents=True, exist_ok=True) if not lexical["n_queries_scored"]: - _refuse_no_queries(args, queries, self_derived, out, len(candidates)) + _refuse_no_queries(args, queries, self_derived, out, len(candidates), queries_source) payload = { "artifact": args.artifact, "status": "ok", "n_candidates": len(candidates), "queries_self_derived": self_derived, + "queries_source": queries_source, "field": _field_record(args, competitors, left_out), "lexical": lexical, } @@ -550,67 +572,18 @@ def cmd_novelty_routing(args: argparse.Namespace) -> None: out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"OK Saved routing measurement: {out}") - _print_routing_summary(payload, len(competitors), self_derived) - - -def _print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: - """The lexical line, the model line and their caveats, printed after a routing run.""" - # Every rate is printed with the number of queries behind it, and a self-derived one - # carries the triage tag on the same line, so a copied figure cannot lose either. - tag = " triage: self-derived" if self_derived else "" - lexical = payload["lexical"] - print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " - f"MRR {lexical['mrr']} n_queries {lexical['n_queries_scored']} of " - f"{lexical['n_queries_given']} given, vs {n_competitors} competitors{tag}" - + (" (R@3 needs more than 3 candidates to mean anything)" - if lexical["recall_at_3"] is None else "")) - if lexical["n_queries_scored"] < 5: - n_scored = lexical["n_queries_scored"] - noun = "query" if n_scored == 1 else "queries" - print(f"WARNING: only {n_scored} {noun} scored; one query moves " - "a rate by 20 points or more, so do not quote it as a routing rate") - if "model" in payload: - row = payload["model"] - print(f"model selection {row['selection_rate']} " - f"({row['selection_rate_among_answered']} among answered; " - f"{row['n_abstained']} abstained, {row['n_failed']} failed) " - f"[{row['model']}] n_queries {row['n_queries']} given{tag}") - if row["n_abstained"]: - of_calls = f" of {row['n_calls']} calls" if row["repeats"] > 1 else "" - print(f"{row['n_abstained']}{of_calls} abstained or unparseable: the raw reply " - "is under per_query[].raw_reply in the output") - if row["noise"]: - noise = row["noise"] - print(f"noise {row['repeats']} repeats per query: mean agreement " - f"{noise['mean_agreement']}, {noise['unstable_queries']} queries picked " - f"differently across repeats; selection by repeat " - f"{noise['selection_rate_by_repeat']}, half range " - f"{noise['half_range']} (a crude sampling floor)") - print("The model figure is an upper bound on harness behaviour: a harness that " - "truncates descriptions in its own listing selects on less text.") - cost = payload["cost"] - if not cost["llm_calls"]: - print("cost 0 calls") - else: - spent = ("USD not known: a call had no price or no usage" if cost["usd"] is None - else "<$0.0001" if 0 < cost["usd"] < 0.00005 else f"${cost['usd']:.4f}") - replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" - if cost["replayed_calls"] else "") - print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " - f"{cost['output_tokens']} output tokens, cache read " - f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " - f"{spent}{replayed}") - if self_derived: - print("Self-derived queries: read as triage only.") + print_routing_summary(payload, len(competitors), self_derived) def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, - out: Path, n_candidates: int) -> NoReturn: + out: Path, n_candidates: int, + queries_source: str | None = None) -> NoReturn: """Write the ``no_queries`` record, then exit saying why no query was scored.""" # Written too, so a batch run leaves a record of why it has no rate. if not queries and self_derived: - reason = (f"no section heading of two or more words outside code fences in " - f"{args.artifact}; headings inside fenced code are not requests") + reason = (f"no section heading or description sentence of two or more words in " + f"{args.artifact} outside code fences; headings inside fenced code are not " + "requests") elif not queries: reason = f"--queries {args.queries} holds no non-blank query" else: @@ -618,6 +591,7 @@ def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derive "like a, the, how do not count") out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries", "reason": reason, "queries_self_derived": self_derived, + "queries_source": queries_source or ("none" if self_derived else None), "n_candidates": n_candidates}, indent=2), encoding="utf-8") usage_exit(f"no usable queries: {reason}. Pass --queries with requests written independently " @@ -870,21 +844,29 @@ def register(sub, positive_int) -> None: "should select this artifact, written independently of it") routing_p.add_argument("--queries-from-headings", action="store_true", dest="queries_from_headings", - help="Derive queries from the artifact's own headings. Triage " - "only: the artifact then defines both the problem scope and " - "its own relevance, and the result is stamped as such") + help="Derive queries from the artifact's own ## and ### headings of " + "two or more words, or, when it has none, from the sentences of " + "its description. Triage only: the artifact then defines both " + "the problem scope and its own relevance, and the result is " + "stamped as such") routing_p.add_argument("--model", default=None, help="Also ask this model to pick from the same cards. Optional: " "without it only the free lexical floor is computed") routing_p.add_argument("--provider", default=DEFAULT_ANSWER_PROVIDER, choices=PROVIDERS) routing_p.add_argument("--max-tokens", type=positive_int, default=None, dest="max_tokens") routing_p.add_argument("--seed", type=int, default=0, - help="Seeds the per-query card permutation, so a position-biased " - "model scores ~1/N and the run is reproducible") + help="Seeds only the card order: each query's cards are listed in " + "a seeded random permutation, so a position-biased model scores " + "about 1/N. It does not make the model's answers repeatable; " + "measure that with --repeats") routing_p.add_argument("--repeats", type=positive_int, default=1, help="Ask the model this many times per query, with the same card " "order each time, and report per-query agreement and the " "run-to-run noise floor. Needs --model; costs N times the calls") + routing_p.add_argument("--estimate-only", action="store_true", dest="estimate_only", + help="Print the calls, input and output tokens and the list-price " + "USD that --model would make, as JSON, and make no model call. " + "Without --model the run is lexical and the bill is zero") routing_p.add_argument("--out", default=None) _add_spend_cap(routing_p) routing_p.set_defaults(func=_stops_at_cap(cmd_novelty_routing)) diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py new file mode 100644 index 000000000..cb35c1895 --- /dev/null +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -0,0 +1,141 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Queries, console lines and refusals of ``novelty routing``. + +Split from ``commands/novelty.py``, which keeps the command bodies, so that file stays under +its growth cap. Nothing here builds a client; a model arm is always the caller's. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from typing import TYPE_CHECKING + +from dokimi.exit_codes import usage_exit + +if TYPE_CHECKING: + from dokimi.novelty.routing import Candidate + + +def routing_queries(args: argparse.Namespace, body: str, + gold: Candidate) -> tuple[list[str], bool, str | None]: + """The queries of one routing run: ``(queries, self_derived, queries_source)``. + + ``--queries`` is read as written. ``--queries-from-headings`` derives them from the artifact + itself and names the source: ``headings``, ``description``, or ``none`` when neither gives any. + """ + from dokimi.novelty.routing import RoutingInputError, load_queries, self_derived_queries + + if args.queries: + try: + return load_queries(args.queries), False, None + except RoutingInputError as exc: + usage_exit(str(exc)) + if not args.queries_from_headings: + usage_exit("pass --queries with requests written independently of the artifact, or " + "--queries-from-headings for a self-derived triage run") + queries, source = self_derived_queries(body, gold.description) + print(f"WARNING: queries derived from the artifact's own " + f"{'headings' if source == 'headings' else 'description'}. The artifact " + "then defines both the problem scope and its own relevance, so this is triage, " + "not a routing rate", file=sys.stderr) + return queries, True, source + + +def print_routing_estimate(args: argparse.Namespace, candidates: list, queries: list[str]) -> None: + """The model arm's calls, tokens and list-price USD, as JSON; nothing is sent. + + No client is built. Without ``--model`` the run is lexical only, so the bill is zero. + """ + from dokimi.llm_pricing import list_cost_usd + from dokimi.novelty.routing import selection_estimate + + # Without --model there is no selector, so no call is made, whatever --repeats says. + estimate = selection_estimate(candidates, queries, + repeats=args.repeats if args.model else 0) + usd: float | None = 0.0 + if args.model and estimate["n_calls"]: + priced = list_cost_usd(args.model, args.provider, estimate["input_tokens"], + estimate["output_tokens"]) + usd = None if priced is None else round(priced, 6) + print(json.dumps({"estimate_only": True, "artifact": args.artifact, "model": args.model, + "provider": args.provider if args.model else None, + "n_candidates": len(candidates), **estimate, + "expected_usd": usd}, indent=2)) + print("Estimate only, nothing was measured. Input is the selection prompt at chars / 4 per " + "call; output is one JSON pick. A reasoning model bills more output than this.", + file=sys.stderr) + if args.model and estimate["n_calls"] and usd is None: + print(f"expected_usd is null: {args.model} has no list price in the litellm price map", + file=sys.stderr) + + +def print_routing_summary(payload: dict, n_competitors: int, self_derived: bool) -> None: + """The lexical line, the model line and their caveats, printed after a routing run.""" + # Every rate is printed with the number of queries behind it, and a self-derived one + # carries the triage tag on the same line, so a copied figure cannot lose either. + tag = " triage: self-derived" if self_derived else "" + lexical = payload["lexical"] + print(f"lexical R@1 {lexical['recall_at_1']} R@3 {lexical['recall_at_3']} " + f"MRR {lexical['mrr']} n_queries {lexical['n_queries_scored']} of " + f"{lexical['n_queries_given']} given, vs {n_competitors} competitors{tag}" + + (" (R@3 needs more than 3 candidates to mean anything)" + if lexical["recall_at_3"] is None else "")) + if lexical["n_queries_scored"] < 5: + n_scored = lexical["n_queries_scored"] + noun = "query" if n_scored == 1 else "queries" + print(f"WARNING: only {n_scored} {noun} scored; one query moves " + "a rate by 20 points or more, so do not quote it as a routing rate") + if "model" in payload: + row = payload["model"] + print(f"model selection {row['selection_rate']} " + f"({row['selection_rate_among_answered']} among answered; " + f"{row['n_abstained']} abstained, {row['n_failed']} failed) " + f"[{row['model']}] n_queries {row['n_queries']} given{tag}") + if row["n_abstained"]: + of_calls = f" of {row['n_calls']} calls" if row["repeats"] > 1 else "" + print(f"{row['n_abstained']}{of_calls} abstained or unparseable: the raw reply " + "is under per_query[].raw_reply in the output") + if row["noise"]: + print_noise(row) + print("The model figure is an upper bound on harness behaviour: a harness that " + "truncates descriptions in its own listing selects on less text.") + print_cost(payload["cost"]) + if self_derived: + print("Self-derived queries: read as triage only.") + + +def print_noise(model_arm: dict) -> None: + """The repeat noise of a model arm, as its one console line.""" + noise = model_arm["noise"] + print(f"noise {model_arm['repeats']} repeats per query: mean agreement " + f"{noise['mean_agreement']}, {noise['unstable_queries']} queries picked " + f"differently across repeats; selection by repeat " + f"{noise['selection_rate_by_repeat']}, half range " + f"{noise['half_range']} (a crude sampling floor)") + + +def print_cost(cost: dict) -> None: + """The usage block of a ``--model`` run, as its one console line.""" + if not cost["llm_calls"]: + print("cost 0 calls") + return + spent = ("USD not known: a call had no price or no usage" if cost["usd"] is None + else "<$0.0001" if 0 < cost["usd"] < 0.00005 else f"${cost['usd']:.4f}") + replayed = (f", {cost['replayed_calls']} replayed from the cache and not counted" + if cost["replayed_calls"] else "") + print(f"cost {cost['llm_calls']} calls, {cost['input_tokens']} input / " + f"{cost['output_tokens']} output tokens, cache read " + f"{cost['cache_read_tokens']} / write {cost['cache_write_tokens']}, " + f"{spent}{replayed}") + + +def print_all_vs_all_model(model_arm: dict, cost: dict | None) -> None: + """The noise and cost lines of an all-vs-all run; silent without ``--model``.""" + if model_arm.get("noise"): + print_noise(model_arm) + if cost is not None: + print_cost(cost) diff --git a/dokimi/dokimi/novelty/all_vs_all.py b/dokimi/dokimi/novelty/all_vs_all.py index f26267c92..21aab97b4 100644 --- a/dokimi/dokimi/novelty/all_vs_all.py +++ b/dokimi/dokimi/novelty/all_vs_all.py @@ -17,7 +17,7 @@ from pathlib import Path from .client import LLMClient -from .routing import BM25, Candidate, RoutingInputError, model_routing, tokenize +from .routing import BM25, Candidate, RoutingInputError, _noise, model_routing, tokenize NO_PICK = "(none)" TIE = "(tie)" @@ -65,12 +65,12 @@ def load_queries_by_skill(path: str) -> dict[str, list[str]]: def all_vs_all(candidates: list[Candidate], queries_by_skill: dict[str, list[str]], *, - client: LLMClient | None = None, seed: int = 0) -> dict: + client: LLMClient | None = None, seed: int = 0, repeats: int = 1) -> dict: """Each skill's queries, routed against the whole field: a confusion matrix and attractors. Every skill that has queries is the gold in turn. Its queries go to the whole field, and the pick is recorded. The lexical arm always runs (free); the model arm runs when - ``client`` is given. A query no card shares a word with is ``(none)``, not the first card: + ``client`` is given, ``repeats`` times per query as :func:`model_routing` asks. A query no card shares a word with is ``(none)``, not the first card: the tie-break in ``lexical_routing`` would otherwise name a card that matched nothing. An attractor is a skill picked for queries written for other skills. ``wrongly_picked`` @@ -84,8 +84,10 @@ def all_vs_all(candidates: list[Candidate], queries_by_skill: dict[str, list[str ``confusion`` (gold skill to picked skill to count, ``(none)`` and ``(tie)`` included), ``own_rate`` per gold skill (``None`` when none of its queries was scored), ``n_scored`` and ``n_unscored`` per gold skill, and ``attractors`` ordered by ``wrongly_picked``. The - model arm also records ``model``, ``seed``, ``n_failed``, ``n_abstained`` and - ``first_error``. + model arm also records ``model``, ``seed``, ``repeats``, ``noise`` (``None`` for one + repeat; otherwise as :func:`dokimi.novelty.routing.model_routing` reports it, pooled over + every skill's queries), ``n_failed``, ``n_abstained`` and ``first_error``. With repeats + the model arm's counts are calls, one per query per repeat. Raises: RoutingInputError: Fewer than two skills in the field; a name repeated in the field; a @@ -101,7 +103,7 @@ def all_vs_all(candidates: list[Candidate], queries_by_skill: dict[str, list[str names, _tied_groups(candidates)), } if client is not None: - result["model"] = _model_arm(candidates, queries_by_skill, names, client, seed) + result["model"] = _model_arm(candidates, queries_by_skill, names, client, seed, repeats) return result @@ -179,7 +181,7 @@ def _lexical_picks(candidates: list[Candidate], queries_by_skill: dict[str, list def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str]], - names: list[str], client: LLMClient, seed: int) -> dict: + names: list[str], client: LLMClient, seed: int, repeats: int) -> dict: """The model arm of :func:`all_vs_all`: attractor table plus failure counts. A query with no content word is not sent: it is unscored here as in the lexical arm, so @@ -187,6 +189,7 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str """ model: dict[str, Counter] = {} unscored: dict[str, int] = {} + groups: list[list[dict]] = [] # one per query, its repeats in order, for the noise block n_failed = n_abstained = 0 first_error = None for index, name in enumerate(names): @@ -194,13 +197,14 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str continue queries = [q for q in queries_by_skill[name] if tokenize(q)] if len(queries) < len(queries_by_skill[name]): - unscored[name] = len(queries_by_skill[name]) - len(queries) + unscored[name] = (len(queries_by_skill[name]) - len(queries)) * repeats if not queries: model[name] = Counter() continue routed = model_routing(candidates, queries, client=client, - gold=index, seed=seed + index) + gold=index, seed=seed + index, repeats=repeats) rows = routed["per_query"] + groups += [rows[start:start + repeats] for start in range(0, len(rows), repeats)] n_failed += routed["n_failed"] n_abstained += routed["n_abstained"] if first_error is None: @@ -210,8 +214,9 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str if routed["n_failed"]: unscored[name] = unscored.get(name, 0) + routed["n_failed"] return dict(_attractor_table(model, unscored, names, _tied_groups(candidates)), - model=client.model, seed=seed, n_failed=n_failed, n_abstained=n_abstained, - first_error=first_error) + model=client.model, seed=seed, repeats=repeats, + noise=_noise(groups) if repeats > 1 and groups else None, + n_failed=n_failed, n_abstained=n_abstained, first_error=first_error) def _attractor_table(picks_by_gold: dict[str, Counter], unscored: dict[str, int], diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 021ba4475..cbef5eb78 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -425,6 +425,61 @@ def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: return [h for h in headings if len(tokenize(h)) >= 2][:limit] +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+") + + +def description_as_queries(description: str, *, limit: int = 40) -> list[str]: + """Sentences of a description, for triage when the artifact has no usable heading. + + A skill whose body is a ``#`` title and no ``##`` heading of two or more words still says + what it is for in its description. Each sentence with two or more content words is + offered, as :func:`headings_as_queries` offers headings. The same self-derived stamp applies. + """ + sentences = _SENTENCE_END.split(" ".join(description.split())) + return [s for s in sentences if len(tokenize(s)) >= 2][:limit] + + +def self_derived_queries(body: str, description: str) -> tuple[list[str], str]: + """The triage queries of an artifact and where they came from. + + Headings are preferred; the description's sentences are the fallback. Returns + ``(queries, source)`` with ``source`` ``"headings"``, ``"description"``, or ``"none"`` when + neither gives a query of two or more content words. + """ + headings = headings_as_queries(body) + if headings: + return headings, "headings" + sentences = description_as_queries(description) + return sentences, "description" if sentences else "none" + + +# A selection reply is one JSON object, ``{"pick": N}``: a few tokens. +PICK_REPLY_TOKENS = 10 + + +def selection_estimate(candidates: list[Candidate], queries: list[str], *, + repeats: int = 1) -> dict: + """The calls and tokens :func:`model_routing` would make, counted without making them. + + A query with no content word is not sent. Each call carries the whole field's cards in the + selection prompt, sized at chars / 4 as the artifact spans are; every repeat sends the same + prompt. The reply is one JSON pick, :data:`PICK_REPLY_TOKENS`. + """ + cards = "\n".join(f"{position}. {candidate.card}" + for position, candidate in enumerate(candidates, 1)) + sent = [query for query in queries if tokenize(query)] + prompt_tokens = sum(len(_SELECT_PROMPT.format(query=query, cards=cards)) // 4 + for query in sent) + calls = len(sent) * repeats + return { + "n_queries": len(queries), + "n_queries_sent": len(sent), + "n_calls": calls, + "input_tokens": prompt_tokens * repeats, + "output_tokens": calls * PICK_REPLY_TOKENS, + } + + def non_skill_reason(ref: str) -> str | None: """Why a ``skill:`` reference names something other than a skill, or ``None``. diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index d751c4643..da76c775c 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1189, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1171, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, @@ -21,7 +21,7 @@ "dokimi/commands/catalog.py::cmd_catalog_trigger_set": {"complexity": 14}, "dokimi/commands/evaluate.py::cmd_eval_judge_validity": {"complexity": 29, "rationale": "grew on main in #73 (judge-validity --context stored) from 25 to 29 before the ratchet landed; split it in its own PR"}, "dokimi/commands/judge_reliability.py::cmd_eval_cross_judge": {"complexity": 30, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 13, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, + "dokimi/commands/novelty.py::cmd_novelty_routing": {"complexity": 11, "rationale": "W8 adds the --repeats guard after main's no_queries handling raised the file; the command is split with the rest of novelty.py in a separate PR"}, "dokimi/commands/repo.py::cmd_repo_scan": {"complexity": 12}, "dokimi/commands/repo_run.py::_context": {"complexity": 11, "rationale": "went over the cap on main in #120 (built-in review graders) after #93 seeded this baseline; split it in its own PR"}, "dokimi/commands/report.py::cmd_report_model_compare": {"complexity": 12}, diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index f209122ba..ac37d9348 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -805,7 +805,7 @@ def test_a_single_usable_repeat_has_no_half_range(): def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path): out = tmp_path / "routing.json" args = _parse("novelty", "routing", - "--artifact", _skill(tmp_path, "a", "profile a binary", + "--artifact", _skill(tmp_path, "a", "Tune.", body="```\n## only in code\n```\n"), "--competitors", _skill(tmp_path, "b", "tune a database"), "--queries-from-headings", "--out", str(out)) @@ -814,6 +814,7 @@ def test_headings_inside_code_only_refuse_the_run_and_say_why(tmp_path): payload = json.loads(out.read_text(encoding="utf-8")) assert payload["status"] == "no_queries" assert "code fences" in payload["reason"] + assert payload["queries_source"] == "none" def test_a_query_file_with_nothing_in_it_refuses_the_run(tmp_path): diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index f3076861b..e6d9464ec 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -7,6 +7,7 @@ import functools import json +import re import pytest @@ -566,8 +567,10 @@ def test_all_vs_all_refuses_the_single_artifact_query_flags(tmp_path, flag): args.func(args) -def test_all_vs_all_refuses_repeats_it_would_not_make(tmp_path, monkeypatch): - # all_vs_all makes one call per query, so --repeats 3 would be dropped without a word. +def test_all_vs_all_takes_repeats_and_prints_the_noise_and_the_cost(tmp_path, capsys, + monkeypatch): + # Three repeats of one query: three calls, the same card order each time, so the + # selector agrees with itself and the noise block says so. import dokimi.commands.novelty as novelty_cmd calls = [] @@ -577,12 +580,78 @@ def test_all_vs_all_refuses_repeats_it_would_not_make(tmp_path, monkeypatch): root = tmp_path / "catalog" _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) queries = _write_queries(tmp_path, {"perf": ["profile my slow binary"]}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(out), "--model", "m", + "--repeats", "3") + args.func(args) + assert len(calls) == 3 + payload = json.loads(out.read_text(encoding="utf-8")) + noise = payload["result"]["model"]["noise"] + assert payload["result"]["model"]["repeats"] == 3 + assert (noise["mean_agreement"], noise["unstable_queries"]) == (1.0, 0) + assert payload["cost"]["llm_calls"] == 3 + stdout = capsys.readouterr().out + assert "noise 3 repeats per query" in stdout + assert "cost 3 calls" in stdout + + +def test_all_vs_all_repeats_still_need_a_model(tmp_path): + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile my slow binary"]}) args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), - "--model", "m", "--repeats", "3") - with pytest.raises(SystemExit, match="takes no --repeats"): + "--repeats", "3") + with pytest.raises(SystemExit, match="--repeats needs --model"): + args.func(args) + assert not (tmp_path / "o.json").exists() + + +def test_all_vs_all_without_model_writes_no_cost_block(tmp_path): + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile my slow binary"]}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(out)) + args.func(args) + assert "cost" not in json.loads(out.read_text(encoding="utf-8")) + + +def test_queries_naming_a_disabled_skill_say_so_and_suggest_the_flag(tmp_path): + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary"}) + (root / "skills" / "grill").mkdir(parents=True) + (root / "skills" / "grill" / "SKILL.md").write_text( + "---\nname: grill\ndescription: relentless interview about a plan\n" + "disable-model-invocation: true\n---\n\n# B\n", encoding="utf-8") + queries = _write_queries(tmp_path, {"perf": ["profile a slow binary"], + "grill": ["interview me about my plan"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json")) + with pytest.raises(SystemExit) as exc: args.func(args) - assert calls == [] and not (tmp_path / "o.json").exists() + message = str(exc.value) + assert "not in the field: `grill`" in message + assert re.search(r"`grill` sets disable-model-invocation.*--include-disabled", message) + assert not (tmp_path / "o.json").exists() + + +def test_the_disabled_skill_routes_once_include_disabled_is_passed(tmp_path, capsys): + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary"}) + (root / "skills" / "grill").mkdir(parents=True) + (root / "skills" / "grill" / "SKILL.md").write_text( + "---\nname: grill\ndescription: relentless interview about a plan\n" + "disable-model-invocation: true\n---\n\n# B\n", encoding="utf-8") + queries = _write_queries(tmp_path, {"perf": ["profile a slow binary"], + "grill": ["interview me about my plan"]}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(out), "--include-disabled") + args.func(args) + assert json.loads(out.read_text(encoding="utf-8"))["result"]["n_skills"] == 2 def test_a_single_artifact_routing_run_records_the_field_rule_too(tmp_path): diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py new file mode 100644 index 000000000..b78cd8694 --- /dev/null +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -0,0 +1,109 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Routing: queries from description sentences, the seed help, and the estimate-only mode.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.novelty.client import LLMClient + + +def _parse(*argv): + return build_parser().parse_args(list(argv)) + + +def _skill(tmp_path, name, description, body="# B\n\n## A section here\n"): + root = tmp_path / name + root.mkdir() + (root / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: {description}\n---\n\n{body}", encoding="utf-8") + return f"skill:{root}" + + +def test_a_skill_with_no_usable_heading_is_queried_by_its_description_sentences(tmp_path, capsys): + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "grill", "Grill the user about a plan. " + "Use when stress-testing a design.", + body="# Grill\n\n## Steps\n"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries-from-headings", "--out", str(out)) + args.func(args) + payload = json.loads(out.read_text(encoding="utf-8")) + assert payload["status"] == "ok" + assert payload["queries_self_derived"] is True # still triage, still stamped + assert payload["queries_source"] == "description" + assert payload["lexical"]["n_queries_given"] == 2 + assert "description" in capsys.readouterr().err + + +def test_description_sentences_are_queries_only_with_two_content_words(): + from dokimi.novelty.routing import description_as_queries + + assert description_as_queries( + "Stop. Grill the user about a plan. Use when stress-testing\na design!" + ) == ["Grill the user about a plan.", "Use when stress-testing a design!"] + + +def test_headings_are_preferred_to_the_description_when_both_give_queries(): + from dokimi.novelty.routing import self_derived_queries + + body = "# Title\n\n## Profile a binary\n" + assert self_derived_queries(body, "Tune a database. Index a table.") == ( + ["Profile a binary"], "headings") + assert self_derived_queries("# Title only\n", "Tune a database.") == ( + ["Tune a database."], "description") + assert self_derived_queries("# Title only\n", "Stop.") == ([], "none") + + +def test_the_seed_help_says_what_it_fixes_and_does_not_call_the_run_reproducible(capsys): + with pytest.raises(SystemExit): + _parse("novelty", "routing", "--help") + text = " ".join(capsys.readouterr().out.split()) + seed_help = text[text.rindex("--seed"):text.index("--repeats", text.rindex("--seed"))] + assert "card order" in seed_help + assert "reproducible" not in seed_help + + +def test_estimate_only_counts_the_calls_prices_them_and_sends_nothing(tmp_path, capsys, + monkeypatch): + import dokimi.llm_pricing as pricing + + monkeypatch.setattr(LLMClient, "text", + lambda self, *a, **k: pytest.fail("the estimate made a model call")) + monkeypatch.setattr(pricing, "list_cost_usd", + lambda model, provider, prompt, completion: prompt * 1e-6 + completion * 2e-6) + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "parallelize my loops", "the a of"]), + encoding="utf-8") + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "profile a binary with perf record"), + "--competitors", _skill(tmp_path, "tbb", "parallelize loops with task groups"), + "--queries", str(queries), "--model", "m", "--repeats", "2", + "--estimate-only", "--out", str(out)) + args.func(args) + estimate = json.loads(capsys.readouterr().out) + assert estimate["estimate_only"] is True + assert estimate["n_queries"] == 3 and estimate["n_calls"] == 4 # 2 scorable x 2 repeats + assert estimate["input_tokens"] > 4 * 50 and estimate["output_tokens"] > 0 + assert estimate["expected_usd"] == round( + estimate["input_tokens"] * 1e-6 + estimate["output_tokens"] * 2e-6, 6) + assert not out.exists() + + +def test_estimate_only_without_a_model_is_lexical_and_free(tmp_path, capsys): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "profile a binary with perf record"), + "--competitors", _skill(tmp_path, "tbb", "parallelize loops"), + "--queries", str(queries), "--estimate-only") + args.func(args) + estimate = json.loads(capsys.readouterr().out) + assert (estimate["n_calls"], estimate["input_tokens"], estimate["expected_usd"]) == (0, 0, 0.0) From 5ae972933e158d6c74b12a8015eb92b2effc29fb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 10:22:08 -0700 Subject: [PATCH 02/14] fix(novelty): routing estimate counts every sent query, all-vs-all refuses --estimate-only, repeats count calls The model estimate skipped contentless queries that the run still sends, --all-vs-all --estimate-only made paid calls, the stderr call count left out the repeats, and the model arm's unscored and tied calls were printed as queries. Fix each and cover it with a test. Routing console output moves to commands/novelty_routing_report.py to keep commands/novelty.py under its growth cap. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 40 +++------------ .../dokimi/commands/novelty_routing_report.py | 31 +++++++++++ dokimi/dokimi/novelty/routing.py | 9 ++-- dokimi/scripts/growth_baseline.json | 2 +- .../tests/test_novelty_routing_all_vs_all.py | 51 ++++++++++++++++++- dokimi/tests/test_novelty_routing_cli_gaps.py | 4 +- 6 files changed, 98 insertions(+), 39 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 006838765..ad4e0323e 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -22,6 +22,7 @@ from dokimi.commands.inputs import placebo_text_for_arms_or_exit from dokimi.commands.novelty_routing_report import ( print_all_vs_all_model, + print_attractors, print_routing_estimate, print_routing_summary, routing_queries, @@ -374,10 +375,11 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: _refuse_queries_naming_disabled(args, field, queries_by_skill) selector = _client(args.model, args.provider, "selector", args) if args.model else None if selector is not None: - # One call per query with a content word, each sending every card in the field. + # One call per query with a content word and per repeat, each sending every card. calls = sum(bool(tokenize(q)) for qs in queries_by_skill.values() for q in qs) + calls *= args.repeats print( - f"all-vs-all: --model makes {calls} model calls, one per query, over " + f"all-vs-all: --model makes {calls} model calls, one per query and repeat, over " f"{len(field)} cards each", file=sys.stderr, ) @@ -414,7 +416,7 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print(f"OK Saved all-vs-all routing: {out}") print(f"all-vs-all: {len(field)} skills in the field, {result['n_queries']} queries over " f"{len(queries_by_skill)} skills with queries") - _print_attractors(result) + print_attractors(result) print_all_vs_all_model(model_arm, payload.get("cost")) @@ -454,6 +456,9 @@ def _check_all_vs_all_args(args: argparse.Namespace) -> None: usage_exit("--all-vs-all takes no --artifact: every skill in the field is the gold in turn") if args.queries or args.queries_from_headings: usage_exit("--all-vs-all takes --queries-by-skill, not --queries or --queries-from-headings") + if args.estimate_only: + usage_exit("--all-vs-all takes no --estimate-only: it would make the calls it was asked " + "to price; drop the flag, or estimate one artifact with routing --artifact") if not args.queries_by_skill: usage_exit("--all-vs-all needs --queries-by-skill : a JSON object of skill " "name to the requests that should select it") @@ -472,35 +477,6 @@ def _warn_blind_skills(result: dict, queries_by_skill: dict[str, list[str]]) -> "(no content word in them), so their lexical own_rate is None", file=sys.stderr) -def _print_attractors(result: dict) -> None: - """The top five attractors of each arm, one line each, and what the arm left out. - - A tie and an unscored query name no pick, so a "no skill was picked" line beside them - would read as a clean result; the line after the attractors says how many there were. - Skills with the same description words are one tied group, named once, not each other's - attractor. - """ - from dokimi.novelty.all_vs_all import TIE - - for group in result["tied_groups"]: - print(f"tied: {', '.join(group)} have the same description; one group, not attractors " - "of each other") - for arm in ("lexical", "model"): - if arm not in result: - continue - section = result[arm] - rows = section["attractors"][:5] - if not rows: - print(f"{arm}: no skill was picked for another skill's query") - for row in rows: - print(f"{arm}: {row['skill']} picked for {row['wrongly_picked']} of " - f"{row['sibling_queries']} sibling queries (from {', '.join(row['from'])})") - tied = sum(picks.get(TIE, 0) for picks in section["confusion"].values()) - unscored = sum(section["n_unscored"].values()) - if tied or unscored: - print(f"{arm}: {tied} tied and {unscored} unscored queries are not counted as picks") - - def cmd_novelty_routing(args: argparse.Namespace) -> None: """Can this artifact be selected against its competitors? Lexical floor, model on top.""" if args.repeats > 1 and not args.model: diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index cb35c1895..fcc4fbaee 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -133,6 +133,37 @@ def print_cost(cost: dict) -> None: f"{spent}{replayed}") +def print_attractors(result: dict) -> None: + """The top five attractors of each arm, one line each, and what the arm left out. + + A tie and an unscored query name no pick, so a "no skill was picked" line beside them + would read as a clean result; the line after the attractors says how many there were. + Skills with the same description words are one tied group, named once, not each other's + attractor. + """ + from dokimi.novelty.all_vs_all import TIE + + for group in result["tied_groups"]: + print(f"tied: {', '.join(group)} have the same description; one group, not attractors " + "of each other") + for arm in ("lexical", "model"): + if arm not in result: + continue + section = result[arm] + rows = section["attractors"][:5] + if not rows: + print(f"{arm}: no skill was picked for another skill's query") + for row in rows: + print(f"{arm}: {row['skill']} picked for {row['wrongly_picked']} of " + f"{row['sibling_queries']} sibling queries (from {', '.join(row['from'])})") + tied = sum(picks.get(TIE, 0) for picks in section["confusion"].values()) + unscored = sum(section["n_unscored"].values()) + # The model arm counts calls, one per query and repeat; the lexical arm counts queries. + unit = "calls" if section.get("repeats", 1) > 1 else "queries" + if tied or unscored: + print(f"{arm}: {tied} tied and {unscored} unscored {unit} are not counted as picks") + + def print_all_vs_all_model(model_arm: dict, cost: dict | None) -> None: """The noise and cost lines of an all-vs-all run; silent without ``--model``.""" if model_arm.get("noise"): diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index cbef5eb78..a9d058143 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -461,13 +461,14 @@ def selection_estimate(candidates: list[Candidate], queries: list[str], *, repeats: int = 1) -> dict: """The calls and tokens :func:`model_routing` would make, counted without making them. - A query with no content word is not sent. Each call carries the whole field's cards in the - selection prompt, sized at chars / 4 as the artifact spans are; every repeat sends the same - prompt. The reply is one JSON pick, :data:`PICK_REPLY_TOKENS`. + :func:`model_routing` sends every query, contentless ones too, unless none has a content + word: then the run refuses before any call, and so does the count. Each call carries the + whole field's cards in the selection prompt, sized at chars / 4 as the artifact spans are; + every repeat sends the same prompt. The reply is one JSON pick, :data:`PICK_REPLY_TOKENS`. """ cards = "\n".join(f"{position}. {candidate.card}" for position, candidate in enumerate(candidates, 1)) - sent = [query for query in queries if tokenize(query)] + sent = list(queries) if any(tokenize(query) for query in queries) else [] prompt_tokens = sum(len(_SELECT_PROMPT.format(query=query, cards=cards)) // 4 for query in sent) calls = len(sent) * repeats diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index da76c775c..c563fd472 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1171, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --repeats refused under --all-vs-all; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1147, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index e6d9464ec..3638415e1 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -212,7 +212,7 @@ def transport(prompt: str, system: str | None): ) args.func(args) assert ( - "all-vs-all: --model makes 4 model calls, one per query, over 3 cards each" + "all-vs-all: --model makes 4 model calls, one per query and repeat, over 3 cards each" in capsys.readouterr().err ) @@ -883,3 +883,52 @@ def test_attractors_tied_on_count_and_golds_tied_in_from_are_in_name_order(): "from": ["mu"]}, {"skill": "zeta", "wrongly_picked": 2, "sibling_queries": 4, "rate": 0.5, "from": ["beta", "mu"]}] + + +def _model_client_for_repeats(monkeypatch): + import dokimi.commands.novelty as novelty_cmd + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model=model, provider=provider, role=role, + transport=lambda prompt, system: (json.dumps({"pick": 1}), None))) + + +def test_all_vs_all_refuses_estimate_only_before_it_calls_the_model(tmp_path, monkeypatch): + import dokimi.commands.novelty as novelty_cmd + + monkeypatch.setattr(novelty_cmd, "_client", + lambda *a, **k: pytest.fail("--estimate-only run made a model call")) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile my slow binary"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m", "--estimate-only") + with pytest.raises(SystemExit, match="takes no --estimate-only"): + args.func(args) + assert not (tmp_path / "o.json").exists() + + +def test_all_vs_all_announces_every_call_it_makes_with_repeats(tmp_path, capsys, monkeypatch): + _model_client_for_repeats(monkeypatch) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile my slow binary"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m", "--repeats", "3") + args.func(args) + assert "--model makes 3 model calls, one per query and repeat" in capsys.readouterr().err + + +def test_console_counts_unscored_model_calls_when_repeats_are_set(tmp_path, capsys, monkeypatch): + # One contentless query, three repeats: three calls not made, so the model line says calls. + _model_client_for_repeats(monkeypatch) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["the a of"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m", "--repeats", "3") + args.func(args) + assert "model: 0 tied and 3 unscored calls are not counted as picks" in capsys.readouterr().out diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index b78cd8694..81302b810 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -90,7 +90,9 @@ def test_estimate_only_counts_the_calls_prices_them_and_sends_nothing(tmp_path, args.func(args) estimate = json.loads(capsys.readouterr().out) assert estimate["estimate_only"] is True - assert estimate["n_queries"] == 3 and estimate["n_calls"] == 4 # 2 scorable x 2 repeats + # Every query is sent, contentless "the a of" included, as routing sends it: 3 x 2 repeats. + assert estimate["n_queries"] == 3 and estimate["n_queries_sent"] == 3 + assert estimate["n_calls"] == 6 assert estimate["input_tokens"] > 4 * 50 and estimate["output_tokens"] > 0 assert estimate["expected_usd"] == round( estimate["input_tokens"] * 1e-6 + estimate["output_tokens"] * 2e-6, 6) From 7bd7506d424d165bf6c2c2bf332dcbaa1f2c1fdd Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 10:31:51 -0700 Subject: [PATCH 03/14] fix(novelty): no derivation warning when --queries-from-headings derives nothing With neither a heading nor a description sentence, the run refuses with no_queries, but the warning still said the queries were derived from the artifact. Print it only when a query was derived. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty_routing_report.py | 3 +++ dokimi/tests/test_novelty_routing_cli_gaps.py | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index fcc4fbaee..282ed1e30 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -38,6 +38,9 @@ def routing_queries(args: argparse.Namespace, body: str, usage_exit("pass --queries with requests written independently of the artifact, or " "--queries-from-headings for a self-derived triage run") queries, source = self_derived_queries(body, gold.description) + if source == "none": + # Nothing was derived, so there is nothing to label; the refusal that follows says why. + return queries, True, source print(f"WARNING: queries derived from the artifact's own " f"{'headings' if source == 'headings' else 'description'}. The artifact " "then defines both the problem scope and its own relevance, so this is triage, " diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 81302b810..013107460 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -42,6 +42,19 @@ def test_a_skill_with_no_usable_heading_is_queried_by_its_description_sentences( assert "description" in capsys.readouterr().err +def test_no_derived_queries_refuses_without_claiming_a_derivation(tmp_path, capsys): + out = tmp_path / "routing.json" + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "grill", "Stop.", body="# Grill\n\n## Steps\n"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + "--queries-from-headings", "--out", str(out)) + with pytest.raises(SystemExit): + args.func(args) + err = capsys.readouterr().err + assert "derived from the artifact" not in err + assert json.loads(out.read_text(encoding="utf-8"))["status"] == "no_queries" + + def test_description_sentences_are_queries_only_with_two_content_words(): from dokimi.novelty.routing import description_as_queries From aa9fcbeb547c9b8b8680396dd5630f5b89bc03b2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 10:39:16 -0700 Subject: [PATCH 04/14] fix(novelty): attractor line says calls, not queries, when the model arm repeats With --repeats above 1 the model arm's sibling count is per call, so the attractor line now names its unit the way the tie and unscored line does. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty_routing_report.py | 6 +++--- dokimi/tests/test_novelty_routing_cli_gaps.py | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index 282ed1e30..219bf63eb 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -153,16 +153,16 @@ def print_attractors(result: dict) -> None: if arm not in result: continue section = result[arm] + # The model arm counts calls, one per query and repeat; the lexical arm counts queries. + unit = "calls" if section.get("repeats", 1) > 1 else "queries" rows = section["attractors"][:5] if not rows: print(f"{arm}: no skill was picked for another skill's query") for row in rows: print(f"{arm}: {row['skill']} picked for {row['wrongly_picked']} of " - f"{row['sibling_queries']} sibling queries (from {', '.join(row['from'])})") + f"{row['sibling_queries']} sibling {unit} (from {', '.join(row['from'])})") tied = sum(picks.get(TIE, 0) for picks in section["confusion"].values()) unscored = sum(section["n_unscored"].values()) - # The model arm counts calls, one per query and repeat; the lexical arm counts queries. - unit = "calls" if section.get("repeats", 1) > 1 else "queries" if tied or unscored: print(f"{arm}: {tied} tied and {unscored} unscored {unit} are not counted as picks") diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 013107460..6b1777cf7 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -42,6 +42,15 @@ def test_a_skill_with_no_usable_heading_is_queried_by_its_description_sentences( assert "description" in capsys.readouterr().err +def test_attractor_line_counts_calls_when_the_model_arm_repeats(capsys): + from dokimi.commands.novelty_routing_report import print_attractors + + row = {"skill": "perf", "wrongly_picked": 3, "sibling_queries": 6, "from": ["db"]} + print_attractors({"tied_groups": [], "model": {"attractors": [row], "confusion": {}, + "n_unscored": {}, "repeats": 3}}) + assert "picked for 3 of 6 sibling calls" in capsys.readouterr().out + + def test_no_derived_queries_refuses_without_claiming_a_derivation(tmp_path, capsys): out = tmp_path / "routing.json" args = _parse("novelty", "routing", From 1c5348bee2b0a4e1ab6382eae86cf1f3fac96f03 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:19:16 -0700 Subject: [PATCH 05/14] fix(routing): price each estimated call, not the run's total The estimate priced the run's summed prompt and picks as one completion. A long-context tier applies to one call's prompt, so many short calls could be charged the higher rate although no real request reaches it. Each query's prompt and one pick are now priced on their own, summed and multiplied by the repeats. Thread: PRRT_kwDOVBg7g86yTJiq Signed-off-by: Nikolay Petrov --- .../dokimi/commands/novelty_routing_report.py | 28 ++++++++++++++--- dokimi/dokimi/novelty/routing.py | 31 ++++++++++++------- dokimi/tests/test_novelty_routing_cli_gaps.py | 27 ++++++++++++++++ 3 files changed, 69 insertions(+), 17 deletions(-) diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index 219bf63eb..15a9c777b 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -48,22 +48,40 @@ def routing_queries(args: argparse.Namespace, body: str, return queries, True, source +def estimate_routing_usd(model: str, provider: str, prompts: list[int], + repeats: int) -> float | None: + """The list-price USD of the calls of a routing run, priced one call at a time. + + A long-context tier applies to the prompt of one call, so the run's total prompt is not + priced as one completion: each query's prompt and one pick are priced on their own, then + summed and multiplied by the repeats. ``None`` when a call has no list price. + """ + from dokimi.llm_pricing import list_cost_usd + from dokimi.novelty.routing import PICK_REPLY_TOKENS + + total = 0.0 + for prompt in prompts: + cost = list_cost_usd(model, provider, prompt, PICK_REPLY_TOKENS) + if cost is None: + return None + total += cost + return round(total * repeats, 6) + + def print_routing_estimate(args: argparse.Namespace, candidates: list, queries: list[str]) -> None: """The model arm's calls, tokens and list-price USD, as JSON; nothing is sent. No client is built. Without ``--model`` the run is lexical only, so the bill is zero. """ - from dokimi.llm_pricing import list_cost_usd - from dokimi.novelty.routing import selection_estimate + from dokimi.novelty.routing import selection_estimate, selection_prompt_tokens # Without --model there is no selector, so no call is made, whatever --repeats says. estimate = selection_estimate(candidates, queries, repeats=args.repeats if args.model else 0) usd: float | None = 0.0 if args.model and estimate["n_calls"]: - priced = list_cost_usd(args.model, args.provider, estimate["input_tokens"], - estimate["output_tokens"]) - usd = None if priced is None else round(priced, 6) + usd = estimate_routing_usd(args.model, args.provider, + selection_prompt_tokens(candidates, queries), args.repeats) print(json.dumps({"estimate_only": True, "artifact": args.artifact, "model": args.model, "provider": args.provider if args.model else None, "n_candidates": len(candidates), **estimate, diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index a9d058143..88a0a267e 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -457,26 +457,33 @@ def self_derived_queries(body: str, description: str) -> tuple[list[str], str]: PICK_REPLY_TOKENS = 10 -def selection_estimate(candidates: list[Candidate], queries: list[str], *, - repeats: int = 1) -> dict: - """The calls and tokens :func:`model_routing` would make, counted without making them. +def selection_prompt_tokens(candidates: list[Candidate], queries: list[str]) -> list[int]: + """The prompt tokens (chars / 4) of each call one repeat of :func:`model_routing` makes. - :func:`model_routing` sends every query, contentless ones too, unless none has a content - word: then the run refuses before any call, and so does the count. Each call carries the - whole field's cards in the selection prompt, sized at chars / 4 as the artifact spans are; - every repeat sends the same prompt. The reply is one JSON pick, :data:`PICK_REPLY_TOKENS`. + One entry per sent query, in query order. :func:`model_routing` sends every query, contentless + ones too, unless none has a content word: then the run refuses before any call, and the list + is empty. Each call carries the whole field's cards in the selection prompt. """ cards = "\n".join(f"{position}. {candidate.card}" for position, candidate in enumerate(candidates, 1)) sent = list(queries) if any(tokenize(query) for query in queries) else [] - prompt_tokens = sum(len(_SELECT_PROMPT.format(query=query, cards=cards)) // 4 - for query in sent) - calls = len(sent) * repeats + return [len(_SELECT_PROMPT.format(query=query, cards=cards)) // 4 for query in sent] + + +def selection_estimate(candidates: list[Candidate], queries: list[str], *, + repeats: int = 1) -> dict: + """The calls and tokens :func:`model_routing` would make, counted without making them. + + Every repeat sends the same prompts (see :func:`selection_prompt_tokens`). The reply is one + JSON pick, :data:`PICK_REPLY_TOKENS`. + """ + prompts = selection_prompt_tokens(candidates, queries) + calls = len(prompts) * repeats return { "n_queries": len(queries), - "n_queries_sent": len(sent), + "n_queries_sent": len(prompts), "n_calls": calls, - "input_tokens": prompt_tokens * repeats, + "input_tokens": sum(prompts) * repeats, "output_tokens": calls * PICK_REPLY_TOKENS, } diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 6b1777cf7..0aedd8c07 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -131,3 +131,30 @@ def test_estimate_only_without_a_model_is_lexical_and_free(tmp_path, capsys): args.func(args) estimate = json.loads(capsys.readouterr().out) assert (estimate["n_calls"], estimate["input_tokens"], estimate["expected_usd"]) == (0, 0, 0.0) + + +def test_estimate_prices_each_call_so_a_long_context_tier_is_not_reached_by_the_run_total( + tmp_path, capsys, monkeypatch): + import dokimi.llm_pricing as pricing + + def tiered(model, provider, prompt, completion): + # A long-context tier as litellm applies it: the rate follows one call's prompt size. + rate = 1e-6 if prompt <= 200_000 else 2e-6 + return prompt * rate + completion * 2e-6 + + monkeypatch.setattr(LLMClient, "text", + lambda self, *a, **k: pytest.fail("the estimate made a model call")) + monkeypatch.setattr(pricing, "list_cost_usd", tiered) + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary", "profile a program"]), encoding="utf-8") + # About 48k tokens per call and six calls: 288k in total, past the tier, no single call past it. + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "profile " + "filler " * 27_000), + "--competitors", _skill(tmp_path, "tbb", "parallelize loops"), + "--queries", str(queries), "--model", "m", "--repeats", "3", + "--estimate-only") + args.func(args) + estimate = json.loads(capsys.readouterr().out) + assert estimate["input_tokens"] > 200_000 and estimate["n_calls"] == 6 + assert estimate["expected_usd"] == pytest.approx( + estimate["input_tokens"] * 1e-6 + estimate["output_tokens"] * 2e-6, rel=1e-6) From 2ccd64ba7c835f540b805d4de0f3a3eed26b33a2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:19:16 -0700 Subject: [PATCH 06/14] fix(novelty): print the cost and noise of a partial all-vs-all run The failure exit came before the cost and noise lines, so a run with some failed calls hid the usage of the calls that succeeded. Those lines now print before the exit; the success summary stays after it. Thread: PRRT_kwDOVBg7g86yTJix Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 4 ++- .../tests/test_novelty_routing_all_vs_all.py | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index ad4e0323e..88b5fd61d 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -409,6 +409,9 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: if selector is not None: payload["cost"] = roll_up_cost([selector]) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + # Before the failure exit: the calls that did succeed still cost money and noise is still + # measured, so their usage is printed on a partial run too. + print_all_vs_all_model(model_arm, payload.get("cost")) if failed: raise SystemExit(f"{failed} of {sent} model call(s) failed " f"({model_arm['first_error']}); the partial result, with those queries " @@ -417,7 +420,6 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print(f"all-vs-all: {len(field)} skills in the field, {result['n_queries']} queries over " f"{len(queries_by_skill)} skills with queries") print_attractors(result) - print_all_vs_all_model(model_arm, payload.get("cost")) def _refuse_queries_naming_disabled(args: argparse.Namespace, field: list, diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index 3638415e1..b32f995d3 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -932,3 +932,33 @@ def test_console_counts_unscored_model_calls_when_repeats_are_set(tmp_path, caps "--model", "m", "--repeats", "3") args.func(args) assert "model: 0 tied and 3 unscored calls are not counted as picks" in capsys.readouterr().out + + +def test_a_partial_failure_still_prints_the_cost_of_the_calls_that_ran(tmp_path, capsys, + monkeypatch): + import dokimi.commands.novelty as novelty_cmd + from dokimi.metrics.usage import UsageRecord + + replies = iter([('{"pick": 1}', UsageRecord(cost_usd=0.01))]) + + def transport(prompt: str, system: str | None = None): + try: + return next(replies) + except StopIteration: + raise RuntimeError("boom") from None + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model=model, provider=provider, role=role, transport=transport)) + root = tmp_path / "catalog" + _catalog_skill(root, "perf", "profile a slow binary with perf record and report") + _catalog_skill(root, "db", "tune a slow database query with an index") + queries = _write_queries(tmp_path, {"perf": ["profile a slow program"], + "db": ["add an index"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m") + with pytest.raises(SystemExit, match="1 of 2 model call.s. failed"): + args.func(args) + out = capsys.readouterr().out + assert "OK Saved" not in out + assert any(line.startswith("cost") for line in out.splitlines()), out From a6fc22d89a85ce4abe4c37aac7eedc6e9bedb18c Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:22:38 -0700 Subject: [PATCH 07/14] test(routing): cover the estimate for a model with no list price The per-call pricing added in 1c5348be returns None for an unpriced model; no test reached that path, which left a patch-coverage gap. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_novelty_routing_cli_gaps.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 0aedd8c07..bb19f15b8 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -158,3 +158,19 @@ def tiered(model, provider, prompt, completion): assert estimate["input_tokens"] > 200_000 and estimate["n_calls"] == 6 assert estimate["expected_usd"] == pytest.approx( estimate["input_tokens"] * 1e-6 + estimate["output_tokens"] * 2e-6, rel=1e-6) + + +def test_estimate_only_gives_a_null_dollar_figure_for_a_routing_model_with_no_price( + tmp_path, capsys): + queries = tmp_path / "q.json" + queries.write_text(json.dumps(["profile a binary"]), encoding="utf-8") + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "perf", "profile a binary with perf record"), + "--competitors", _skill(tmp_path, "tbb", "parallelize loops"), + "--queries", str(queries), "--model", "foo-bar-9", "--provider", "openai", + "--estimate-only") + args.func(args) + captured = capsys.readouterr() + estimate = json.loads(captured.out) + assert estimate["n_calls"] == 1 and estimate["expected_usd"] is None + assert "expected_usd is null" in captured.err From 8611ba877b2e6ff984be98d185afb15caa041e29 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 13:42:07 -0700 Subject: [PATCH 08/14] fix(novelty): keep the all-vs-all cost line on success; ratchet for the partial exit CI on a6fc22d8 failed the growth ratchet: novelty.py was 1149 lines against a baseline of 1147. The partial-failure exit moves into exit_partial_all_vs_all in novelty_routing_report.py, so the command body shrinks. The success path keeps its original order: the summary, then the attractors, then the noise and cost lines. A test pins that the cost line still follows the saved-file line on a clean run. The baseline drops to 1146 as check_growth.py --update reports. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/novelty.py | 9 +++---- .../dokimi/commands/novelty_routing_report.py | 17 ++++++++++++- dokimi/scripts/growth_baseline.json | 2 +- .../tests/test_novelty_routing_all_vs_all.py | 25 +++++++++++++++++++ 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 88b5fd61d..84ec16e75 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -21,6 +21,7 @@ from dokimi.commands.inputs import placebo_text_for_arms_or_exit from dokimi.commands.novelty_routing_report import ( + exit_partial_all_vs_all, print_all_vs_all_model, print_attractors, print_routing_estimate, @@ -409,17 +410,13 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: if selector is not None: payload["cost"] = roll_up_cost([selector]) out.write_text(json.dumps(payload, indent=2), encoding="utf-8") - # Before the failure exit: the calls that did succeed still cost money and noise is still - # measured, so their usage is printed on a partial run too. - print_all_vs_all_model(model_arm, payload.get("cost")) if failed: - raise SystemExit(f"{failed} of {sent} model call(s) failed " - f"({model_arm['first_error']}); the partial result, with those queries " - f"left out of every rate, is in {out}") + exit_partial_all_vs_all(model_arm, payload.get("cost"), failed, sent, out) print(f"OK Saved all-vs-all routing: {out}") print(f"all-vs-all: {len(field)} skills in the field, {result['n_queries']} queries over " f"{len(queries_by_skill)} skills with queries") print_attractors(result) + print_all_vs_all_model(model_arm, payload.get("cost")) def _refuse_queries_naming_disabled(args: argparse.Namespace, field: list, diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index 15a9c777b..495319ddd 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -12,11 +12,13 @@ import argparse import json import sys -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NoReturn from dokimi.exit_codes import usage_exit if TYPE_CHECKING: + from pathlib import Path + from dokimi.novelty.routing import Candidate @@ -191,3 +193,16 @@ def print_all_vs_all_model(model_arm: dict, cost: dict | None) -> None: print_noise(model_arm) if cost is not None: print_cost(cost) + + +def exit_partial_all_vs_all(model_arm: dict, cost: dict | None, failed: int, sent: int, + out: Path) -> NoReturn: + """The failure exit of an all-vs-all run where some model calls failed. + + The usage of the calls that did succeed still costs money and noise is still measured, so the + noise and cost lines print before the exit; the partial result is already saved in ``out``. + """ + print_all_vs_all_model(model_arm, cost) + raise SystemExit(f"{failed} of {sent} model call(s) failed " + f"({model_arm['first_error']}); the partial result, with those queries " + f"left out of every rate, is in {out}") diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index c563fd472..3e1750636 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1147, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1146, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index b32f995d3..e03c346e8 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -962,3 +962,28 @@ def transport(prompt: str, system: str | None = None): out = capsys.readouterr().out assert "OK Saved" not in out assert any(line.startswith("cost") for line in out.splitlines()), out + + +def test_a_clean_all_vs_all_run_still_prints_its_cost_after_the_summary(tmp_path, capsys, + monkeypatch): + import dokimi.commands.novelty as novelty_cmd + from dokimi.metrics.usage import UsageRecord + + def transport(prompt: str, system: str | None = None): + return '{"pick": 1}', UsageRecord(cost_usd=0.01) + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model=model, provider=provider, role=role, transport=transport)) + root = tmp_path / "catalog" + _catalog_skill(root, "perf", "profile a slow binary with perf record and report") + _catalog_skill(root, "db", "tune a slow database query with an index") + queries = _write_queries(tmp_path, {"perf": ["profile a slow program"], + "db": ["add an index"]}) + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json"), + "--model", "m") + args.func(args) + lines = capsys.readouterr().out.splitlines() + saved = next(i for i, line in enumerate(lines) if line.startswith("OK Saved")) + cost = [i for i, line in enumerate(lines) if line.startswith("cost")] + assert cost and cost[-1] > saved From 0fdc2810227ceaff21256f0949ad7368359f2b4b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 18:04:57 -0700 Subject: [PATCH 09/14] fix(routing): answer review of estimate, gold and disabled-skill paths - --estimate-only refuses a run the real run would refuse (no query with a content word), with the same reason, and writes nothing. - The disabled-skill refusal reads the field once: the loader reports the names it left out, so field warnings no longer print twice. - Each all-vs-all noise row names its gold skill. - A sentence split does not break after e.g., i.e., vs. or cf. - _noise is public as repeat_noise: two modules use it. Signed-off-by: Nikolay Petrov --- dokimi/docs/skill-novelty.md | 3 +- dokimi/dokimi/commands/novelty.py | 84 ++++++++----------- .../dokimi/commands/novelty_routing_report.py | 44 ++++++++++ dokimi/dokimi/novelty/all_vs_all.py | 11 ++- dokimi/dokimi/novelty/routing.py | 37 ++++++-- dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_novelty_routing.py | 4 +- .../tests/test_novelty_routing_all_vs_all.py | 10 +++ dokimi/tests/test_novelty_routing_cli_gaps.py | 30 +++++++ dokimi/tests/test_novelty_routing_field.py | 19 +++++ 10 files changed, 177 insertions(+), 67 deletions(-) diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index ce5a38719..bd3680a01 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -648,7 +648,8 @@ Two numbers, reported side by side and never merged: fresh sample, so under `DOKIMI_RESPONSE_CACHE` a repeat is not a replay. Repeats need `--model`, and cost N times the calls. With repeats, `n_calls` is queries x repeats, `n_abstained` and `n_failed` count calls, and each `per_query` row - carries its `repeat` index. + carries its `repeat` index. In an `--all-vs-all` run each `noise` row also names its `gold` + skill, since two skills can list the same request. A single-artifact `--model` run also writes a `cost` block (calls, input and output tokens, cache read and write tokens, and `usd`, which is `null` when any call this run paid for had no known price) and prints the diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 84ec16e75..8a575a7ba 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -26,6 +26,9 @@ print_attractors, print_routing_estimate, print_routing_summary, + no_queries_reason, + refuse_estimate_without_queries, + refuse_queries_naming_disabled, routing_queries, ) from dokimi.defaults import ( @@ -301,21 +304,10 @@ def _routing_card(ref: str): return candidate, body -def _routing_field(args: argparse.Namespace, exclude: set[str], - left_out: list[str] | None = None) -> list: - """The field ``--competitors`` and ``--catalog`` name, without the names in ``exclude``. - - What was left out of the field, and why, is appended to ``left_out`` when given. - - Raises: - OSError, ValueError, RecursionError: A reference or the catalog cannot be loaded. - """ - from dokimi.novelty.routing import ( - catalog_competitors, - disables_model_invocation, - non_skill_reason, - skill_file, - ) +def _named_competitors(args: argparse.Namespace, left_out: list[str] | None, + disabled: list[str] | None) -> list: + """The ``--competitors`` references that stay in the field, each as a candidate.""" + from dokimi.novelty.routing import disables_model_invocation, non_skill_reason, skill_file competitors = [] for ref in args.competitors or []: @@ -333,13 +325,33 @@ def _routing_field(args: argparse.Namespace, exclude: set[str], "(--include-disabled keeps it)", file=sys.stderr) if left_out is not None: left_out.append(f"{ref}: disable-model-invocation") + if disabled is not None: + disabled.append(candidate.name) continue competitors.append(candidate) + return competitors + + +def _routing_field(args: argparse.Namespace, exclude: set[str], + left_out: list[str] | None = None, + disabled: list[str] | None = None) -> list: + """The field ``--competitors`` and ``--catalog`` name, without the names in ``exclude``. + + What was left out of the field, and why, is appended to ``left_out`` when given. The + names left out for disable-model-invocation are appended to ``disabled`` when given. + + Raises: + OSError, ValueError, RecursionError: A reference or the catalog cannot be loaded. + """ + from dokimi.novelty.routing import catalog_competitors + + competitors = _named_competitors(args, left_out, disabled) if args.catalog: notes: list[str] = [] catalog = catalog_competitors(args.catalog, exclude | {c.name for c in competitors}, notes, - include_disabled=args.include_disabled) + include_disabled=args.include_disabled, + disabled=disabled) for note in notes: print(f"WARNING: {note}", file=sys.stderr) if left_out is not None: @@ -362,8 +374,9 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: _check_all_vs_all_args(args) left_out: list[str] = [] + disabled: list[str] = [] try: - field = _routing_field(args, set(), left_out) + field = _routing_field(args, set(), left_out, disabled) except RoutingInputError as exc: usage_exit(str(exc)) except (OSError, ValueError, RecursionError) as exc: @@ -373,7 +386,7 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: queries_by_skill = load_queries_by_skill(args.queries_by_skill) except (OSError, UnicodeDecodeError) as exc: usage_exit(f"cannot read the queries: {exc}") - _refuse_queries_naming_disabled(args, field, queries_by_skill) + refuse_queries_naming_disabled(args, field, queries_by_skill, disabled) selector = _client(args.model, args.provider, "selector", args) if args.model else None if selector is not None: # One call per query with a content word and per repeat, each sending every card. @@ -419,30 +432,6 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print_all_vs_all_model(model_arm, payload.get("cost")) -def _refuse_queries_naming_disabled(args: argparse.Namespace, field: list, - queries_by_skill: dict[str, list[str]]) -> None: - """Refuse queries for a skill the field leaves out only because it sets disable-model-invocation. - - The generic "not in the field" refusal does not say why, so this names the disabled skills - and the flag that keeps them. Other unknown names fall through to that refusal. - """ - if args.include_disabled: - return - held = {c.name for c in field} - unknown = sorted(set(queries_by_skill) - held) - if not unknown: - return - everyone = _routing_field(argparse.Namespace(**{**vars(args), "include_disabled": True}), - set()) - disabled = [name for name in unknown if name in {c.name for c in everyone}] - if disabled: - usage_exit("the queries name skills that are not in the field: " - + ", ".join(f"`{n}`" for n in unknown) + ". " - + "; ".join(f"`{n}` sets disable-model-invocation, so the field leaves it out" - for n in disabled) - + "; pass --include-disabled to keep those skills in the field") - - def _field_record(args: argparse.Namespace, competitors: list, left_out: list[str]) -> dict: """The field rule a routing result was measured under, saved next to the result.""" return {"include_disabled": bool(args.include_disabled), @@ -515,6 +504,7 @@ def cmd_novelty_routing(args: argparse.Namespace) -> None: queries, self_derived, queries_source = routing_queries(args, body, gold) candidates = [gold, *competitors] if args.estimate_only: + refuse_estimate_without_queries(args, queries, self_derived) print_routing_estimate(args, candidates, queries) return lexical = dict(lexical_routing(candidates, queries, gold=0), @@ -555,15 +545,7 @@ def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derive queries_source: str | None = None) -> NoReturn: """Write the ``no_queries`` record, then exit saying why no query was scored.""" # Written too, so a batch run leaves a record of why it has no rate. - if not queries and self_derived: - reason = (f"no section heading or description sentence of two or more words in " - f"{args.artifact} outside code fences; headings inside fenced code are not " - "requests") - elif not queries: - reason = f"--queries {args.queries} holds no non-blank query" - else: - reason = (f"none of the {len(queries)} queries has a content word, since words " - "like a, the, how do not count") + reason = no_queries_reason(args, queries, self_derived) out.write_text(json.dumps({"artifact": args.artifact, "status": "no_queries", "reason": reason, "queries_self_derived": self_derived, "queries_source": queries_source or ("none" if self_derived else None), diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index 495319ddd..cc65e0856 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -70,6 +70,50 @@ def estimate_routing_usd(model: str, provider: str, prompts: list[int], return round(total * repeats, 6) +def no_queries_reason(args: argparse.Namespace, queries: list[str], self_derived: bool) -> str: + """Why a routing run has no query to score.""" + if not queries and self_derived: + return (f"no section heading or description sentence of two or more words in " + f"{args.artifact} outside code fences; headings inside fenced code are not " + "requests") + if not queries: + return f"--queries {args.queries} holds no non-blank query" + return (f"none of the {len(queries)} queries has a content word, since words " + "like a, the, how do not count") + + +def refuse_estimate_without_queries(args: argparse.Namespace, queries: list[str], + self_derived: bool) -> None: + """Exit as the real run would when no query has a content word, so no budget is read off it.""" + from dokimi.novelty.routing import tokenize + + if not any(tokenize(query) for query in queries): + usage_exit(f"no usable queries: {no_queries_reason(args, queries, self_derived)}. " + "Nothing is estimated, since the run would be refused") + + +def refuse_queries_naming_disabled(args: argparse.Namespace, field: list, + queries_by_skill: dict[str, list[str]], + disabled: list[str]) -> None: + """Refuse queries for a skill the field leaves out only because it sets disable-model-invocation. + + ``disabled`` is the names :func:`_routing_field` left out for that reason, so the field is + not read again. The generic "not in the field" refusal does not say why, so this names the + disabled skills and the flag that keeps them. Other unknown names fall through to that refusal. + """ + if args.include_disabled: + return + held = {c.name for c in field} + unknown = sorted(set(queries_by_skill) - held) + named = [name for name in unknown if name in disabled] + if named: + usage_exit("the queries name skills that are not in the field: " + + ", ".join(f"`{n}`" for n in unknown) + ". " + + "; ".join(f"`{n}` sets disable-model-invocation, so the field leaves it out" + for n in named) + + "; pass --include-disabled to keep those skills in the field") + + def print_routing_estimate(args: argparse.Namespace, candidates: list, queries: list[str]) -> None: """The model arm's calls, tokens and list-price USD, as JSON; nothing is sent. diff --git a/dokimi/dokimi/novelty/all_vs_all.py b/dokimi/dokimi/novelty/all_vs_all.py index 21aab97b4..78d900ff3 100644 --- a/dokimi/dokimi/novelty/all_vs_all.py +++ b/dokimi/dokimi/novelty/all_vs_all.py @@ -17,7 +17,7 @@ from pathlib import Path from .client import LLMClient -from .routing import BM25, Candidate, RoutingInputError, _noise, model_routing, tokenize +from .routing import BM25, Candidate, RoutingInputError, model_routing, repeat_noise, tokenize NO_PICK = "(none)" TIE = "(tie)" @@ -190,6 +190,7 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str model: dict[str, Counter] = {} unscored: dict[str, int] = {} groups: list[list[dict]] = [] # one per query, its repeats in order, for the noise block + golds: list[str] = [] # the gold of each group, so a noise row says which skill it is for n_failed = n_abstained = 0 first_error = None for index, name in enumerate(names): @@ -205,6 +206,7 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str gold=index, seed=seed + index, repeats=repeats) rows = routed["per_query"] groups += [rows[start:start + repeats] for start in range(0, len(rows), repeats)] + golds += [name] * len(queries) n_failed += routed["n_failed"] n_abstained += routed["n_abstained"] if first_error is None: @@ -213,9 +215,12 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str model[name] = Counter(row.get("picked") or NO_PICK for row in rows if not row.get("error")) if routed["n_failed"]: unscored[name] = unscored.get(name, 0) + routed["n_failed"] + noise = repeat_noise(groups) if repeats > 1 and groups else None + if noise is not None: + for row, gold in zip(noise["per_query"], golds, strict=True): + row["gold"] = gold return dict(_attractor_table(model, unscored, names, _tied_groups(candidates)), - model=client.model, seed=seed, repeats=repeats, - noise=_noise(groups) if repeats > 1 and groups else None, + model=client.model, seed=seed, repeats=repeats, noise=noise, n_failed=n_failed, n_abstained=n_abstained, first_error=first_error) diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 88a0a267e..86d1e1c47 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -327,7 +327,7 @@ def model_routing( "selection_rate_among_answered": ( round(hits / len(answered), 4) if answered else None ), - "noise": _noise(groups) if repeats > 1 else None, + "noise": repeat_noise(groups) if repeats > 1 else None, "per_query": picks, "note": ( "An upper bound on harness behaviour, not a harness measurement: a harness " @@ -337,7 +337,7 @@ def model_routing( } -def _noise(groups: list[list[dict]]) -> dict: +def repeat_noise(groups: list[list[dict]]) -> dict: """Agreement across the repeats of each query, and the run-to-run spread of the rate. A query's agreement is the share of its answered repeats that gave its most common @@ -426,6 +426,20 @@ def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: _SENTENCE_END = re.compile(r"(?<=[.!?])\s+") +# A period that ends an abbreviation, not a sentence: "e.g.", "i.e.", "vs.", "cf.". +_ABBREVIATION_END = re.compile(r"(?:^|\s)(?:e\.g|i\.e|vs|cf)\.$", re.IGNORECASE) + + +def _sentences(text: str) -> list[str]: + """``text`` split after each sentence-ending period, not after an abbreviation's.""" + sentences, start = [], 0 + for match in _SENTENCE_END.finditer(text): + if _ABBREVIATION_END.search(text[start:match.start()]): + continue + sentences.append(text[start:match.start()]) + start = match.end() + sentences.append(text[start:]) + return sentences def description_as_queries(description: str, *, limit: int = 40) -> list[str]: @@ -435,7 +449,7 @@ def description_as_queries(description: str, *, limit: int = 40) -> list[str]: what it is for in its description. Each sentence with two or more content words is offered, as :func:`headings_as_queries` offers headings. The same self-derived stamp applies. """ - sentences = _SENTENCE_END.split(" ".join(description.split())) + sentences = _sentences(" ".join(description.split())) return [s for s in sentences if len(tokenize(s)) >= 2][:limit] @@ -547,16 +561,18 @@ def load_candidate(ref: str) -> tuple[Candidate, str, str | None]: def catalog_competitors(root: str, exclude: set[str], notes: list[str] | None = None, *, - include_disabled: bool = False) -> list[Candidate]: + include_disabled: bool = False, + disabled: list[str] | None = None) -> list[Candidate]: """Every skill under ``root`` whose name is not in ``exclude``. The field a harness would list next to the artifact, rather than one the caller picks. Skills come from the catalog reader, so a ``skills.yaml`` manifest decides which skills the catalog has. A declared skill that cannot be loaded is left out and described in ``notes`` (when given) rather than aborting the run. A skill with - ``disable-model-invocation`` is left out too, unless ``include_disabled``. A mirror, a - byte-identical ``SKILL.md`` under another path (a plugin copy of the ``skills/`` tree), is - left out too: the copy nearest the catalog root stands for the card. + ``disable-model-invocation`` is left out too, unless ``include_disabled``; its name is + appended to ``disabled`` (when given), so a caller need not read the catalog again. A + mirror, a byte-identical ``SKILL.md`` under another path (a plugin copy of the ``skills/`` + tree), is left out too: the copy nearest the catalog root stands for the card. Raises: ValueError: ``root`` is not a directory, or its ``skills.yaml`` is invalid (with no @@ -566,7 +582,7 @@ def catalog_competitors(root: str, exclude: set[str], """ if not Path(root).is_dir(): raise ValueError(f"--catalog is not a directory: {root}") - eligible = _eligible_catalog_skills(root, exclude, notes, include_disabled) + eligible = _eligible_catalog_skills(root, exclude, notes, include_disabled, disabled) _refuse_conflicting_names(eligible) keeper = _keepers_by_copy(eligible) candidates: list[Candidate] = [] @@ -581,7 +597,8 @@ def catalog_competitors(root: str, exclude: set[str], def _eligible_catalog_skills(root: str, exclude: set[str], notes: list[str] | None, - include_disabled: bool) -> list[tuple[Skill, str]]: + include_disabled: bool, + disabled: list[str] | None = None) -> list[tuple[Skill, str]]: """Each loadable, non-excluded catalog skill with the sha256 of its ``SKILL.md``.""" from dokimi.external.catalog import read_skill_manifest @@ -598,6 +615,8 @@ def _eligible_catalog_skills(root: str, exclude: set[str], notes: list[str] | No if notes is not None: notes.append(f"left out catalog skill {skill.name!r}: " "disable-model-invocation (--include-disabled keeps it)") + if disabled is not None: + disabled.append(skill.name) continue eligible.append((skill, hashlib.sha256(skill.path.read_bytes()).hexdigest())) return eligible diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 3e1750636..9f3bc1a8f 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1146, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1128, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_routing.py b/dokimi/tests/test_novelty_routing.py index ac37d9348..d0d47296b 100644 --- a/dokimi/tests/test_novelty_routing.py +++ b/dokimi/tests/test_novelty_routing.py @@ -795,9 +795,9 @@ def test_the_modal_pick_is_the_most_frequent_not_the_first(): def test_a_single_usable_repeat_has_no_half_range(): - from dokimi.novelty.routing import _noise + from dokimi.novelty.routing import repeat_noise - noise = _noise([[{"query": "q", "repeat": 0, "picked": PERF.name, "correct": True}]]) + noise = repeat_noise([[{"query": "q", "repeat": 0, "picked": PERF.name, "correct": True}]]) assert noise["selection_rate_by_repeat"] == [1.0] assert noise["half_range"] is None diff --git a/dokimi/tests/test_novelty_routing_all_vs_all.py b/dokimi/tests/test_novelty_routing_all_vs_all.py index e03c346e8..2d22d5413 100644 --- a/dokimi/tests/test_novelty_routing_all_vs_all.py +++ b/dokimi/tests/test_novelty_routing_all_vs_all.py @@ -987,3 +987,13 @@ def transport(prompt: str, system: str | None = None): saved = next(i for i, line in enumerate(lines) if line.startswith("OK Saved")) cost = [i for i, line in enumerate(lines) if line.startswith("cost")] assert cost and cost[-1] > saved + + +def test_every_noise_row_names_the_gold_skill_it_belongs_to(): + same = "profile a slow program" + queries = {"linux-perf": [same], "mkl-advisor": [same]} + result = all_vs_all(ALL_FIELD, queries, client=_positional_client(lambda p: 1), + seed=0, repeats=2) + rows = result["model"]["noise"]["per_query"] + assert sorted(row["gold"] for row in rows) == ["linux-perf", "mkl-advisor"] + assert all(row["query"] == same for row in rows) diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index bb19f15b8..9d7921014 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -174,3 +174,33 @@ def test_estimate_only_gives_a_null_dollar_figure_for_a_routing_model_with_no_pr estimate = json.loads(captured.out) assert estimate["n_calls"] == 1 and estimate["expected_usd"] is None assert "expected_usd is null" in captured.err + + +@pytest.mark.parametrize("queries", [None, ["the a of", "how do i"]], + ids=["no-query-lines", "only-stop-words"]) +def test_estimate_only_refuses_the_run_the_real_run_refuses(tmp_path, capsys, queries): + out = tmp_path / "routing.json" + flags = ["--queries-from-headings"] + if queries is not None: + qfile = tmp_path / "q.json" + qfile.write_text(json.dumps(queries), encoding="utf-8") + flags = ["--queries", str(qfile)] + args = _parse("novelty", "routing", + "--artifact", _skill(tmp_path, "grill", "Stop.", body="# Grill\n\n## Steps\n"), + "--competitors", _skill(tmp_path, "b", "tune a database"), + *flags, "--model", "m", "--estimate-only", "--out", str(out)) + with pytest.raises(SystemExit): + args.func(args) + assert "no usable queries" in capsys.readouterr().err + assert not out.exists() + + +def test_a_sentence_with_e_g_or_i_e_is_one_description_query(): + from dokimi.novelty.routing import description_as_queries + + assert description_as_queries( + "Profile code, e.g. PyTorch training loops, and report hotspots.") == [ + "Profile code, e.g. PyTorch training loops, and report hotspots."] + assert description_as_queries( + "Tune kernels, i.e. hot loops. Report vs. baseline timings.") == [ + "Tune kernels, i.e. hot loops.", "Report vs. baseline timings."] diff --git a/dokimi/tests/test_novelty_routing_field.py b/dokimi/tests/test_novelty_routing_field.py index 2de05770d..e3b4d6975 100644 --- a/dokimi/tests/test_novelty_routing_field.py +++ b/dokimi/tests/test_novelty_routing_field.py @@ -286,3 +286,22 @@ def test_two_same_name_copies_that_differ_only_late_in_the_body_are_refused(tmp_ (copy / "SKILL.md").write_text(head + tail, encoding="utf-8") with pytest.raises(RoutingInputError, match="catalog has db at .* different content"): catalog_competitors(str(tmp_path), set(), []) + + +def test_a_disabled_skill_named_by_a_query_warns_once_and_counts_the_field_once(tmp_path, capsys): + root = tmp_path / "catalog" + _catalog_skill(root, "perf", "profile a slow binary") + (root / "skills" / "grill").mkdir(parents=True) + (root / "skills" / "grill" / "SKILL.md").write_text( + "---\nname: grill\ndescription: relentless interview about a plan\n" + "disable-model-invocation: true\n---\n\n# B\n", encoding="utf-8") + queries = tmp_path / "queries_by_skill.json" + queries.write_text(json.dumps({"perf": ["profile a slow binary"], + "grill": ["interview me about my plan"]}), encoding="utf-8") + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(tmp_path / "o.json")) + with pytest.raises(SystemExit): + args.func(args) + err = capsys.readouterr().err + assert err.count("competitors") == 1 + assert err.count("left out catalog skill") == 1 From 12f8c623755763c1fd2d8011dc4a4648895b29ab Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 31 Oct 2026 22:41:40 -0700 Subject: [PATCH 10/14] fix(routing): zero-repeat estimate sends no query; say what failed calls leave out --estimate-only without --model passed repeats=0, which still counted the prompts, so n_queries_sent was non-zero for a run that sends nothing. The failed-call message said the queries were left out of every rate. A call that failed is only left out of the model-arm counts; a query with a failed repeat is left out of the repeat-noise rates. Say both. Signed-off-by: Nikolay Petrov --- ...261031_120000_napetrov_routing-cli-gaps.md | 2 + dokimi/dokimi/commands/novelty.py | 4 +- .../dokimi/commands/novelty_routing_report.py | 5 +- dokimi/dokimi/novelty/all_vs_all.py | 2 +- dokimi/dokimi/novelty/routing.py | 4 +- dokimi/tests/test_novelty_routing_cli_gaps.py | 10 +++ .../test_novelty_routing_repeat_failures.py | 66 +++++++++++++++++++ 7 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 dokimi/tests/test_novelty_routing_repeat_failures.py diff --git a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md index d669f3469..be6b634b7 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md @@ -13,4 +13,6 @@ SPDX-License-Identifier: Apache-2.0 - **A query that names a skill left out for `disable-model-invocation` says so.** `--all-vs-all` refused it as "not in the field" with no hint of why; the message now names each such skill and points to `--include-disabled`. - **`--queries-from-headings` no longer gives up on a skill with no usable heading.** Only `##` and `###` headings of two or more content words were offered, so a skill whose headings were one word each, or a `#` title alone, had no queries (24 of 144 real skills). Its description's sentences are now the fallback, still stamped as self-derived triage, and the result records `queries_source`. A skill with neither still exits non-zero with the reason. +- **`--estimate-only` without `--model` reports no query sent** (`n_queries_sent` 0), as it makes no call. +- **A failed model call's message says what it leaves out.** Failed calls are left out of the model arm's scoring; a query with a failed repeat is also left out of the repeat-noise rates. The partial result keeps the calls that succeeded. - **`--seed` help says what it fixes.** It seeds the card order only; it does not make the model's answers repeatable, which is what `--repeats` measures. The help no longer calls the run reproducible. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 8a575a7ba..f4ed7e5f4 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -412,8 +412,8 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: sent = sum(model_arm.get("n_scored", {}).values()) + failed # contentless ones are not sent if failed: print(f"WARNING: {failed} model call(s) failed (first error: " - f"{model_arm['first_error']}); those queries are left out of " - "every rate in the model arm", file=sys.stderr) + f"{model_arm['first_error']}); failed calls are left out of model-arm scoring, " + "and a query with a failed repeat out of the repeat-noise rates", file=sys.stderr) out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index cc65e0856..31adc6bc1 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -248,5 +248,6 @@ def exit_partial_all_vs_all(model_arm: dict, cost: dict | None, failed: int, sen """ print_all_vs_all_model(model_arm, cost) raise SystemExit(f"{failed} of {sent} model call(s) failed " - f"({model_arm['first_error']}); the partial result, with those queries " - f"left out of every rate, is in {out}") + f"({model_arm['first_error']}); the partial result leaves the failed calls " + "out of model-arm scoring, and a query with a failed repeat out of the " + f"repeat-noise rates, is in {out}") diff --git a/dokimi/dokimi/novelty/all_vs_all.py b/dokimi/dokimi/novelty/all_vs_all.py index 78d900ff3..d7672f8dd 100644 --- a/dokimi/dokimi/novelty/all_vs_all.py +++ b/dokimi/dokimi/novelty/all_vs_all.py @@ -211,7 +211,7 @@ def _model_arm(candidates: list[Candidate], queries_by_skill: dict[str, list[str n_abstained += routed["n_abstained"] if first_error is None: first_error = next((r["error"] for r in rows if r.get("error")), None) - # A failed call measured nothing: it is not a pick, so it is left out of every rate. + # A failed call measured nothing: it is not a pick, so it is left out of the counts. model[name] = Counter(row.get("picked") or NO_PICK for row in rows if not row.get("error")) if routed["n_failed"]: unscored[name] = unscored.get(name, 0) + routed["n_failed"] diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 86d1e1c47..97d328b91 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -489,9 +489,9 @@ def selection_estimate(candidates: list[Candidate], queries: list[str], *, """The calls and tokens :func:`model_routing` would make, counted without making them. Every repeat sends the same prompts (see :func:`selection_prompt_tokens`). The reply is one - JSON pick, :data:`PICK_REPLY_TOKENS`. + JSON pick, :data:`PICK_REPLY_TOKENS`. Zero repeats send nothing, so no query is sent. """ - prompts = selection_prompt_tokens(candidates, queries) + prompts = selection_prompt_tokens(candidates, queries) if repeats else [] calls = len(prompts) * repeats return { "n_queries": len(queries), diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 9d7921014..ddb6e6fdc 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -133,6 +133,16 @@ def test_estimate_only_without_a_model_is_lexical_and_free(tmp_path, capsys): assert (estimate["n_calls"], estimate["input_tokens"], estimate["expected_usd"]) == (0, 0, 0.0) +def test_a_zero_repeat_estimate_sends_no_queries(): + from dokimi.novelty.routing import Candidate, selection_estimate + + field = [Candidate("perf", "profile a slow binary"), Candidate("db", "tune a database")] + estimate = selection_estimate(field, ["profile my binary", "index a table"], repeats=0) + assert estimate["n_queries"] == 2 + assert (estimate["n_queries_sent"], estimate["n_calls"], estimate["input_tokens"], + estimate["output_tokens"]) == (0, 0, 0, 0) + + def test_estimate_prices_each_call_so_a_long_context_tier_is_not_reached_by_the_run_total( tmp_path, capsys, monkeypatch): import dokimi.llm_pricing as pricing diff --git a/dokimi/tests/test_novelty_routing_repeat_failures.py b/dokimi/tests/test_novelty_routing_repeat_failures.py new file mode 100644 index 000000000..4b3aa629d --- /dev/null +++ b/dokimi/tests/test_novelty_routing_repeat_failures.py @@ -0,0 +1,66 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A repeat that fails after a good one: the good pick is scored, the failure is said so.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.novelty.client import LLMClient + + +def _parse(*argv): + return build_parser().parse_args(list(argv)) + + +def _cat(root, names_descriptions): + for name, description in names_descriptions.items(): + skill_dir = root / "skills" / name + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + f"---\nname: {name}\ndescription: {description}\n---\n\n# B\n", encoding="utf-8") + + +def _write_queries(tmp_path, by_skill): + path = tmp_path / "queries_by_skill.json" + path.write_text(json.dumps(by_skill), encoding="utf-8") + return path + + +def test_a_repeat_that_fails_after_a_good_one_keeps_the_good_pick_in_model_scoring( + tmp_path, capsys, monkeypatch): + import dokimi.commands.novelty as novelty_cmd + + answered = [] + + def transport(prompt: str, system: str | None): + if answered: + raise RuntimeError("rate limited") + answered.append(1) + return '{"pick": 1}', None + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model=model, provider=provider, role=role, transport=transport)) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary with perf", "db": "tune a slow database query"}) + queries = _write_queries(tmp_path, {"perf": ["profile it"]}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries), "--out", str(out), "--model", "m", + "--repeats", "2") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert "1 of 2 model call" in str(exc.value.code) + assert "out of model-arm scoring" in str(exc.value.code) + assert "repeat-noise rates" in str(exc.value.code) + model = json.loads(out.read_text(encoding="utf-8"))["result"]["model"] + # The good repeat is a scored call; the failed one is unscored. The query has a failed + # repeat, so it is in no repeat-noise rate. + assert model["n_scored"]["perf"] == 1 and model["n_unscored"] == {"perf": 1} + assert model["own_rate"]["perf"] is not None + assert model["noise"]["n_queries_compared"] == 0 + assert model["noise"]["selection_rate_by_repeat"] == [None, None] From 94f84fa28fc41fde54b0d896e4d279b8086d010f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 1 Nov 2026 13:27:34 +0000 Subject: [PATCH 11/14] Name the estimate's price key expected_usd in help, doc, changelog Signed-off-by: Anton Volkov --- dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md | 2 +- dokimi/docs/skill-novelty.md | 2 +- dokimi/dokimi/commands/novelty.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md index be6b634b7..72e546175 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **`novelty routing --estimate-only`** prints, as JSON, the calls, input and output tokens and the list-price `usd` that `--model` would make, from the same selection prompt the run sends, and makes no model call. Without `--model` the bill is zero. `--all-vs-all` has no estimate yet; its stderr line still says how many calls the model arm makes. +- **`novelty routing --estimate-only`** prints, as JSON, the calls, input and output tokens and the list-price `expected_usd` that `--model` would make, from the same selection prompt the run sends, and makes no model call. Without `--model` the bill is zero. `--all-vs-all` has no estimate yet; its stderr line still says how many calls the model arm makes. - **`novelty routing --all-vs-all --repeats N`** asks each query N times, as a single-artifact run does, and prints and writes the `noise` block (agreement per query, the rate of each repeat, `half_range`). The model arm's counts are then calls. `--repeats` still needs `--model`. - **`novelty routing --all-vs-all --model`** writes the same `cost` block as the single-artifact run, on stdout and in the result file. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index bd3680a01..c77d45e42 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -618,7 +618,7 @@ A catalog that holds the same `SKILL.md` under two paths (a plugin copy of its ` tree) counts it once: the copy nearest the catalog root stands for the card, and the others are named on stderr. -`--estimate-only` prints, as JSON, the calls, input and output tokens and the list-price `usd` the +`--estimate-only` prints, as JSON, the calls, input and output tokens and the list-price `expected_usd` the `--model` arm would make, and makes no model call. Without `--model` the run is lexical and the bill is zero. Input is the selection prompt sized at chars / 4, so it is a projection, not a count. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index f4ed7e5f4..7a76f562b 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -822,7 +822,7 @@ def register(sub, positive_int) -> None: "run-to-run noise floor. Needs --model; costs N times the calls") routing_p.add_argument("--estimate-only", action="store_true", dest="estimate_only", help="Print the calls, input and output tokens and the list-price " - "USD that --model would make, as JSON, and make no model call. " + "expected_usd that --model would make, as JSON, and make no model call. " "Without --model the run is lexical and the bill is zero") routing_p.add_argument("--out", default=None) _add_spend_cap(routing_p) From 81746955d6d1ac0f2113eb969ba73201095dd52d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:36:20 -0800 Subject: [PATCH 12/14] fix(routing): keep a parenthesised abbreviation inside its query; pin the estimate and the noise rows - "(e.g." and "(i.e." no longer end a sentence, so "Profile code (e.g. PyTorch training loops) and report hotspots." is one query. - A test records the prompts the run sends and checks each one's length against the estimate, so the two cannot drift apart unseen. - A test with one skill, two queries and two repeats checks that each noise row names its own query and its own gold skill. Thread: PRRT_kwDOVBg7g86yazxp Thread: PRRT_kwDOVBg7g86yauiU Thread: PRRT_kwDOVBg7g86yauiV Signed-off-by: Nikolay Petrov --- ...261031_120000_napetrov_routing-cli-gaps.md | 1 + dokimi/dokimi/novelty/routing.py | 5 ++-- dokimi/tests/test_novelty_routing_cli_gaps.py | 25 +++++++++++++++++++ .../test_novelty_routing_repeat_failures.py | 23 +++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md index 72e546175..6354e356e 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md @@ -15,4 +15,5 @@ SPDX-License-Identifier: Apache-2.0 - **`--queries-from-headings` no longer gives up on a skill with no usable heading.** Only `##` and `###` headings of two or more content words were offered, so a skill whose headings were one word each, or a `#` title alone, had no queries (24 of 144 real skills). Its description's sentences are now the fallback, still stamped as self-derived triage, and the result records `queries_source`. A skill with neither still exits non-zero with the reason. - **`--estimate-only` without `--model` reports no query sent** (`n_queries_sent` 0), as it makes no call. - **A failed model call's message says what it leaves out.** Failed calls are left out of the model arm's scoring; a query with a failed repeat is also left out of the repeat-noise rates. The partial result keeps the calls that succeeded. +- **A description with a parenthesised abbreviation is one query.** `(e.g. …)` and `(i.e. …)` no longer split the description after the abbreviation's period. - **`--seed` help says what it fixes.** It seeds the card order only; it does not make the model's answers repeatable, which is what `--repeats` measures. The help no longer calls the run reproducible. diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 97d328b91..8598fb198 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -426,8 +426,9 @@ def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: _SENTENCE_END = re.compile(r"(?<=[.!?])\s+") -# A period that ends an abbreviation, not a sentence: "e.g.", "i.e.", "vs.", "cf.". -_ABBREVIATION_END = re.compile(r"(?:^|\s)(?:e\.g|i\.e|vs|cf)\.$", re.IGNORECASE) +# A period that ends an abbreviation, not a sentence: "e.g.", "i.e.", "vs.", "cf.". It may follow +# an opening bracket or quote: "(e.g." +_ABBREVIATION_END = re.compile(r"(?:^|[\s(\[{\"'])(?:e\.g|i\.e|vs|cf)\.$", re.IGNORECASE) def _sentences(text: str) -> list[str]: diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index ddb6e6fdc..9d6009abd 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -214,3 +214,28 @@ def test_a_sentence_with_e_g_or_i_e_is_one_description_query(): assert description_as_queries( "Tune kernels, i.e. hot loops. Report vs. baseline timings.") == [ "Tune kernels, i.e. hot loops.", "Report vs. baseline timings."] + assert description_as_queries( + "Profile code (e.g. PyTorch training loops) and report hotspots.") == [ + "Profile code (e.g. PyTorch training loops) and report hotspots."] + + +def test_the_estimate_counts_the_prompts_the_run_sends(): + # The estimate and the run each build the selection prompt. Every prompt the run sends must + # have the length the estimate counts for it, so a drift between the two shows here. + from dokimi.novelty.routing import Candidate, model_routing, selection_prompt_tokens + + field = [Candidate("perf", "profile a slow binary with perf"), + Candidate("db", "tune a database index"), + Candidate("tbb", "parallelize loops with threads")] + queries = ["profile my binary", "index a table", "parallelize the loops"] + sent = [] + + class Recorder: + model = "m" + + def text(self, prompt, *, system=None, accept=None, draw=0): + sent.append(prompt) + return '{"pick": 1}' + + model_routing(field, queries, client=Recorder(), gold=0, repeats=1) + assert [len(prompt) // 4 for prompt in sent] == selection_prompt_tokens(field, queries) diff --git a/dokimi/tests/test_novelty_routing_repeat_failures.py b/dokimi/tests/test_novelty_routing_repeat_failures.py index 4b3aa629d..9fd324834 100644 --- a/dokimi/tests/test_novelty_routing_repeat_failures.py +++ b/dokimi/tests/test_novelty_routing_repeat_failures.py @@ -64,3 +64,26 @@ def transport(prompt: str, system: str | None): assert model["own_rate"]["perf"] is not None assert model["noise"]["n_queries_compared"] == 0 assert model["noise"]["selection_rate_by_repeat"] == [None, None] + + +def test_every_noise_row_of_an_all_vs_all_names_its_own_query_and_gold(tmp_path, monkeypatch): + # One skill with two queries and two repeats: each noise row must name its query and its gold. + import dokimi.commands.novelty as novelty_cmd + + def transport(prompt: str, system: str | None): + return '{"pick": 1}', None + + monkeypatch.setattr(novelty_cmd, "_client", lambda model, provider, role, args: LLMClient( + model=model, provider=provider, role=role, transport=transport)) + root = tmp_path / "catalog" + _cat(root, {"perf": "profile a slow binary with perf", "db": "tune a slow database query"}) + queries = ["profile it", "profile my binary"] + queries_file = _write_queries(tmp_path, {"perf": queries}) + out = tmp_path / "o.json" + args = _parse("novelty", "routing", "--all-vs-all", "--catalog", str(root), + "--queries-by-skill", str(queries_file), "--out", str(out), "--model", "m", + "--repeats", "2") + args.func(args) + per_query = json.loads(out.read_text(encoding="utf-8"))["result"]["model"]["noise"]["per_query"] + assert sorted((row["query"], row["gold"]) for row in per_query) == [ + ("profile it", "perf"), ("profile my binary", "perf")] From 66341f4c43a3d44a270710d667ec243bc0dea914 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 1 Nov 2026 09:46:53 -0800 Subject: [PATCH 13/14] test(routing): the no-heading shape falls back to its description The round changed --queries-from-headings: a skill with no usable heading now takes its description's sentences as triage queries (changelog bullet), so the door-check shape runs instead of refusing. The shape test still expected the refusal from intel/main. A skill with neither still refuses; test_novelty_routing_cli_gaps covers that case. Thread: PRRT_kwDOVBg7g86yauiV Signed-off-by: Nikolay Petrov --- dokimi/tests/test_skill_repo_shapes.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py index 0f01d3e0b..64cbd514e 100644 --- a/dokimi/tests/test_skill_repo_shapes.py +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -220,16 +220,18 @@ def test_all_vs_all_names_the_flag_when_queries_name_a_disabled_skill(shape_path assert code == 0 -def test_headings_as_queries_on_a_skill_with_no_headings_is_a_usage_refusal(shape_paths, tmp_path, - capsys): +def test_headings_as_queries_on_a_skill_with_no_headings_falls_back_to_its_description( + shape_paths, tmp_path, capsys): root, _, _ = shape_paths build_shape("no_usable_headings", root) artifact = root / "skills" / "door-check" # its only body line is prose, no heading out_file = tmp_path / "routing.json" argv = ["novelty", "routing", "--artifact", f"skill:{artifact}", "--catalog", str(root), "--queries-from-headings", "--out", str(out_file)] - code, _, err = _run(argv, capsys) - assert code == 2 and "no usable queries" in err and "headings" in err - # The refusal still leaves a record, so a batch run shows why it has no rate. + code, _, _ = _run(argv, capsys) + # No heading to offer, so the description's sentences are the triage queries (stamped as + # self-derived); a skill with neither still refuses, as in the cli-gaps tests. + assert code == 0 record = json.loads(out_file.read_text(encoding="utf-8")) - assert record["status"] == "no_queries" and record["queries_self_derived"] is True + assert record["status"] == "ok" and record["queries_source"] == "description" + assert record["queries_self_derived"] is True From 91f59371e5ac106675317bf59c3f3c730f1a51c5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 2 Nov 2026 09:44:47 -0800 Subject: [PATCH 14/14] fix(routing): drop the description fallback; a skill with no heading is refused A skill with no usable heading is refused with no_queries again, as on main. Its description's sentences are not triage queries: each scores near 1.0 against its own card. Removes description_as_queries and self_derived_queries, the abbreviation split, the description help text, docs and changelog bullet. Restores test_skill_repo_shapes from main and adds a refusal test. Growth baseline lowered for novelty.py. Signed-off-by: Nikolay Petrov --- ...261031_120000_napetrov_routing-cli-gaps.md | 3 +- dokimi/docs/skill-novelty.md | 6 +- dokimi/dokimi/commands/novelty.py | 10 ++- .../dokimi/commands/novelty_routing_report.py | 27 ++++---- dokimi/dokimi/novelty/routing.py | 43 ------------ dokimi/scripts/growth_baseline.json | 2 +- dokimi/tests/test_novelty_routing_cli_gaps.py | 65 +++++-------------- dokimi/tests/test_skill_repo_shapes.py | 14 ++-- 8 files changed, 42 insertions(+), 128 deletions(-) diff --git a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md index 6354e356e..6d280ada7 100644 --- a/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md +++ b/dokimi/changelog.d/20261031_120000_napetrov_routing-cli-gaps.md @@ -12,8 +12,7 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed - **A query that names a skill left out for `disable-model-invocation` says so.** `--all-vs-all` refused it as "not in the field" with no hint of why; the message now names each such skill and points to `--include-disabled`. -- **`--queries-from-headings` no longer gives up on a skill with no usable heading.** Only `##` and `###` headings of two or more content words were offered, so a skill whose headings were one word each, or a `#` title alone, had no queries (24 of 144 real skills). Its description's sentences are now the fallback, still stamped as self-derived triage, and the result records `queries_source`. A skill with neither still exits non-zero with the reason. +- **`--queries-from-headings` names its source.** The result records `queries_source` (`headings`, or `none` when no heading gives a query). A skill with no usable heading still exits non-zero with the reason; its description is not used as a query. - **`--estimate-only` without `--model` reports no query sent** (`n_queries_sent` 0), as it makes no call. - **A failed model call's message says what it leaves out.** Failed calls are left out of the model arm's scoring; a query with a failed repeat is also left out of the repeat-noise rates. The partial result keeps the calls that succeeded. -- **A description with a parenthesised abbreviation is one query.** `(e.g. …)` and `(i.e. …)` no longer split the description after the abbreviation's period. - **`--seed` help says what it fixes.** It seeds the card order only; it does not make the model's answers repeatable, which is what `--repeats` measures. The help no longer calls the run reproducible. diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index c77d45e42..12f9ed057 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -710,9 +710,9 @@ Three properties worth knowing before quoting either: `queries_self_derived: true` into the result; corpus-wide those self-derived queries gave recall@1 0.225, and a skill's own bundled `evals/` questions gave 0.667. Headings inside fenced code blocks are not offered as queries. Only `##` and `###` headings of two or more - content words count; a skill whose headings give none falls back to the sentences of its - description with two or more content words, and `queries_source` (`headings`, `description` or - `none`) says which was used. A run with no usable query exits non-zero, + content words count, and a skill whose headings give none has no self-derived query: its + description is not used as one. `queries_source` (`headings` or `none`) says what was used. + A run with no usable query exits non-zero, says why, and still writes the result with `"status": "no_queries"`. Every rate line prints its `n_queries` (and `triage: self-derived` when it is self-derived), and fewer than five scored queries draw a warning. diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 7a76f562b..21834ec21 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -501,7 +501,7 @@ def cmd_novelty_routing(args: argparse.Namespace) -> None: except RoutingInputError as exc: usage_exit(str(exc)) - queries, self_derived, queries_source = routing_queries(args, body, gold) + queries, self_derived, queries_source = routing_queries(args, body) candidates = [gold, *competitors] if args.estimate_only: refuse_estimate_without_queries(args, queries, self_derived) @@ -801,11 +801,9 @@ def register(sub, positive_int) -> None: "should select this artifact, written independently of it") routing_p.add_argument("--queries-from-headings", action="store_true", dest="queries_from_headings", - help="Derive queries from the artifact's own ## and ### headings of " - "two or more words, or, when it has none, from the sentences of " - "its description. Triage only: the artifact then defines both " - "the problem scope and its own relevance, and the result is " - "stamped as such") + help="Derive queries from the artifact's own headings. Triage " + "only: the artifact then defines both the problem scope and " + "its own relevance, and the result is stamped as such") routing_p.add_argument("--model", default=None, help="Also ask this model to pick from the same cards. Optional: " "without it only the free lexical floor is computed") diff --git a/dokimi/dokimi/commands/novelty_routing_report.py b/dokimi/dokimi/commands/novelty_routing_report.py index 31adc6bc1..18e15beec 100644 --- a/dokimi/dokimi/commands/novelty_routing_report.py +++ b/dokimi/dokimi/commands/novelty_routing_report.py @@ -19,17 +19,14 @@ if TYPE_CHECKING: from pathlib import Path - from dokimi.novelty.routing import Candidate - -def routing_queries(args: argparse.Namespace, body: str, - gold: Candidate) -> tuple[list[str], bool, str | None]: +def routing_queries(args: argparse.Namespace, body: str) -> tuple[list[str], bool, str | None]: """The queries of one routing run: ``(queries, self_derived, queries_source)``. - ``--queries`` is read as written. ``--queries-from-headings`` derives them from the artifact - itself and names the source: ``headings``, ``description``, or ``none`` when neither gives any. + ``--queries`` is read as written. ``--queries-from-headings`` derives them from the artifact's + own headings; ``queries_source`` is ``headings``, or ``none`` when no heading gives any. """ - from dokimi.novelty.routing import RoutingInputError, load_queries, self_derived_queries + from dokimi.novelty.routing import RoutingInputError, headings_as_queries, load_queries if args.queries: try: @@ -39,15 +36,14 @@ def routing_queries(args: argparse.Namespace, body: str, if not args.queries_from_headings: usage_exit("pass --queries with requests written independently of the artifact, or " "--queries-from-headings for a self-derived triage run") - queries, source = self_derived_queries(body, gold.description) - if source == "none": + queries = headings_as_queries(body) + if not queries: # Nothing was derived, so there is nothing to label; the refusal that follows says why. - return queries, True, source - print(f"WARNING: queries derived from the artifact's own " - f"{'headings' if source == 'headings' else 'description'}. The artifact " + return queries, True, "none" + print("WARNING: queries derived from the artifact's own headings. The artifact " "then defines both the problem scope and its own relevance, so this is triage, " "not a routing rate", file=sys.stderr) - return queries, True, source + return queries, True, "headings" def estimate_routing_usd(model: str, provider: str, prompts: list[int], @@ -73,9 +69,8 @@ def estimate_routing_usd(model: str, provider: str, prompts: list[int], def no_queries_reason(args: argparse.Namespace, queries: list[str], self_derived: bool) -> str: """Why a routing run has no query to score.""" if not queries and self_derived: - return (f"no section heading or description sentence of two or more words in " - f"{args.artifact} outside code fences; headings inside fenced code are not " - "requests") + return (f"no section heading of two or more words outside code fences in " + f"{args.artifact}; headings inside fenced code are not requests") if not queries: return f"--queries {args.queries} holds no non-blank query" return (f"none of the {len(queries)} queries has a content word, since words " diff --git a/dokimi/dokimi/novelty/routing.py b/dokimi/dokimi/novelty/routing.py index 8598fb198..5616f3fe2 100644 --- a/dokimi/dokimi/novelty/routing.py +++ b/dokimi/dokimi/novelty/routing.py @@ -425,49 +425,6 @@ def headings_as_queries(body: str, *, limit: int = 40) -> list[str]: return [h for h in headings if len(tokenize(h)) >= 2][:limit] -_SENTENCE_END = re.compile(r"(?<=[.!?])\s+") -# A period that ends an abbreviation, not a sentence: "e.g.", "i.e.", "vs.", "cf.". It may follow -# an opening bracket or quote: "(e.g." -_ABBREVIATION_END = re.compile(r"(?:^|[\s(\[{\"'])(?:e\.g|i\.e|vs|cf)\.$", re.IGNORECASE) - - -def _sentences(text: str) -> list[str]: - """``text`` split after each sentence-ending period, not after an abbreviation's.""" - sentences, start = [], 0 - for match in _SENTENCE_END.finditer(text): - if _ABBREVIATION_END.search(text[start:match.start()]): - continue - sentences.append(text[start:match.start()]) - start = match.end() - sentences.append(text[start:]) - return sentences - - -def description_as_queries(description: str, *, limit: int = 40) -> list[str]: - """Sentences of a description, for triage when the artifact has no usable heading. - - A skill whose body is a ``#`` title and no ``##`` heading of two or more words still says - what it is for in its description. Each sentence with two or more content words is - offered, as :func:`headings_as_queries` offers headings. The same self-derived stamp applies. - """ - sentences = _sentences(" ".join(description.split())) - return [s for s in sentences if len(tokenize(s)) >= 2][:limit] - - -def self_derived_queries(body: str, description: str) -> tuple[list[str], str]: - """The triage queries of an artifact and where they came from. - - Headings are preferred; the description's sentences are the fallback. Returns - ``(queries, source)`` with ``source`` ``"headings"``, ``"description"``, or ``"none"`` when - neither gives a query of two or more content words. - """ - headings = headings_as_queries(body) - if headings: - return headings, "headings" - sentences = description_as_queries(description) - return sentences, "description" if sentences else "none" - - # A selection reply is one JSON object, ``{"pick": N}``: a few tokens. PICK_REPLY_TOKENS = 10 diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 9f3bc1a8f..ffeed74bb 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1128, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1126, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 899, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1093, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_routing_cli_gaps.py b/dokimi/tests/test_novelty_routing_cli_gaps.py index 9d6009abd..07c5f27d4 100644 --- a/dokimi/tests/test_novelty_routing_cli_gaps.py +++ b/dokimi/tests/test_novelty_routing_cli_gaps.py @@ -25,7 +25,17 @@ def _skill(tmp_path, name, description, body="# B\n\n## A section here\n"): return f"skill:{root}" -def test_a_skill_with_no_usable_heading_is_queried_by_its_description_sentences(tmp_path, capsys): +def test_attractor_line_counts_calls_when_the_model_arm_repeats(capsys): + from dokimi.commands.novelty_routing_report import print_attractors + + row = {"skill": "perf", "wrongly_picked": 3, "sibling_queries": 6, "from": ["db"]} + print_attractors({"tied_groups": [], "model": {"attractors": [row], "confusion": {}, + "n_unscored": {}, "repeats": 3}}) + assert "picked for 3 of 6 sibling calls" in capsys.readouterr().out + + +def test_a_skill_with_no_usable_heading_is_refused_even_with_a_description(tmp_path, capsys): + # The description is not a fallback: only headings are triage queries. out = tmp_path / "routing.json" args = _parse("novelty", "routing", "--artifact", _skill(tmp_path, "grill", "Grill the user about a plan. " @@ -33,22 +43,12 @@ def test_a_skill_with_no_usable_heading_is_queried_by_its_description_sentences( body="# Grill\n\n## Steps\n"), "--competitors", _skill(tmp_path, "b", "tune a database"), "--queries-from-headings", "--out", str(out)) - args.func(args) + with pytest.raises(SystemExit): + args.func(args) payload = json.loads(out.read_text(encoding="utf-8")) - assert payload["status"] == "ok" - assert payload["queries_self_derived"] is True # still triage, still stamped - assert payload["queries_source"] == "description" - assert payload["lexical"]["n_queries_given"] == 2 - assert "description" in capsys.readouterr().err - - -def test_attractor_line_counts_calls_when_the_model_arm_repeats(capsys): - from dokimi.commands.novelty_routing_report import print_attractors - - row = {"skill": "perf", "wrongly_picked": 3, "sibling_queries": 6, "from": ["db"]} - print_attractors({"tied_groups": [], "model": {"attractors": [row], "confusion": {}, - "n_unscored": {}, "repeats": 3}}) - assert "picked for 3 of 6 sibling calls" in capsys.readouterr().out + assert payload["status"] == "no_queries" + assert payload["queries_source"] == "none" + assert "no section heading" in payload["reason"] def test_no_derived_queries_refuses_without_claiming_a_derivation(tmp_path, capsys): @@ -64,25 +64,6 @@ def test_no_derived_queries_refuses_without_claiming_a_derivation(tmp_path, caps assert json.loads(out.read_text(encoding="utf-8"))["status"] == "no_queries" -def test_description_sentences_are_queries_only_with_two_content_words(): - from dokimi.novelty.routing import description_as_queries - - assert description_as_queries( - "Stop. Grill the user about a plan. Use when stress-testing\na design!" - ) == ["Grill the user about a plan.", "Use when stress-testing a design!"] - - -def test_headings_are_preferred_to_the_description_when_both_give_queries(): - from dokimi.novelty.routing import self_derived_queries - - body = "# Title\n\n## Profile a binary\n" - assert self_derived_queries(body, "Tune a database. Index a table.") == ( - ["Profile a binary"], "headings") - assert self_derived_queries("# Title only\n", "Tune a database.") == ( - ["Tune a database."], "description") - assert self_derived_queries("# Title only\n", "Stop.") == ([], "none") - - def test_the_seed_help_says_what_it_fixes_and_does_not_call_the_run_reproducible(capsys): with pytest.raises(SystemExit): _parse("novelty", "routing", "--help") @@ -205,20 +186,6 @@ def test_estimate_only_refuses_the_run_the_real_run_refuses(tmp_path, capsys, qu assert not out.exists() -def test_a_sentence_with_e_g_or_i_e_is_one_description_query(): - from dokimi.novelty.routing import description_as_queries - - assert description_as_queries( - "Profile code, e.g. PyTorch training loops, and report hotspots.") == [ - "Profile code, e.g. PyTorch training loops, and report hotspots."] - assert description_as_queries( - "Tune kernels, i.e. hot loops. Report vs. baseline timings.") == [ - "Tune kernels, i.e. hot loops.", "Report vs. baseline timings."] - assert description_as_queries( - "Profile code (e.g. PyTorch training loops) and report hotspots.") == [ - "Profile code (e.g. PyTorch training loops) and report hotspots."] - - def test_the_estimate_counts_the_prompts_the_run_sends(): # The estimate and the run each build the selection prompt. Every prompt the run sends must # have the length the estimate counts for it, so a drift between the two shows here. diff --git a/dokimi/tests/test_skill_repo_shapes.py b/dokimi/tests/test_skill_repo_shapes.py index 64cbd514e..0f01d3e0b 100644 --- a/dokimi/tests/test_skill_repo_shapes.py +++ b/dokimi/tests/test_skill_repo_shapes.py @@ -220,18 +220,16 @@ def test_all_vs_all_names_the_flag_when_queries_name_a_disabled_skill(shape_path assert code == 0 -def test_headings_as_queries_on_a_skill_with_no_headings_falls_back_to_its_description( - shape_paths, tmp_path, capsys): +def test_headings_as_queries_on_a_skill_with_no_headings_is_a_usage_refusal(shape_paths, tmp_path, + capsys): root, _, _ = shape_paths build_shape("no_usable_headings", root) artifact = root / "skills" / "door-check" # its only body line is prose, no heading out_file = tmp_path / "routing.json" argv = ["novelty", "routing", "--artifact", f"skill:{artifact}", "--catalog", str(root), "--queries-from-headings", "--out", str(out_file)] - code, _, _ = _run(argv, capsys) - # No heading to offer, so the description's sentences are the triage queries (stamped as - # self-derived); a skill with neither still refuses, as in the cli-gaps tests. - assert code == 0 + code, _, err = _run(argv, capsys) + assert code == 2 and "no usable queries" in err and "headings" in err + # The refusal still leaves a record, so a batch run shows why it has no rate. record = json.loads(out_file.read_text(encoding="utf-8")) - assert record["status"] == "ok" and record["queries_source"] == "description" - assert record["queries_self_derived"] is True + assert record["status"] == "no_queries" and record["queries_self_derived"] is True